diff --git a/README.md b/README.md index a8b3a9d08..59ec0d1c1 100644 --- a/README.md +++ b/README.md @@ -28,13 +28,13 @@ See [Quick start](https://open.oceanbase.com/quickStart) to try out OceanBase Da ## System architecture -## ![image.png](https://cdn.nlark.com/yuque/0/2022/png/106206/1660643534074-2649c2e4-473a-4d07-8021-d8d1a2b2da49.png#clientId=u5d9acd3a-25ad-4&crop=0&crop=0&crop=1&crop=1&from=paste&height=1101&id=u46f6dd09&margin=%5Bobject%20Object%5D&name=image.png&originHeight=1101&originWidth=1746&originalType=binary&ratio=1&rotation=0&showTitle=false&size=130476&status=done&style=none&taskId=ud87a5d0f-5140-45f2-9274-c8375b3c7c0&title=&width=1746) +![image.png](https://cdn.nlark.com/yuque/0/2022/png/25820454/1667369873624-c1707034-471a-4f79-980f-6d1760dac8eb.png) ## Roadmap -![image.png](https://cdn.nlark.com/yuque/0/2022/png/106206/1660643534293-a5f53258-a9ac-462c-b9fd-9832901853c2.png#clientId=u5d9acd3a-25ad-4&crop=0&crop=0&crop=1&crop=1&from=paste&height=352&id=u0f71535e&margin=%5Bobject%20Object%5D&name=image.png&originHeight=528&originWidth=1683&originalType=binary&ratio=1&rotation=0&showTitle=false&size=719961&status=done&style=none&taskId=u676e97dd-1309-42b9-b380-a423c27199c&title=&width=1122) +![image.png](https://cdn.nlark.com/yuque/0/2022/png/25820454/1667369873613-44957682-76fe-42c2-b4c7-9356ed5b35f0.png) -Link: [3.1.5 function list](https://github.com/oceanbase/oceanbase/milestone/6) +Link: [4.0.0 function list](https://github.com/oceanbase/oceanbase/milestone/3) ## Case study For our success stories, see [Success stories](https://www.oceanbase.com/en/customer/home). diff --git a/deps/init/oceanbase.el7.aarch64.deps b/deps/init/oceanbase.el7.aarch64.deps index bca2d3c2b..d3f7708bf 100644 --- a/deps/init/oceanbase.el7.aarch64.deps +++ b/deps/init/oceanbase.el7.aarch64.deps @@ -40,7 +40,7 @@ obdevtools-gcc9-9.3.0-52022092914.el7.aarch64.rpm obdevtools-llvm-11.0.1-312022092921.el7.aarch64.rpm [tools-deps] -devdeps-oblogmsg-1.0-32022100420.el7.aarch64.rpm +devdeps-oblogmsg-1.0-42022101823.el7.aarch64.rpm devdeps-rocksdb-6.22.1.1-52022100420.el7.aarch64.rpm [test-utils] diff --git a/deps/init/oceanbase.el7.x86_64.deps b/deps/init/oceanbase.el7.x86_64.deps index d3e74f669..aca074a85 100644 --- a/deps/init/oceanbase.el7.x86_64.deps +++ b/deps/init/oceanbase.el7.x86_64.deps @@ -36,7 +36,7 @@ obdevtools-gcc9-9.3.0-52022092914.el7.x86_64.rpm obdevtools-llvm-11.0.1-312022092921.el7.x86_64.rpm [tools-deps] -devdeps-oblogmsg-1.0-32022100420.el7.x86_64.rpm +devdeps-oblogmsg-1.0-42022101823.el7.x86_64.rpm devdeps-rocksdb-6.22.1.1-52022100420.el7.x86_64.rpm [test-utils] diff --git a/deps/init/oceanbase.el8.aarch64.deps b/deps/init/oceanbase.el8.aarch64.deps index f3f3ccc2e..499f98b72 100644 --- a/deps/init/oceanbase.el8.aarch64.deps +++ b/deps/init/oceanbase.el8.aarch64.deps @@ -35,7 +35,7 @@ obdevtools-gcc9-9.3.0-52022092914.el8.aarch64.rpm obdevtools-llvm-11.0.1-312022092921.el8.aarch64.rpm [tools-deps] -devdeps-oblogmsg-1.0-32022100420.el8.aarch64.rpm +devdeps-oblogmsg-1.0-42022101823.el8.aarch64.rpm devdeps-rocksdb-6.22.1.1-52022100420.el8.aarch64.rpm [test-utils] diff --git a/deps/init/oceanbase.el8.x86_64.deps b/deps/init/oceanbase.el8.x86_64.deps index b0e4891c3..a374e3d01 100644 --- a/deps/init/oceanbase.el8.x86_64.deps +++ b/deps/init/oceanbase.el8.x86_64.deps @@ -36,7 +36,7 @@ obdevtools-gcc9-9.3.0-52022092914.el8.x86_64.rpm obdevtools-llvm-11.0.1-312022092921.el8.x86_64.rpm [tools-deps] -devdeps-oblogmsg-1.0-32022100420.el8.x86_64.rpm +devdeps-oblogmsg-1.0-42022101823.el8.x86_64.rpm devdeps-rocksdb-6.22.1.1-52022100420.el8.x86_64.rpm [test-utils] diff --git a/deps/oblib/src/CMakeLists.txt b/deps/oblib/src/CMakeLists.txt index 695bfd27f..531c38f67 100644 --- a/deps/oblib/src/CMakeLists.txt +++ b/deps/oblib/src/CMakeLists.txt @@ -45,7 +45,7 @@ if (OB_USE_CLANG) if (ENABLE_SANITY) target_compile_options(oblib_base_base INTERFACE - "$<$:SHELL:-Xclang -load -Xclang ${CMAKE_CURRENT_SOURCE_DIR}/lib/alloc/libsanity.so>" + "$<$:SHELL:-Xclang -load -Xclang ${CMAKE_CURRENT_SOURCE_DIR}/lib/alloc/libsanity_${ARCHITECTURE}.so>" -fno-builtin-memset -fno-builtin-bzero -fno-builtin-memcpy diff --git a/deps/oblib/src/lib/alloc/libsanity_aarch64.so b/deps/oblib/src/lib/alloc/libsanity_aarch64.so new file mode 100755 index 000000000..fa91e73f3 Binary files /dev/null and b/deps/oblib/src/lib/alloc/libsanity_aarch64.so differ diff --git a/deps/oblib/src/lib/alloc/libsanity.so b/deps/oblib/src/lib/alloc/libsanity_x86_64.so similarity index 100% rename from deps/oblib/src/lib/alloc/libsanity.so rename to deps/oblib/src/lib/alloc/libsanity_x86_64.so diff --git a/deps/oblib/src/lib/allocator/ob_slice_alloc.h b/deps/oblib/src/lib/allocator/ob_slice_alloc.h index dee840ae9..4df8e3b07 100644 --- a/deps/oblib/src/lib/allocator/ob_slice_alloc.h +++ b/deps/oblib/src/lib/allocator/ob_slice_alloc.h @@ -288,18 +288,21 @@ public: blk_ref_[ObBlockSlicer::hash((uint64_t)old_blk) % MAX_REF_NUM].sync(); if (old_blk->release()) { blk_list_.add(&old_blk->dlink_); - if (old_blk->recycle()) { - destroy_block(old_blk); - } else { - _LIB_LOG(ERROR, "there was memory leak, stock=%d, remain=%d", old_blk->stock(), old_blk->remain()); - } } } } ObDLink* dlink = nullptr; - if (OB_NOT_NULL(dlink = blk_list_.top())) { + dlink = blk_list_.top(); + while (OB_NOT_NULL(dlink)) { Block* blk = CONTAINER_OF(dlink, Block, dlink_); - _LIB_LOG(ERROR, "there was memory leak, stock=%d, remain=%d", blk->stock(), blk->remain()); + if (blk->recycle()) { + destroy_block(blk); + dlink = blk_list_.top(); + } else { + _LIB_LOG(ERROR, "there was memory leak, stock=%d, total=%d, remain=%d" + , blk->stock(), blk->total(), blk->remain()); + dlink = nullptr; // break + } } tmallocator_ = NULL; bsize_ = 0; diff --git a/deps/oblib/src/lib/ash/ob_active_session_guard.h b/deps/oblib/src/lib/ash/ob_active_session_guard.h index 56d88682e..50a8c097b 100644 --- a/deps/oblib/src/lib/ash/ob_active_session_guard.h +++ b/deps/oblib/src/lib/ash/ob_active_session_guard.h @@ -31,6 +31,7 @@ public: tenant_id_(0), user_id_(0), session_id_(0), + plan_id_(0), sample_time_(0), event_no_(0), wait_time_(0), @@ -75,6 +76,7 @@ public: uint64_t tenant_id_; uint64_t user_id_; uint64_t session_id_; + uint64_t plan_id_; int64_t sample_time_; // sample time int64_t event_no_; // wait event no uint64_t wait_time_; // how long the event waited in total diff --git a/deps/oblib/src/lib/resource/achunk_mgr.cpp b/deps/oblib/src/lib/resource/achunk_mgr.cpp index 16f1eb80f..214a13416 100644 --- a/deps/oblib/src/lib/resource/achunk_mgr.cpp +++ b/deps/oblib/src/lib/resource/achunk_mgr.cpp @@ -130,7 +130,7 @@ void *AChunkMgr::low_alloc(const uint64_t size, const bool can_use_huge_page, bo void *ptr = nullptr; huge_page_used = false; const int prot = PROT_READ | PROT_WRITE; - const int flags = MAP_PRIVATE | MAP_ANONYMOUS | (SANITY_BOOL_EXPR(alloc_shadow) ? MAP_FIXED : 0); + int flags = MAP_PRIVATE | MAP_ANONYMOUS | (SANITY_BOOL_EXPR(alloc_shadow) ? MAP_FIXED : 0); int huge_flags = flags; #ifdef MAP_HUGETLB if (OB_LIKELY(can_use_huge_page)) { @@ -143,8 +143,11 @@ void *AChunkMgr::low_alloc(const uint64_t size, const bool can_use_huge_page, bo if (SANITY_BOOL_EXPR(alloc_shadow)) { int64_t new_addr = ATOMIC_FAA(&global_canonical_addr, size); if (!SANITY_ADDR_IN_RANGE((void*)new_addr)) { - LOG_ERROR("sanity address exhausted", K(errno), KP(new_addr)); + LOG_WARN("sanity address exhausted", K(errno), KP(new_addr)); + ATOMIC_FAA(&global_canonical_addr, -size); ptr = NULL; // let it goon, it means no shadow, same as out of checker! + // in aarch64, mmap will return EPERM error when NULL address and MAP_FIXED are privided at the same time + flags &= ~MAP_FIXED; } else { ptr = (void*)new_addr; } diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index 438fdbf1b..30a31ff65 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -444,6 +444,7 @@ PCODE_DEF(OB_CANCEL_DDL_TASK, 0x4A3) PCODE_DEF(OB_HA_FETCH_SSTABLE_MACRO_INFO, 0x4A4) PCODE_DEF(OB_HA_FETCH_LS_MEMBER_LIST, 0x4A5) PCODE_DEF(OB_HA_FETCH_LS_META_INFO, 0x4A6) +PCODE_DEF(OB_NOTIFY_CREATE_TENANT_USER_LS, 0x4A7) // sql, including executor // sql diff --git a/src/logservice/CMakeLists.txt b/src/logservice/CMakeLists.txt index 48a5ecbda..e8062e49d 100644 --- a/src/logservice/CMakeLists.txt +++ b/src/logservice/CMakeLists.txt @@ -130,7 +130,6 @@ ob_set_subtarget(ob_logservice palf palf/palf_handle_impl.cpp palf/palf_handle_impl_guard.cpp palf/log_block_pool_interface.cpp - palf/scn.cpp palf/palf_options.cpp palf/log_mode_mgr.cpp ) diff --git a/src/logservice/applyservice/ob_log_apply_service.cpp b/src/logservice/applyservice/ob_log_apply_service.cpp index 16c11db7b..d1443a5c1 100644 --- a/src/logservice/applyservice/ob_log_apply_service.cpp +++ b/src/logservice/applyservice/ob_log_apply_service.cpp @@ -14,7 +14,7 @@ #include "logservice/ob_log_handler.h" #include "logservice/ob_ls_adapter.h" #include "logservice/palf/palf_env.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/rc/ob_tenant_base.h" #include "share/ob_thread_mgr.h" #include "storage/tx_storage/ob_ls_service.h" @@ -391,7 +391,7 @@ int ObApplyStatus::push_append_cb(AppendCb *cb) LSN palf_committed_end_lsn; const LSN cb_lsn = cb->__get_lsn(); const SCN cb_scn = cb->__get_scn(); - const uint64_t cb_sign = cb_scn.get_val_for_lsn_allocator(); + const uint64_t cb_sign = cb_scn.get_val_for_logservice(); int64_t thread_index = cb_sign & (APPLY_TASK_QUEUE_SIZE - 1); ObLink *link = AppendCb::__get_member_address(cb); cb_queues_[thread_index].inc_total_submit_cb_cnt(); @@ -676,7 +676,7 @@ int ObApplyStatus::get_min_unapplied_scn(SCN &scn) } else if (FOLLOWER == role_) { palf::LSN palf_end_lsn; palf::LSN apply_end_lsn; - palf::SCN palf_end_scn; + SCN palf_end_scn; bool is_done = false; if (OB_FAIL(is_apply_done(is_done, apply_end_lsn))) { CLOG_LOG(WARN, "check is_apply_done failed", K(ret), KPC(this)); @@ -720,7 +720,7 @@ int ObApplyStatus::get_min_unapplied_scn(SCN &scn) CLOG_LOG(ERROR, "max_applied_cb_scn_ larger than last_check_scn_, unexpected", K(ret), KPC(this)); } if (OB_SUCC(ret) && max_applied_cb_scn_.is_valid()) { - scn = palf::SCN::plus(max_applied_cb_scn_, 1); + scn = SCN::plus(max_applied_cb_scn_, 1); } CLOG_LOG(TRACE, "get_min_unapplied_scn finish", K(ret), KPC(this), K(scn)); if (palf_reach_time_interval(5 * 1000 * 1000, get_info_debug_time_)) { diff --git a/src/logservice/applyservice/ob_log_apply_service.h b/src/logservice/applyservice/ob_log_apply_service.h index 67a791760..1e85141fd 100644 --- a/src/logservice/applyservice/ob_log_apply_service.h +++ b/src/logservice/applyservice/ob_log_apply_service.h @@ -18,15 +18,18 @@ #include "lib/thread/ob_thread_lease.h" #include "logservice/palf/palf_callback.h" #include "logservice/palf/palf_handle.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_ls_id.h" namespace oceanbase { +namespace share +{ +class SCN; +} namespace palf { class PalfEnv; -class SCN; } namespace logservice { @@ -163,7 +166,7 @@ public: int unregister_file_size_cb(); void close_palf_handle(); //最大连续回调位点 - int get_min_unapplied_scn(palf::SCN &scn); + int get_min_unapplied_scn(share::SCN &scn); int stat(LSApplyStat &stat) const; int handle_drop_cb(); TO_STRING_KV(K(ls_id_), @@ -174,7 +177,7 @@ public: K(max_applied_cb_scn_)); private: int submit_task_to_apply_service_(ObApplyServiceTask &task); - int check_and_update_max_applied_scn_(const palf::SCN scn); + int check_and_update_max_applied_scn_(const share::SCN scn); int update_last_check_scn_(); int handle_drop_cb_queue_(ObApplyServiceQueueTask &cb_queue); int switch_to_follower_(); @@ -185,7 +188,7 @@ private: int64_t &cb_first_handle_time, int64_t &cb_start_time); void statistics_cb_cost_(const palf::LSN &lsn, - const palf::SCN &scn, + const share::SCN &scn, const int64_t append_start_time, const int64_t append_finish_time, const int64_t cb_first_handle_time, @@ -209,8 +212,8 @@ private: palf::LSN palf_committed_end_lsn_; //LSN standy_committed_end_lsn_; //palf::LSN min_committed_end_lsn_; - palf::SCN last_check_scn_; //当前待确认的最大连续回调位点 - palf::SCN max_applied_cb_scn_; //该位点前的cb保证都已经回调完成 + share::SCN last_check_scn_; //当前待确认的最大连续回调位点 + share::SCN max_applied_cb_scn_; //该位点前的cb保证都已经回调完成 ObApplyServiceSubmitTask submit_task_; ObApplyServiceQueueTask cb_queues_[APPLY_TASK_QUEUE_SIZE]; palf::PalfEnv *palf_env_; @@ -249,7 +252,7 @@ public: palf::LSN &end_lsn); int switch_to_leader(const share::ObLSID &id, const int64_t proposal_id); int switch_to_follower(const share::ObLSID &id); - int get_min_unapplied_scn(const share::ObLSID &id, palf::SCN &scn); + int get_min_unapplied_scn(const share::ObLSID &id, share::SCN &scn); int push_task(ObApplyServiceTask *task); int wait_append_sync(const share::ObLSID &ls_id); int stat_for_each(const common::ObFunction &func); diff --git a/src/logservice/archiveservice/ob_archive_define.cpp b/src/logservice/archiveservice/ob_archive_define.cpp index 4e4c4e0ac..68447bce2 100644 --- a/src/logservice/archiveservice/ob_archive_define.cpp +++ b/src/logservice/archiveservice/ob_archive_define.cpp @@ -21,6 +21,7 @@ namespace oceanbase { using namespace palf; +using namespace share; namespace archive { // =================================== ObArchiveLease ================================= // diff --git a/src/logservice/archiveservice/ob_archive_define.h b/src/logservice/archiveservice/ob_archive_define.h index 04cce42f9..4ab8a6d57 100644 --- a/src/logservice/archiveservice/ob_archive_define.h +++ b/src/logservice/archiveservice/ob_archive_define.h @@ -19,11 +19,11 @@ #include "share/backup/ob_archive_piece.h" // ObArchivePiece #include "share/backup/ob_backup_struct.h" // ObBackupPathString #include "logservice/palf/lsn.h" // LSN -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // share::SCN namespace oceanbase { -namespace palf +namespace share { class SCN; } @@ -88,7 +88,7 @@ class LogFileTuple { public: LogFileTuple(); - LogFileTuple(const LSN &lsn, const palf::SCN &scn, const ObArchivePiece &piece); + LogFileTuple(const LSN &lsn, const share::SCN &scn, const ObArchivePiece &piece); ~LogFileTuple(); public: @@ -98,14 +98,14 @@ public: LogFileTuple &operator=(const LogFileTuple &other); const LSN &get_lsn() const { return offset_; } - const palf::SCN &get_scn() const { return scn_; } + const share::SCN &get_scn() const { return scn_; } const ObArchivePiece &get_piece() const { return piece_; }; void compensate_piece(); TO_STRING_KV(K_(offset), K_(scn), K_(piece)); private: LSN offset_; // 文件内偏移 - palf::SCN scn_; // 最大log scn + share::SCN scn_; // 最大log scn ObArchivePiece piece_; // 所属piece }; diff --git a/src/logservice/archiveservice/ob_archive_fetcher.h b/src/logservice/archiveservice/ob_archive_fetcher.h index b969523dd..6b3478bac 100644 --- a/src/logservice/archiveservice/ob_archive_fetcher.h +++ b/src/logservice/archiveservice/ob_archive_fetcher.h @@ -20,7 +20,7 @@ #include "share/ob_ls_id.h" // ObLSID #include "share/backup/ob_archive_piece.h" // ObArchivePiece #include "logservice/palf/lsn.h" // LSN -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // SCN #include "logservice/palf/palf_iterator.h" // PalfGroupBufferIterator #include "ob_archive_define.h" @@ -31,10 +31,13 @@ namespace logservice class ObLogService; } +namespace share +{ +class SCN; +} namespace palf { struct LSN; -class SCN; class PalfHandleGuard; class LogGroupEntry; } @@ -82,7 +85,7 @@ public: // 开启关闭归档接口 int set_archive_info(const int64_t piece_interval_us, - const palf::SCN &genesis_scn, + const share::SCN &genesis_scn, const int64_t base_piece_id, const int64_t unit_size, const bool need_compress, @@ -133,7 +136,7 @@ private: const LSN &end_offset, bool &data_enough, bool &data_full); // 1.1.2 检查日志流落后程度是否需要触发归档 - bool check_scn_enough_(const palf::SCN &fetch_scn, const palf::SCN &end_scn) const; + bool check_scn_enough_(const share::SCN &fetch_scn, const share::SCN &end_scn) const; // 1.2 初始化TmpMemoryHelper int init_helper_(ObArchiveLogFetchTask &task, TmpMemoryHelper &helper); @@ -231,7 +234,7 @@ private: const LSN &get_start_offset() const { return start_offset_; } const LSN &get_end_offset() const { return end_offset_; } const LSN &get_cur_offset() const { return cur_offset_; } - const palf::SCN &get_unitized_scn() const { return unitized_scn_; } + const share::SCN &get_unitized_scn() const { return unitized_scn_; } int64_t get_capaicity() const { return end_offset_ - cur_offset_; } bool original_buffer_enough(const int64_t size); int get_original_buf(char *&buf, int64_t &buf_size); @@ -281,14 +284,14 @@ private: int64_t origin_buf_pos_; // 当前处理原数日志最大offset/scn LSN cur_offset_; - palf::SCN cur_scn_; + share::SCN cur_scn_; // 中间缺少加密用buffer char *ec_buf_; int64_t ec_buf_size_; // 需要根据数据量计算 int64_t ec_buf_pos_; // 做了单元化处理的日志最大offset/scn LSN unitized_offset_; - palf::SCN unitized_scn_; + share::SCN unitized_scn_; // 当前正在处理piece, 单个helper包含从start_offset到当前piece的全部数据 ObArchivePiece cur_piece_; // 读取日志过程中, 遇到更大piece说明当前piece已经结束 @@ -303,7 +306,7 @@ private: // 压缩加密处理单元 int64_t unit_size_; int64_t piece_interval_; - palf::SCN genesis_scn_; + share::SCN genesis_scn_; int64_t base_piece_id_; bool need_compress_; // 压缩算法 diff --git a/src/logservice/archiveservice/ob_archive_persist_mgr.h b/src/logservice/archiveservice/ob_archive_persist_mgr.h index 2f5d78726..29998a4ac 100644 --- a/src/logservice/archiveservice/ob_archive_persist_mgr.h +++ b/src/logservice/archiveservice/ob_archive_persist_mgr.h @@ -35,6 +35,7 @@ namespace share class ObTenantArchiveRoundAttr; class ObLSID; class ObLSArchivePersistInfo; +class SCN; } namespace storage @@ -45,7 +46,6 @@ class ObLSService; namespace palf { struct LSN; -class SCN; } namespace archive @@ -114,7 +114,7 @@ public: void persist_and_load(); // 获取日志流创建时间戳 - int get_ls_create_scn(const share::ObLSID &id, palf::SCN &scn); + int get_ls_create_scn(const share::ObLSID &id, share::SCN &scn); private: // 1. 持久化当前server做归档日志流归档进度 diff --git a/src/logservice/archiveservice/ob_archive_round_mgr.h b/src/logservice/archiveservice/ob_archive_round_mgr.h index 12217e2c6..a051f52c2 100644 --- a/src/logservice/archiveservice/ob_archive_round_mgr.h +++ b/src/logservice/archiveservice/ob_archive_round_mgr.h @@ -68,7 +68,7 @@ private: private: ArchiveKey key_; - palf::SCN round_start_scn_; + share::SCN round_start_scn_; bool compatible_; // 该轮次兼容性处理 ObArchiveRoundState log_archive_state_; share::ObBackupDest backup_dest_; diff --git a/src/logservice/archiveservice/ob_archive_sender.cpp b/src/logservice/archiveservice/ob_archive_sender.cpp index 99852b8e7..59662ba5c 100644 --- a/src/logservice/archiveservice/ob_archive_sender.cpp +++ b/src/logservice/archiveservice/ob_archive_sender.cpp @@ -39,7 +39,7 @@ #include "ob_archive_io.h" // ObArchiveIO #include "share/backup/ob_backup_path.h" // ObBackupPath #include "share/backup/ob_archive_path.h" // ObArchivePathUtil -#include "logservice/palf/scn.h" // ObArchivePathUtil +#include "share/scn.h" // ObArchivePathUtil namespace oceanbase { diff --git a/src/logservice/archiveservice/ob_archive_sequencer.h b/src/logservice/archiveservice/ob_archive_sequencer.h index bf4ca7188..ad59a366f 100644 --- a/src/logservice/archiveservice/ob_archive_sequencer.h +++ b/src/logservice/archiveservice/ob_archive_sequencer.h @@ -19,7 +19,7 @@ #include "ob_archive_define.h" // DEFAULT_THREAD_RUN_INTERVAL #include "share/ob_ls_id.h" // ObLSID #include "logservice/palf/lsn.h" // LSN -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // SCN #include namespace oceanbase @@ -100,7 +100,7 @@ private: ObArchiveLSMgr *ls_mgr_; ObArchiveRoundMgr *round_mgr_; // 全部日志流最小定序任务scn - palf::SCN min_scn_; + share::SCN min_scn_; // 添加开启归档/日志流归档任务/以及消费LogFetchTask唤醒sequencer common::ObCond seq_cond_; @@ -121,7 +121,7 @@ public: min_scn_(), id_() {} bool operator()(const ObLSID &id, ObLSArchiveTask *ls_archive_task); - void get_min_log_info(ObLSID &id, palf::SCN &min_scn); + void get_min_log_info(ObLSID &id, share::SCN &min_scn); TO_STRING_KV(K_(incarnation), K_(round), K_(succ_count), K_(total_count), K_(min_scn), K_(id)); private: @@ -129,7 +129,7 @@ private: int64_t round_; int64_t succ_count_; int64_t total_count_; - palf::SCN min_scn_; + share::SCN min_scn_; //MaxLogFileInfo min_log_info_; ObLSID id_; }; diff --git a/src/logservice/archiveservice/ob_archive_service.cpp b/src/logservice/archiveservice/ob_archive_service.cpp index a238d2cd1..a8c00e58d 100644 --- a/src/logservice/archiveservice/ob_archive_service.cpp +++ b/src/logservice/archiveservice/ob_archive_service.cpp @@ -19,7 +19,7 @@ #include "share/rc/ob_tenant_base.h" // MTL_* #include "observer/ob_server_struct.h" // GCTX #include "logservice/palf/lsn.h" // LSN -#include "logservice/palf/scn.h" // LSN +#include "share/scn.h" // LSN #include "share/backup/ob_backup_connectivity.h" #include "share/ob_debug_sync.h" diff --git a/src/logservice/archiveservice/ob_archive_task.cpp b/src/logservice/archiveservice/ob_archive_task.cpp index 763f3d176..8983c9002 100644 --- a/src/logservice/archiveservice/ob_archive_task.cpp +++ b/src/logservice/archiveservice/ob_archive_task.cpp @@ -18,6 +18,7 @@ namespace oceanbase { using namespace palf; +using namespace share; namespace archive { diff --git a/src/logservice/archiveservice/ob_archive_task.h b/src/logservice/archiveservice/ob_archive_task.h index ae29123cc..6764e1fd5 100644 --- a/src/logservice/archiveservice/ob_archive_task.h +++ b/src/logservice/archiveservice/ob_archive_task.h @@ -18,7 +18,7 @@ #include "ob_archive_define.h" // ArchiveWorkStation #include "share/backup/ob_archive_piece.h" // ObArchivePiece #include "logservice/palf/lsn.h" // LSN -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // SCN #include namespace oceanbase @@ -77,7 +77,7 @@ public: const LSN &get_start_offset() const { return start_offset_; } const LSN &get_cur_offset() const { return cur_offset_; } const LSN &get_end_offset() const { return end_offset_; } - const palf::SCN &get_max_scn() const { return max_scn_; } + const share::SCN &get_max_scn() const { return max_scn_; } int64_t get_log_fetch_size() const { return (int64_t)(end_offset_ - cur_offset_); } const ObArchivePiece &get_piece() const { return cur_piece_; } int set_next_piece(const ObArchivePiece &piece); @@ -89,7 +89,7 @@ public: int back_fill(const ObArchivePiece &cur_piece, const LSN &start_offset, const LSN &end_offset, - const palf::SCN &max_scn, + const share::SCN &max_scn, ObArchiveSendTask *send_task); bool has_fetch_log() const { return NULL != send_task_; } bool is_continuous_with(const LSN &lsn) const; @@ -114,7 +114,7 @@ private: LSN start_offset_; // 起始拉日志文件起始offset LSN end_offset_; // 拉取日志文件结束offset(如果是拉取完整文件任务，该值为infoblock起点) LSN cur_offset_; - palf::SCN max_scn_; // 任务包含最大日志log scn + share::SCN max_scn_; // 任务包含最大日志log scn ObArchiveSendTask *send_task_; }; @@ -159,7 +159,7 @@ public: const ObArchivePiece &piece, const LSN &start_offset, const LSN &end_offset, - const palf::SCN &max_scn, + const share::SCN &max_scn, char *data, const int64_t data_len); bool is_valid() const; @@ -171,7 +171,7 @@ public: const LSN &get_end_lsn() const { return end_offset_; }; int get_buffer(char *&data, int64_t &data_len) const; int64_t get_buf_size() const { return data_len_;} - const palf::SCN &get_max_scn() const { return max_scn_; } + const share::SCN &get_max_scn() const { return max_scn_; } int set_buffer(char *buf, const int64_t buf_size); bool is_continuous_with(const ObArchiveSendTask &pre_task) const; TO_STRING_KV(K_(tenant_id), @@ -190,7 +190,7 @@ private: ObArchivePiece piece_; LSN start_offset_; // 归档数据在文件起始offset LSN end_offset_; // 归档数据在文件终止offset - palf::SCN max_scn_; // 该task包含数据最大log scn + share::SCN max_scn_; // 该task包含数据最大log scn char *data_; // 发送数据 int64_t data_len_; // 发送数据长度 }; diff --git a/src/logservice/archiveservice/ob_ls_mgr.h b/src/logservice/archiveservice/ob_ls_mgr.h index 98cddc4ee..a7281d65d 100644 --- a/src/logservice/archiveservice/ob_ls_mgr.h +++ b/src/logservice/archiveservice/ob_ls_mgr.h @@ -20,7 +20,7 @@ #include "common/ob_queue_thread.h" // ObCond #include "ob_ls_task.h" // ObLSArchiveTask #include "ob_start_archive_helper.h" // StartArchiveHelper -#include "logservice/palf/scn.h" // SCn +#include "share/scn.h" // SCn #include namespace oceanbase @@ -79,14 +79,14 @@ public: void stop(); void wait(); void destroy(); - int set_archive_info(const palf::SCN &round_start_scn, - const int64_t piece_interval, const palf::SCN &genesis_scn, const int64_t base_piece_id); + int set_archive_info(const share::SCN &round_start_scn, + const int64_t piece_interval, const share::SCN &genesis_scn, const int64_t base_piece_id); void clear_archive_info(); void notify_start(); void notify_stop(); int revert_ls_task(ObLSArchiveTask *task); int get_ls_guard(const ObLSID &id, ObArchiveLSGuard &guard); - int authorize_ls_archive_task(const ObLSID &id, const int64_t epoch, const palf::SCN &start_scn); + int authorize_ls_archive_task(const ObLSID &id, const int64_t epoch, const share::SCN &start_scn); void reset_task(); int64_t get_ls_task_count() const { return ls_map_.count(); } int mark_fata_error(const ObLSID &id, const ArchiveKey &key, const ObArchiveInterruptReason &reason); @@ -107,9 +107,9 @@ private: private: bool inited_; uint64_t tenant_id_; - palf::SCN round_start_scn_; + share::SCN round_start_scn_; int64_t piece_interval_; - palf::SCN genesis_scn_; + share::SCN genesis_scn_; int64_t base_piece_id_; LSArchiveMap ls_map_; diff --git a/src/logservice/archiveservice/ob_ls_task.h b/src/logservice/archiveservice/ob_ls_task.h index 676145f69..62e576a18 100644 --- a/src/logservice/archiveservice/ob_ls_task.h +++ b/src/logservice/archiveservice/ob_ls_task.h @@ -20,7 +20,7 @@ #include "ob_archive_define.h" // ArchiveWorkStation LogFileTuple #include "share/ob_ls_id.h" // ObLSID #include "logservice/palf/lsn.h" // LSN -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // SCN #include "lib/utility/ob_print_utils.h" // print #include @@ -97,7 +97,7 @@ public: // 获取fetch进度 int get_fetcher_progress(const ArchiveWorkStation &station, palf::LSN &offset, - palf::SCN &scn); + share::SCN &scn); int compensate_piece(const ArchiveWorkStation &station, const int64_t next_compensate_piece_id); @@ -147,21 +147,21 @@ private: public: void init(const LSN &piece_min_lsn, const LSN &lsn, const int64_t file_id, const int64_t file_offset, const share::ObArchivePiece &piece, - const palf::SCN &max_archived_scn, const bool is_log_gap_exist, + const share::SCN &max_archived_scn, const bool is_log_gap_exist, ObArchiveAllocator *allocator); void destroy(); void get_sequencer_progress(LSN &offset) const; int update_sequencer_progress(const int64_t size, const LSN &offset); void get_fetcher_progress(LogFileTuple &tuple) const; - int update_fetcher_progress(const palf::SCN &round_start_scn, const LogFileTuple &tuple); + int update_fetcher_progress(const share::SCN &round_start_scn, const LogFileTuple &tuple); int push_fetch_log(ObArchiveLogFetchTask &task); int push_send_task(ObArchiveSendTask &task, ObArchiveWorker &worker); int get_top_fetch_log(ObArchiveLogFetchTask *&task); int pop_fetch_log(ObArchiveLogFetchTask *&task); int compensate_piece(const int64_t piece_id); - void get_max_archive_progress(LSN &piece_min_lsn, LSN &lsn, palf::SCN &scn, ObArchivePiece &piece, + void get_max_archive_progress(LSN &piece_min_lsn, LSN &lsn, share::SCN &scn, ObArchivePiece &piece, int64_t &file_id, int64_t &file_offset, bool &error_exist); - int update_archive_progress(const palf::SCN &round_start_scn, const int64_t file_id, const int64_t file_offset, const LogFileTuple &tuple); + int update_archive_progress(const share::SCN &round_start_scn, const int64_t file_id, const int64_t file_offset, const LogFileTuple &tuple); void get_archive_progress(int64_t &file_id, int64_t &file_offset, LogFileTuple &tuple); void get_archive_send_arg(ObArchiveSendDestArg &arg); void mark_error(); @@ -203,7 +203,7 @@ private: ObLSID id_; uint64_t tenant_id_; ArchiveWorkStation station_; - palf::SCN round_start_scn_; + share::SCN round_start_scn_; ArchiveDest dest_; ObArchiveAllocator *allocator_; mutable RWLock rwlock_; @@ -218,17 +218,17 @@ struct LSArchiveStat int64_t round_; // dest_type dest_value int64_t lease_id_; - palf::SCN round_start_scn_; + share::SCN round_start_scn_; int64_t max_issued_log_lsn_; int64_t issued_task_count_; int64_t issued_task_size_; int64_t max_prepared_piece_id_; int64_t max_prepared_lsn_; - palf::SCN max_prepared_scn_; + share::SCN max_prepared_scn_; int64_t wait_send_task_count_; int64_t archive_piece_id_; int64_t archive_lsn_; - palf::SCN archive_scn_; + share::SCN archive_scn_; int64_t archive_file_id_; int64_t archive_file_offset_; TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(dest_id), K_(incarnation), K_(round), diff --git a/src/logservice/archiveservice/ob_start_archive_helper.cpp b/src/logservice/archiveservice/ob_start_archive_helper.cpp index 2e5c7f233..53e08c7c8 100644 --- a/src/logservice/archiveservice/ob_start_archive_helper.cpp +++ b/src/logservice/archiveservice/ob_start_archive_helper.cpp @@ -30,6 +30,7 @@ namespace archive { using namespace oceanbase::logservice; using namespace oceanbase::palf; +using namespace oceanbase::share; StartArchiveHelper::StartArchiveHelper(const ObLSID &id, const uint64_t tenant_id, const ArchiveWorkStation &station, @@ -239,7 +240,7 @@ int StartArchiveHelper::get_local_start_scn_(SCN &scn) if (OB_FAIL(persist_mgr_->get_ls_create_scn(id_, create_scn))) { ARCHIVE_LOG(WARN, "get ls create scn failed", K(ret), K(id_)); } else { - SCN last_scn = palf::SCN::minus(min_scn_, 1); + SCN last_scn = SCN::minus(min_scn_, 1); scn = create_scn > last_scn ? create_scn : last_scn; if (!scn.is_valid()) { ret = OB_ERR_UNEXPECTED; diff --git a/src/logservice/archiveservice/ob_start_archive_helper.h b/src/logservice/archiveservice/ob_start_archive_helper.h index b4222f5de..57ece1a73 100644 --- a/src/logservice/archiveservice/ob_start_archive_helper.h +++ b/src/logservice/archiveservice/ob_start_archive_helper.h @@ -17,7 +17,7 @@ #include "share/ob_ls_id.h" // ObLSID #include "share/backup/ob_backup_struct.h" // ObBackupPathString #include "logservice/palf/lsn.h" // LSN -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // SCN #include "ob_archive_define.h" // ArchiveWorkStation #include "ob_archive_persist_mgr.h" // ObArchivePersistMgr ObLSArchivePersistInfo @@ -48,9 +48,9 @@ public: explicit StartArchiveHelper(const ObLSID &id, const uint64_t tenant_id, const ArchiveWorkStation &station, - const palf::SCN &min_scn, + const share::SCN &min_scn, const int64_t piece_interval, - const palf::SCN &genesis_scn, + const share::SCN &genesis_scn, const int64_t base_piece_id, ObArchivePersistMgr *persist_mgr); @@ -66,8 +66,8 @@ public: const LSN &get_offset() const { return start_offset_; } int64_t get_file_id() const { return archive_file_id_; } int64_t get_file_offset() const { return archive_file_offset_; } - const palf::SCN &get_round_start_scn() const { return min_scn_; } - const palf::SCN &get_max_archived_scn() const { return max_archived_scn_; } + const share::SCN &get_round_start_scn() const { return min_scn_; } + const share::SCN &get_max_archived_scn() const { return max_archived_scn_; } const share::ObArchivePiece &get_piece() const { return piece_; } bool is_log_gap_exist() const { return log_gap_exist_; } TO_STRING_KV(K_(id), @@ -90,22 +90,22 @@ private: int locate_round_start_archive_point_(); int cal_archive_file_id_offset_(const LSN &lsn, const int64_t archive_file_id, const int64_t archive_file_offset); int get_local_base_lsn_(palf::LSN &lsn, bool &log_gap); - int get_local_start_scn_(palf::SCN &scn); + int get_local_start_scn_(share::SCN &scn); private: ObLSID id_; uint64_t tenant_id_; ArchiveWorkStation station_; bool log_gap_exist_; - palf::SCN min_scn_; + share::SCN min_scn_; int64_t piece_interval_; - palf::SCN genesis_scn_; + share::SCN genesis_scn_; int64_t base_piece_id_; LSN piece_min_lsn_; LSN start_offset_; int64_t archive_file_id_; int64_t archive_file_offset_; - palf::SCN max_archived_scn_; + share::SCN max_archived_scn_; share::ObArchivePiece piece_; ObArchivePersistMgr *persist_mgr_; diff --git a/src/logservice/cdcservice/ob_cdc_fetcher.cpp b/src/logservice/cdcservice/ob_cdc_fetcher.cpp index d6540a00c..a6c3eb54f 100644 --- a/src/logservice/cdcservice/ob_cdc_fetcher.cpp +++ b/src/logservice/cdcservice/ob_cdc_fetcher.cpp @@ -344,7 +344,7 @@ void ObCdcFetcher::check_next_group_entry_(const LSN &next_lsn, bool &reach_upper_limit) { //TODO(scn) - int64_t submit_ts = next_log_group_entry.get_scn().get_val_for_lsn_allocator(); + int64_t submit_ts = next_log_group_entry.get_scn().get_val_for_logservice(); int64_t entry_size = next_log_group_entry.get_serialize_size(); bool is_buf_full = (! resp.has_enough_buffer(entry_size)); diff --git a/src/logservice/cdcservice/ob_cdc_start_lsn_locator.cpp b/src/logservice/cdcservice/ob_cdc_start_lsn_locator.cpp index e03f8e0cf..3639344ef 100644 --- a/src/logservice/cdcservice/ob_cdc_start_lsn_locator.cpp +++ b/src/logservice/cdcservice/ob_cdc_start_lsn_locator.cpp @@ -180,7 +180,7 @@ int ObCdcStartLsnLocator::do_locate_ls_(const ObLocateLSNByTsReq::LocateParam &l } else if (OB_FAIL(group_iter.get_entry(log_group_entry, result_lsn))) { LOG_WARN("group_iter get_entry fail", KR(ret), K_(tenant_id), K(ls_id)); } else { - result_ts_ns = log_group_entry.get_scn().get_val_for_lsn_allocator(); + result_ts_ns = log_group_entry.get_scn().get_val_for_logservice(); } // Unconditional setting ret code diff --git a/src/logservice/leader_coordinator/election_priority_impl/election_priority_impl.h b/src/logservice/leader_coordinator/election_priority_impl/election_priority_impl.h index d4bc1e4d1..1e867aec6 100644 --- a/src/logservice/leader_coordinator/election_priority_impl/election_priority_impl.h +++ b/src/logservice/leader_coordinator/election_priority_impl/election_priority_impl.h @@ -20,7 +20,7 @@ #include "lib/guard/ob_unique_guard.h" #include "lib/utility/ob_unify_serialize.h" #include "logservice/palf/election/interface/election_priority.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_ls_id.h" #include "share/ob_cluster_version.h" #include @@ -111,7 +111,7 @@ public: protected: // 刷新优先级的方法 virtual int refresh_(const share::ObLSID &ls_id) override; - int get_scn_(const share::ObLSID &ls_id, palf::SCN &scn); + int get_scn_(const share::ObLSID &ls_id, share::SCN &scn); private: int compare_observer_stopped_(int &ret, const PriorityV1&) const; int compare_server_stopped_flag_(int &ret, const PriorityV1&) const; @@ -130,7 +130,7 @@ private: common::ObSArray fatal_failures_;// negative infos bool is_primary_region_; common::ObSArray serious_failures_;// negative infos - palf::SCN scn_; + share::SCN scn_; bool is_in_blacklist_; common::ObStringHolder in_blacklist_reason_; bool is_manual_leader_; diff --git a/src/logservice/leader_coordinator/election_priority_impl/election_priority_v1.cpp b/src/logservice/leader_coordinator/election_priority_impl/election_priority_v1.cpp index ff2b4ebb8..3fd9d7034 100644 --- a/src/logservice/leader_coordinator/election_priority_impl/election_priority_v1.cpp +++ b/src/logservice/leader_coordinator/election_priority_impl/election_priority_v1.cpp @@ -28,6 +28,7 @@ namespace oceanbase { +using namespace share; namespace logservice { namespace coordinator @@ -79,7 +80,7 @@ int PriorityV1::compare(const AbstractPriority &rhs, int &result, ObStringHolder #undef PRINT_WRAPPER } -int PriorityV1::get_scn_(const share::ObLSID &ls_id, palf::SCN &scn) +int PriorityV1::get_scn_(const share::ObLSID &ls_id, SCN &scn) { #define PRINT_WRAPPER KR(ret), K(MTL_ID()), K(ls_id), K(*this) int ret = OB_SUCCESS; @@ -94,7 +95,7 @@ int PriorityV1::get_scn_(const share::ObLSID &ls_id, palf::SCN &scn) } else { common::ObRole role; int64_t unused_pid = -1; - palf::SCN min_unreplay_scn; + SCN min_unreplay_scn; if (OB_FAIL(palf_handle_guard.get_role(role, unused_pid))) { COORDINATOR_LOG_(WARN, "get_role failed"); } else if (OB_FAIL(palf_handle_guard.get_max_scn(scn))) { @@ -105,9 +106,9 @@ int PriorityV1::get_scn_(const share::ObLSID &ls_id, palf::SCN &scn) ret = OB_SUCCESS; } else if (!min_unreplay_scn.is_valid_and_not_min()) { scn.set_min(); - } else if (palf::SCN::minus(min_unreplay_scn, 1) < scn) { + } else if (SCN::minus(min_unreplay_scn, 1) < scn) { // For restore case, min_unreplay_scn may be larger than max_ts. - scn = palf::SCN::minus(min_unreplay_scn, 1) ; + scn = SCN::minus(min_unreplay_scn, 1) ; } else {} } else {} COORDINATOR_LOG_(TRACE, "get_scn_ finished", K(role), K(min_unreplay_scn), K(scn)); @@ -127,7 +128,7 @@ int PriorityV1::refresh_(const share::ObLSID &ls_id) ObLeaderCoordinator* coordinator = MTL(ObLeaderCoordinator*); ObFailureDetector* detector = MTL(ObFailureDetector*); LsElectionReferenceInfo election_reference_info; - palf::SCN scn = palf::SCN::min_scn(); + SCN scn = SCN::min_scn(); if (OB_ISNULL(coordinator) || OB_ISNULL(detector)) { ret = OB_ERR_UNEXPECTED; COORDINATOR_LOG_(ERROR, "unexpected nullptr"); diff --git a/src/logservice/libobcdc/src/ob_cdc_part_trans_resolver.cpp b/src/logservice/libobcdc/src/ob_cdc_part_trans_resolver.cpp index ad26a5bc2..8909987b3 100644 --- a/src/logservice/libobcdc/src/ob_cdc_part_trans_resolver.cpp +++ b/src/logservice/libobcdc/src/ob_cdc_part_trans_resolver.cpp @@ -728,7 +728,7 @@ int ObCDCPartTransResolver::handle_commit_( ret = OB_ERR_UNEXPECTED; LOG_ERROR("invalid trans_type", KR(ret), K_(tls_id), K(tx_id), K(commit_log), K(lsn)); } else if (OB_UNLIKELY(OB_INVALID_VERSION == - (trans_commit_version = get_trans_commit_version_(submit_ts, commit_log.get_commit_version().get_val_for_lsn_allocator())))) { + (trans_commit_version = get_trans_commit_version_(submit_ts, commit_log.get_commit_version().get_val_for_logservice())))) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("invalid trans_commit_version", KR(ret), K_(tls_id), K(tx_id), K(lsn), K(submit_ts), K(commit_log)); } else if (!serve_info.is_served(trans_commit_version)) { diff --git a/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp b/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp index 44e242f56..ab0f0fb00 100644 --- a/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp +++ b/src/logservice/libobcdc/src/ob_log_ls_fetch_ctx.cpp @@ -257,7 +257,7 @@ int LSFetchCtx::read_log( int ret = OB_SUCCESS; const char *buf = log_entry.get_data_buf(); const int64_t buf_len = log_entry.get_data_len(); - const int64_t submit_ts = log_entry.get_scn().get_val_for_lsn_allocator(); + const int64_t submit_ts = log_entry.get_scn().get_val_for_logservice(); int64_t pos = 0; logservice::ObLogBaseHeader log_base_header; @@ -318,7 +318,7 @@ int LSFetchCtx::read_miss_tx_log( int ret = OB_SUCCESS; const char *buf = log_entry.get_data_buf(); const int64_t buf_len = log_entry.get_data_len(); - const int64_t submit_ts = log_entry.get_scn().get_val_for_lsn_allocator(); + const int64_t submit_ts = log_entry.get_scn().get_val_for_logservice(); int64_t pos = 0; logservice::ObLogBaseHeader log_base_header; @@ -354,7 +354,7 @@ int LSFetchCtx::update_progress( const palf::LSN &group_entry_lsn) { int ret = OB_SUCCESS; - const int64_t submit_ts = group_entry.get_scn().get_val_for_lsn_allocator(); + const int64_t submit_ts = group_entry.get_scn().get_val_for_logservice(); const int64_t group_entry_serialize_size = group_entry.get_serialize_size(); // Verifying log continuity diff --git a/src/logservice/libobcdc/src/ob_log_meta_manager.cpp b/src/logservice/libobcdc/src/ob_log_meta_manager.cpp index be526f982..8d1c2e6ab 100644 --- a/src/logservice/libobcdc/src/ob_log_meta_manager.cpp +++ b/src/logservice/libobcdc/src/ob_log_meta_manager.cpp @@ -768,15 +768,28 @@ int ObLogMetaManager::set_column_meta_(IColMeta *col_meta, col_meta->setValuesOfEnumSet(extended_type_info_vec); //mysql treat it as MYSQL_TYPE_STRING, it is not suitable for libobcdc - if (ObEnumType == column_schema.get_data_type()) { + if (ObEnumType == col_type) { mysql_type = obmysql::MYSQL_TYPE_ENUM; - } else if (ObSetType == column_schema.get_data_type()) { + } else if (ObSetType == col_type) { mysql_type = obmysql::MYSQL_TYPE_SET; } + } else if (ObNumberType == col_type || ObUNumberType == col_type) { + col_meta->setScale(column_schema.get_data_scale()); + col_meta->setPrecision(column_schema.get_data_precision()); } bool signed_flag = ((type_flag & UNSIGNED_FLAG) == 0); + if (ObBitType == col_type) { + // the length of BIT type is required, + // the "length" of the BIT type is store in precision + col_meta->setLength(column_schema.get_data_precision()); + } else { + // for types with valid length(string\enumset\rowid\json\raw\lob\geo), + // get_data_length returns the valid length, returns 0 for other types. + col_meta->setLength(column_schema.get_data_length()); + } + col_meta->setName(column_schema.get_column_name()); col_meta->setType(static_cast(mysql_type)); col_meta->setSigned(signed_flag); diff --git a/src/logservice/libobcdc/tests/ob_binlog_record_printer.cpp b/src/logservice/libobcdc/tests/ob_binlog_record_printer.cpp index c92a0b82e..ac02ec87f 100644 --- a/src/logservice/libobcdc/tests/ob_binlog_record_printer.cpp +++ b/src/logservice/libobcdc/tests/ob_binlog_record_printer.cpp @@ -520,6 +520,9 @@ int ObBinlogRecordPrinter::output_data_file_column_data(IBinlogRecord *br, const char *is_not_null = col_meta ? (col_meta->isNotNull() ? "true" : "false") : "NULL"; // const char *default_val = col_meta ? col_meta->getDefault() : "NULL"; const char *is_signed = col_meta ? (col_meta->isSigned() ? "true" : "false") : "NULL"; + const long scale = col_meta ? col_meta->getScale(): 0; + const long precision = col_meta ? col_meta->getPrecision(): 0; + const long col_data_length = col_meta ? col_meta->getLength(): 0; bool is_generated_column = col_meta ? col_meta->isGenerated() : false; bool is_hidden_row_key_column = col_meta ? col_meta->isHiddenRowKey() : false; bool is_lob = is_lob_type(ctype); @@ -533,23 +536,40 @@ int ObBinlogRecordPrinter::output_data_file_column_data(IBinlogRecord *br, ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_signed:%s", column_index, is_signed); ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_encoding:%s", column_index, encoding); ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_not_null:%s", column_index, is_not_null); - if (enable_print_detail && is_hidden_row_key_column) { - ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_hidden_rowkey:%d", column_index, is_hidden_row_key_column); + if (enable_print_detail) { + if (is_hidden_row_key_column) { + ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_is_hidden_rowkey:%d", column_index, is_hidden_row_key_column); + } + // print the length of varchar only in print detail mode, + // because there have been many test cases with varchar type before the varchar length info is added into column meta + if (oceanbase::obmysql::MYSQL_TYPE_VAR_STRING == ctype || oceanbase::obmysql::MYSQL_TYPE_BIT == ctype) { + ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_define_length:%ld", column_index, col_data_length); + } + else if ((oceanbase::obmysql::MYSQL_TYPE_ENUM == ctype) || (oceanbase::obmysql::MYSQL_TYPE_SET == ctype)) { + const std::string delim = ","; + for (int i = 0; i < values_of_enum_set->size(); i++) { + enum_set_values_str += (*values_of_enum_set)[i]; + if (i != values_of_enum_set->size() - 1) { + enum_set_values_str += delim; + } + } + ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_extend_info:%s", column_index, enum_set_values_str.c_str()); + } + // print precision & scale only in print detail mode, becacuse INT in oracle mode is also a kind of NUMBER(DECIMAL) + // whose precision is 38 and scale is 0, more importantly, the default precision(-1, PRECISION_UNKNOWN_YET) + // and scale(-85, ORA_NUMBER_SCALE_UNKNOWN_YET) of NUMBER in oracle mode is confusing, so we decide not to + // modify test results for oracle mode temporarily for convenience and efficiency. + // TODO + else if ((oceanbase::obmysql::MYSQL_TYPE_DECIMAL == ctype) || (oceanbase::obmysql::MYSQL_TYPE_NEWDECIMAL == ctype)) { + // Not sure if MYSQL_TYPE_DECIMAL is deprecated, DECIMAL in mysql & oracle mode should be MYSQL_TYPE_NEWDECIMAL + ROW_PRINTF(ptr, size, pos , ri, "[C%ld] column_precision:%ld", column_index, precision); + ROW_PRINTF(ptr, size, pos , ri, "[C%ld] column_scale:%ld", column_index, scale); + } else { } } if (is_generated_column) { ROW_PRINTF(ptr, size, pos, ri, "[C%ld] is_generated_column:%d", column_index, is_generated_column); } - if ((oceanbase::obmysql::MYSQL_TYPE_ENUM == ctype) || (oceanbase::obmysql::MYSQL_TYPE_SET == ctype)) { - const std::string delim = ","; - for (int i = 0; i < values_of_enum_set->size(); i++) { - enum_set_values_str += (*values_of_enum_set)[i]; - if (i != values_of_enum_set->size() - 1) { - enum_set_values_str += delim; - } - } - ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_extend_info:%s", column_index, enum_set_values_str.c_str()); - } // FIXME: does not check the value of the field until the length of the default value can be obtained // ROW_PRINTF(ptr, size, pos, ri, "[C%ld] column_default_value:%s", column_index, default_val); diff --git a/src/logservice/logrpc/ob_log_rpc_req.cpp b/src/logservice/logrpc/ob_log_rpc_req.cpp index af3af90e6..fe415f3ab 100644 --- a/src/logservice/logrpc/ob_log_rpc_req.cpp +++ b/src/logservice/logrpc/ob_log_rpc_req.cpp @@ -14,6 +14,7 @@ namespace oceanbase { +using namespace share; namespace logservice { @@ -167,7 +168,7 @@ OB_SERIALIZE_MEMBER(LogGetPalfStatReq, src_, palf_id_); // ============= LogGetPalfStatResp begin =========== LogGetPalfStatResp::LogGetPalfStatResp( - const palf::SCN &max_scn) + const SCN &max_scn) : max_scn_(max_scn) { } LogGetPalfStatResp::~LogGetPalfStatResp() diff --git a/src/logservice/logrpc/ob_log_rpc_req.h b/src/logservice/logrpc/ob_log_rpc_req.h index 5a6c97bb4..be92c62d9 100644 --- a/src/logservice/logrpc/ob_log_rpc_req.h +++ b/src/logservice/logrpc/ob_log_rpc_req.h @@ -16,7 +16,7 @@ #include "lib/utility/ob_unify_serialize.h" // OB_UNIS_VERSION #include "lib/utility/ob_print_utils.h" // TO_STRING_KV #include "common/ob_member_list.h" // ObMemberList -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -131,12 +131,12 @@ struct LogGetPalfStatResp { OB_UNIS_VERSION(1); public: LogGetPalfStatResp() {}; - LogGetPalfStatResp(const palf::SCN &max_scn); + LogGetPalfStatResp(const share::SCN &max_scn); ~LogGetPalfStatResp(); bool is_valid() const; void reset(); TO_STRING_KV(K_(max_scn)); - palf::SCN max_scn_; + share::SCN max_scn_; }; } // end namespace logservice diff --git a/src/logservice/ob_append_callback.h b/src/logservice/ob_append_callback.h index 6ef18e9fa..a600d9bd9 100644 --- a/src/logservice/ob_append_callback.h +++ b/src/logservice/ob_append_callback.h @@ -13,8 +13,8 @@ #ifndef OCEANBASE_LOGSERVICE_OB_APPEND_CALLBACK_ #define OCEANBASE_LOGSERVICE_OB_APPEND_CALLBACK_ #include "palf/lsn.h" -#include "palf/scn.h" #include "lib/utility/utility.h" +#include "share/scn.h" namespace oceanbase { @@ -35,15 +35,15 @@ public: void __reset() { __start_lsn_.reset(); __next_ = NULL; __scn_.reset();} const palf::LSN &__get_lsn() const { return __start_lsn_; } void __set_lsn(const palf::LSN &lsn) { __start_lsn_ = lsn; } - const palf::SCN& __get_scn() const { return __scn_; } - void __set_scn(const palf::SCN& scn) { __scn_ = scn; } + const share::SCN& __get_scn() const { return __scn_; } + void __set_scn(const share::SCN& scn) { __scn_ = scn; } static AppendCb* __get_class_address(ObLink *ptr); static ObLink* __get_member_address(AppendCb *ptr); ObLink *__next_; VIRTUAL_TO_STRING_KV(KP(__next_), K(__start_lsn_), K(__scn_)); private: palf::LSN __start_lsn_; - palf::SCN __scn_; + share::SCN __scn_; }; class AppendCb : public AppendCbBase diff --git a/src/logservice/ob_garbage_collector.cpp b/src/logservice/ob_garbage_collector.cpp index 21222721f..26b194af1 100644 --- a/src/logservice/ob_garbage_collector.cpp +++ b/src/logservice/ob_garbage_collector.cpp @@ -16,7 +16,7 @@ #include "ob_log_service.h" #include "ob_switch_leader_adapter.h" #include "archiveservice/ob_archive_service.h" -#include "palf/scn.h" +#include "share/scn.h" #include "rpc/obrpc/ob_rpc_net_handler.h" #include "storage/high_availability/ob_storage_ha_struct.h" #include "storage/ls/ob_ls.h" @@ -430,9 +430,9 @@ int ObGCHandler::replay(const void *buffer, return ret; } -palf::SCN ObGCHandler::get_rec_scn() +SCN ObGCHandler::get_rec_scn() { - return palf::SCN::max_scn(); + return SCN::max_scn(); } int ObGCHandler::flush(SCN &scn) diff --git a/src/logservice/ob_garbage_collector.h b/src/logservice/ob_garbage_collector.h index 3eb47567e..51baaccef 100644 --- a/src/logservice/ob_garbage_collector.h +++ b/src/logservice/ob_garbage_collector.h @@ -43,7 +43,7 @@ namespace storage class ObLSService; class ObLS; } -namespace palf +namespace share { class SCN; } @@ -198,7 +198,7 @@ public: virtual int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, - const palf::SCN &scn) override; + const share::SCN &scn) override; // for role change virtual void switch_to_follower_forcedly() override; @@ -207,8 +207,8 @@ public: virtual int resume_leader() override; // for checkpoint - virtual palf::SCN get_rec_scn() override; - virtual int flush(palf::SCN &scn) override; + virtual share::SCN get_rec_scn() override; + virtual int flush(share::SCN &scn) override; TO_STRING_KV(K(is_inited_), K(gc_seq_invalid_member_)); @@ -249,7 +249,7 @@ private: private: const int64_t LS_CLOG_ALIVE_TIMEOUT_US = 100 * 1000; //100ms const int64_t GET_GTS_TIMEOUT_US = 10L * 1000 * 1000; //10s - int get_gts_(const int64_t timeout_us, palf::SCN >s_scn); + int get_gts_(const int64_t timeout_us, share::SCN >s_scn); bool is_ls_blocked_state_(const LSGCState &state); bool is_ls_offline_state_(const LSGCState &state); bool is_ls_wait_gc_state_(const LSGCState &state); @@ -260,9 +260,9 @@ private: void try_check_and_set_wait_gc_(ObGarbageCollector::LSStatus &ls_status); void submit_log_(const ObGCLSLOGType log_type); void update_ls_gc_state_after_submit_log_(const ObGCLSLOGType log_type, - const palf::SCN &scn); - void block_ls_transfer_in_(const palf::SCN &block_scn); - void offline_ls_(const palf::SCN &offline_scn); + const share::SCN &scn); + void block_ls_transfer_in_(const share::SCN &block_scn); + void offline_ls_(const share::SCN &offline_scn); int get_palf_role_(common::ObRole &role); void handle_gc_ls_dropping_(const ObGarbageCollector::LSStatus &ls_status); void handle_gc_ls_offline_(ObGarbageCollector::LSStatus &ls_status); diff --git a/src/logservice/ob_log_base_type.h b/src/logservice/ob_log_base_type.h index 2596346c5..0dd0cc6ce 100644 --- a/src/logservice/ob_log_base_type.h +++ b/src/logservice/ob_log_base_type.h @@ -17,7 +17,7 @@ namespace oceanbase { -namespace palf +namespace share { class SCN; } @@ -125,7 +125,7 @@ public: virtual int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, - const palf::SCN &scn) = 0; + const share::SCN &scn) = 0; }; class ObIRoleChangeSubHandler @@ -158,8 +158,8 @@ public: class ObICheckpointSubHandler { public: - virtual palf::SCN get_rec_scn() = 0; - virtual int flush(palf::SCN &scn) = 0; + virtual share::SCN get_rec_scn() = 0; + virtual int flush(share::SCN &scn) = 0; }; #define REGISTER_TO_LOGSERVICE(type, subhandler) \ diff --git a/src/logservice/ob_log_handler.cpp b/src/logservice/ob_log_handler.cpp index e833ce08e..1f4e02b44 100644 --- a/src/logservice/ob_log_handler.cpp +++ b/src/logservice/ob_log_handler.cpp @@ -23,7 +23,7 @@ #include "logservice/logrpc/ob_log_rpc_req.h" #include "logservice/palf/log_define.h" #include "logservice/palf/lsn.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "logservice/palf/palf_env.h" #include "logservice/palf/log_group_entry.h" #include "logservice/palf/palf_options.h" @@ -31,6 +31,7 @@ namespace oceanbase { +using namespace share; namespace logservice { using namespace palf; @@ -295,7 +296,7 @@ int ObLogHandler::get_access_mode(int64_t &mode_version, palf::AccessMode &acces int ObLogHandler::change_access_mode(const int64_t mode_version, const palf::AccessMode &access_mode, - const palf::SCN &ref_scn) + const SCN &ref_scn) { int ret = OB_SUCCESS; RLockGuard guard(lock_); @@ -367,7 +368,7 @@ int ObLogHandler::locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn) return palf_handle_.locate_by_scn_coarsely(scn, result_lsn); } -int ObLogHandler::locate_by_lsn_coarsely(const LSN &lsn, palf::SCN &result_scn) +int ObLogHandler::locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn) { RLockGuard guard(lock_); return palf_handle_.locate_by_lsn_coarsely(lsn, result_scn); @@ -391,13 +392,13 @@ int ObLogHandler::get_max_lsn(LSN &lsn) const return palf_handle_.get_max_lsn(lsn); } -int ObLogHandler::get_max_scn(palf::SCN &scn) const +int ObLogHandler::get_max_scn(SCN &scn) const { RLockGuard guard(lock_); return palf_handle_.get_max_scn(scn); } -int ObLogHandler::get_end_scn(palf::SCN &scn) const +int ObLogHandler::get_end_scn(SCN &scn) const { RLockGuard guard(lock_); return palf_handle_.get_end_scn(scn); @@ -1116,7 +1117,7 @@ void ObLogHandler::wait_append_sync() { } int ObLogHandler::enable_replay(const palf::LSN &lsn, - const palf::SCN &scn) + const SCN &scn) { int ret = OB_SUCCESS; RLockGuard guard(lock_); diff --git a/src/logservice/ob_log_handler.h b/src/logservice/ob_log_handler.h index c99045122..6e64cd64b 100644 --- a/src/logservice/ob_log_handler.h +++ b/src/logservice/ob_log_handler.h @@ -30,6 +30,10 @@ namespace common { class ObAddr; } +namespace share +{ +class SCN; +} namespace transaction { class ObTsMgr; @@ -38,7 +42,6 @@ namespace palf { class PalfEnv; class LSN; -class SCN; } namespace logservice { @@ -53,21 +56,21 @@ public: virtual bool is_valid() const = 0; virtual int append(const void *buffer, const int64_t nbytes, - const palf::SCN &ref_scn, + const share::SCN &ref_scn, const bool need_nonblock, AppendCb *cb, palf::LSN &lsn, - palf::SCN &scn) = 0; + share::SCN &scn) = 0; virtual int get_role(common::ObRole &role, int64_t &proposal_id) const = 0; virtual int change_access_mode(const int64_t mode_version, const palf::AccessMode &access_mode, - const palf::SCN &ref_scn) = 0; + const share::SCN &ref_scn) = 0; virtual int get_access_mode(int64_t &mode_version, palf::AccessMode &access_mode) const = 0; virtual int seek(const palf::LSN &lsn, palf::PalfBufferIterator &iter) = 0; virtual int seek(const palf::LSN &lsn, palf::PalfGroupBufferIterator &iter) = 0; - virtual int seek(const palf::SCN &scn, palf::PalfGroupBufferIterator &iter) = 0; + virtual int seek(const share::SCN &scn, palf::PalfGroupBufferIterator &iter) = 0; virtual int set_initial_member_list(const common::ObMemberList &member_list, const int64_t paxos_replica_num) = 0; virtual int set_initial_member_list(const common::ObMemberList &member_list, @@ -77,14 +80,14 @@ public: virtual int set_election_priority(palf::election::ElectionPriority *priority) = 0; virtual int reset_election_priority() = 0; - virtual int locate_by_scn_coarsely(const palf::SCN &scn, palf::LSN &result_lsn) = 0; - virtual int locate_by_lsn_coarsely(const palf::LSN &lsn, palf::SCN &result_scn) = 0; + virtual int locate_by_scn_coarsely(const share::SCN &scn, palf::LSN &result_lsn) = 0; + virtual int locate_by_lsn_coarsely(const palf::LSN &lsn, share::SCN &result_scn) = 0; virtual int advance_base_lsn(const palf::LSN &lsn) = 0; virtual int get_end_lsn(palf::LSN &lsn) const = 0; virtual int get_max_lsn(palf::LSN &lsn) const = 0; - virtual int get_max_scn(palf::SCN &scn) const = 0; - virtual int get_end_scn(palf::SCN &scn) const = 0; + virtual int get_max_scn(share::SCN &scn) const = 0; + virtual int get_end_scn(share::SCN &scn) const = 0; virtual int get_paxos_member_list(common::ObMemberList &member_list, int64_t &paxos_replica_num) const = 0; virtual int get_global_learner_list(common::GlobalLearnerList &learner_list) const = 0; virtual int change_replica_num(const common::ObMemberList &member_list, @@ -123,9 +126,9 @@ public: virtual int advance_base_info(const palf::PalfBaseInfo &palf_base_info, const bool is_rebuild) = 0; virtual int is_valid_member(const common::ObAddr &addr, bool &is_valid) const = 0; virtual void wait_append_sync() = 0; - virtual int enable_replay(const palf::LSN &initial_lsn, const palf::SCN &initial_scn) = 0; + virtual int enable_replay(const palf::LSN &initial_lsn, const share::SCN &initial_scn) = 0; virtual int disable_replay() = 0; - virtual int get_max_decided_scn(palf::SCN &scn) = 0; + virtual int get_max_decided_scn(share::SCN &scn) = 0; virtual int pend_submit_replay_log() = 0; virtual int restore_submit_replay_log() = 0; virtual bool is_replay_enabled() const = 0; @@ -171,11 +174,11 @@ public: // NB: only support for primary(AccessMode::APPEND) int append(const void *buffer, const int64_t nbytes, - const palf::SCN &ref_scn, + const share::SCN &ref_scn, const bool need_nonblock, AppendCb *cb, palf::LSN &lsn, - palf::SCN &scn) override final; + share::SCN &scn) override final; // @brief switch log_handle role, to LEADER or FOLLOWER // @param[in], role, LEADER or FOLLOWER @@ -214,7 +217,7 @@ public: // get 'mode_version' and pass it to 'change_access_mode' int change_access_mode(const int64_t mode_mode_version, const palf::AccessMode &access_mode, - const palf::SCN &ref_scn) override final; + const share::SCN &ref_scn) override final; // @brief alloc PalfBufferIterator, this iterator will get something append by caller. // @param[in] const LSN &, the start lsn of iterator. @@ -229,7 +232,7 @@ public: // @brief alloc PalfBufferIterator, this iterator will get something append by caller. // @param[in] const int64_t, the start ts of iterator. // @param[out] PalfBufferIterator &, scn of first log in this iterator must be higher than/equal to start_scn. - int seek(const palf::SCN &scn, + int seek(const share::SCN &scn, palf::PalfGroupBufferIterator &iter) override final; // @brief set the initial member list of paxos group // @param[in] ObMemberList, the initial member list @@ -264,7 +267,7 @@ public: // - OB_ENTRY_NOT_EXIST: there is no log in disk // - OB_ERR_OUT_OF_LOWER_BOUND: scn is too old, log files may have been recycled // - others: bug - int locate_by_scn_coarsely(const palf::SCN &scn, palf::LSN &result_lsn) override final; + int locate_by_scn_coarsely(const share::SCN &scn, palf::LSN &result_lsn) override final; // @desc: query coarse ts by lsn, that means there is a log in disk, // its lsn and scn are result_lsn and result_scn, and result_lsn <= lsn. @@ -275,7 +278,7 @@ public: // - OB_INVALID_ARGUMENT // - OB_ERR_OUT_OF_LOWER_BOUND: lsn is too small, log files may have been recycled // - others: bug - int locate_by_lsn_coarsely(const palf::LSN &lsn, palf::SCN &result_scn) override final; + int locate_by_lsn_coarsely(const palf::LSN &lsn, share::SCN &result_scn) override final; // @brief, set the recycable lsn, palf will ensure that the data before recycable lsn readable. // @param[in] const LSN&, recycable lsn. int advance_base_lsn(const palf::LSN &lsn) override final; @@ -285,10 +288,10 @@ public: int get_max_lsn(palf::LSN &lsn) const override final; // @brief, get max log ts. // @param[out] int64_t&, max log ts. - int get_max_scn(palf::SCN &scn) const override final; + int get_max_scn(share::SCN &scn) const override final; // @brief, get timestamp of end lsn. // @param[out] int64_t, timestamp. - int get_end_scn(palf::SCN &scn) const override final; + int get_end_scn(share::SCN &scn) const override final; // @brief, get paxos member list of this paxos group // @param[out] common::ObMemberList& // @param[out] int64_t& @@ -484,7 +487,7 @@ public: // @param[in] const palf::LSN &initial_lsn: replay new start lsn. // @param[in] const int64_t &initial_scn: replay new start ts. int enable_replay(const palf::LSN &initial_lsn, - const palf::SCN &initial_scn) override final; + const share::SCN &initial_scn) override final; // @brief, disable replay for current ls. int disable_replay() override final; // @brief, pending sumbit replay log @@ -496,7 +499,7 @@ public: bool is_replay_enabled() const override final; // @brief, get max decided log ts considering both apply and replay. // @param[out] int64_t&, max decided log ts ns. - int get_max_decided_scn(palf::SCN &scn) override final; + int get_max_decided_scn(share::SCN &scn) override final; // @brief: store a persistent flag which means this paxos replica // can not reply ack when receiving logs. // By default, paxos replica can reply ack. @@ -510,7 +513,7 @@ public: TO_STRING_KV(K_(role), K_(proposal_id), KP(palf_env_), K(is_in_stop_state_), K(is_inited_)); private: int submit_config_change_cmd_(const LogConfigChangeCmd &req); - int get_leader_max_scn_(palf::SCN &max_scn) const; + int get_leader_max_scn_(share::SCN &max_scn) const; DISALLOW_COPY_AND_ASSIGN(ObLogHandler); private: common::ObAddr self_; diff --git a/src/logservice/ob_log_service.h b/src/logservice/ob_log_service.h index 51bf25a3b..a479eda4e 100644 --- a/src/logservice/ob_log_service.h +++ b/src/logservice/ob_log_service.h @@ -51,6 +51,7 @@ namespace share { class ObLSID; class ObLocationService; +class SCN; } namespace palf @@ -59,7 +60,6 @@ class PalfHandleGuard; class PalfRoleChangeCb; class PalfDiskOptions; class PalfEnv; -class SCN; } namespace storage { @@ -101,7 +101,7 @@ public: int create_ls(const share::ObLSID &id, const common::ObReplicaType &replica_type, const share::ObTenantRole &tenant_role, - const palf::SCN &create_scn, + const share::SCN &create_scn, const bool allow_log_sync, ObLogHandler &log_handler, ObLogRestoreHandler &restore_handler); @@ -150,7 +150,7 @@ public: common::ObRole &role, int64_t &proposal_id); - int update_replayable_point(const palf::SCN &replayable_point); + int update_replayable_point(const share::SCN &replayable_point); int get_palf_disk_usage(int64_t &used_size_byte, int64_t &total_size_byte); int update_palf_disk_options(const palf::PalfDiskOptions &disk_options); // why we need update 'log_disk_size_' and 'log_disk_util_threshold' separately. diff --git a/src/logservice/palf/log_block_header.cpp b/src/logservice/palf/log_block_header.cpp index f61516713..7a5ee70af 100644 --- a/src/logservice/palf/log_block_header.cpp +++ b/src/logservice/palf/log_block_header.cpp @@ -19,6 +19,7 @@ namespace oceanbase { using namespace common; +using namespace share; namespace palf { LogBlockHeader::LogBlockHeader() diff --git a/src/logservice/palf/log_block_header.h b/src/logservice/palf/log_block_header.h index c1c736715..4cd1b5277 100644 --- a/src/logservice/palf/log_block_header.h +++ b/src/logservice/palf/log_block_header.h @@ -17,7 +17,7 @@ #include "lib/ob_define.h" #include "lib/utility/ob_print_utils.h" #include "logservice/palf/lsn.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -31,11 +31,11 @@ public: bool is_valid() const; void reset(); // NB: not thread safe - void update_lsn_and_scn(const LSN &lsn, const SCN &scn); + void update_lsn_and_scn(const LSN &lsn, const share::SCN &scn); void update_palf_id_and_curr_block_id(const int64_t palf_id, const block_id_t curr_block_id); - void mark_block_can_be_reused(const SCN &max_scn); + void mark_block_can_be_reused(const share::SCN &max_scn); block_id_t get_curr_block_id() const; - const SCN &get_min_scn() const; + const share::SCN &get_min_scn() const; LSN get_min_lsn() const; // This interface is just used for PalfIterator @@ -45,7 +45,7 @@ public: // an empty block. // // if this block is a reused block, return 'max_scn_', otherwise return 'min_scn_'. - const SCN get_scn_used_for_iterator() const; + const share::SCN get_scn_used_for_iterator() const; void calc_checksum(); bool check_integrity() const; NEED_SERIALIZE_AND_DESERIALIZE; @@ -64,8 +64,8 @@ private: // The min lsn of curr block LSN min_lsn_; // The min timstamp of curr block - SCN min_scn_; - SCN max_scn_; + share::SCN min_scn_; + share::SCN max_scn_; // Assume that, multiple PALF instances share a directory, and each block names with // 'block_%d'(the number is monotonic, and it may be not same with 'curr_block_id_'). diff --git a/src/logservice/palf/log_config_mgr.h b/src/logservice/palf/log_config_mgr.h index 2555a5b6c..3d322bb13 100644 --- a/src/logservice/palf/log_config_mgr.h +++ b/src/logservice/palf/log_config_mgr.h @@ -18,11 +18,11 @@ #include "lib/random/ob_random.h" // ObRandom #include "lib/hash/ob_array_hash_map.h" // ObArrayHashMap #include "lib/function/ob_function.h" // ObFunction +#include "share/scn.h" //SCN #include "log_define.h" // utils #include "log_meta_info.h" // LogMembershipMeta #include "log_req.h" // LogLearnerReqType #include "log_simple_member_list.h" // LogSimpleMemberList -#include "scn.h" //SCN #include "log_state_mgr.h" // LogStateMgr #include "palf_callback.h" // PalfLocationCacheCb @@ -134,7 +134,7 @@ public: type_(INVALID_LOG_CONFIG_CHANGE_TYPE) { } LogConfigChangeArgs(const LogConfigVersion &config_version, - const SCN &ref_scn, + const share::SCN &ref_scn, const LogConfigChangeType type) : server_(), curr_member_list_(), curr_replica_num_(0), new_replica_num_(0), config_version_(config_version), ref_scn_(ref_scn), type_(type) { } @@ -191,7 +191,7 @@ public: int64_t curr_replica_num_; int64_t new_replica_num_; LogConfigVersion config_version_; - SCN ref_scn_; + share::SCN ref_scn_; LogConfigChangeType type_; }; diff --git a/src/logservice/palf/log_define.h b/src/logservice/palf/log_define.h index e1381cee2..73b76bf5a 100644 --- a/src/logservice/palf/log_define.h +++ b/src/logservice/palf/log_define.h @@ -26,6 +26,11 @@ namespace common { class ObIAllocator; } + +namespace share +{ +class SCN; +} namespace palf { #define TMP_SUFFIX ".tmp" @@ -37,7 +42,6 @@ typedef uint64_t block_id_t ; typedef uint64_t offset_t; constexpr int64_t INVALID_PALF_ID = -1; class LSN; -class SCN; class LogWriteBuf; // ==================== palf env start ============================= @@ -60,7 +64,7 @@ constexpr offset_t LOG_DIO_ALIGN_SIZE = 4 * 1024; constexpr offset_t LOG_DIO_ALIGNED_BUF_SIZE = MAX_LOG_BUFFER_SIZE + LOG_DIO_ALIGN_SIZE; constexpr block_id_t LOG_MAX_BLOCK_ID = UINT64_MAX/PALF_BLOCK_SIZE - 1; constexpr block_id_t LOG_INVALID_BLOCK_ID = LOG_MAX_BLOCK_ID + 1; -typedef common::ObFixedArray SCNArray; +typedef common::ObFixedArray SCNArray; typedef common::ObFixedArray LSNArray; typedef common::ObFixedArray LogWriteBufArray; // ==================== block and log end =========================== diff --git a/src/logservice/palf/log_engine.cpp b/src/logservice/palf/log_engine.cpp index b9e527e27..205e3b5c4 100644 --- a/src/logservice/palf/log_engine.cpp +++ b/src/logservice/palf/log_engine.cpp @@ -38,7 +38,7 @@ namespace palf // ===================== LogEngine start ======================= LogEngine::LogEngine() : - min_block_scn_(), + min_block_max_scn_(), min_block_id_(LOG_INVALID_BLOCK_ID), base_lsn_for_block_gc_(PALF_INITIAL_LSN_VAL), log_meta_lock_(), @@ -158,7 +158,7 @@ void LogEngine::destroy() log_storage_.destroy(); base_lsn_for_block_gc_.reset(); min_block_id_ = LOG_INVALID_BLOCK_ID; - min_block_scn_.reset(); + min_block_max_scn_.reset(); } } @@ -537,8 +537,7 @@ int LogEngine::truncate_prefix_blocks(const LSN &lsn) } else if (OB_FAIL(log_storage_.truncate_prefix_blocks(lsn))) { PALF_LOG(WARN, "truncate_prefix_blocks failed", K(ret), K_(palf_id), K_(is_inited), K(lsn)); } else { - PALF_LOG(INFO, "truncate_prefix_blocks success", K(ret), K_(palf_id), K_(is_inited), K(lsn)); - } + PALF_LOG(INFO, "truncate_prefix_blocks success", K(ret), K_(palf_id), K_(is_inited), K(lsn)); } return ret; } @@ -551,7 +550,7 @@ int LogEngine::delete_block(const block_id_t &block_id) { int ret = OB_SUCCESS; block_id_t next_block_id = block_id + 1; - SCN next_block_min_scn; + SCN next_block_max_scn; if (IS_NOT_INIT) { ret = OB_NOT_INIT; PALF_LOG(ERROR, "LogEngine not inited!!!", K(ret), K_(palf_id), K_(is_inited)); @@ -567,11 +566,12 @@ int LogEngine::delete_block(const block_id_t &block_id) PALF_LOG(INFO, "delete success", K(block_id), K_(palf_id), K_(is_inited)); } - if (OB_SUCC(ret) && OB_FAIL(get_block_min_scn(next_block_id , next_block_min_scn))) { - PALF_LOG(WARN, "get the min scn of next block failed", K(ret), K(next_block_id)); + int tmp_ret = OB_SUCCESS; + if (OB_SUCC(ret) && OB_SUCCESS != (tmp_ret = get_block_min_scn(next_block_id + 1, next_block_max_scn))) { + PALF_LOG(WARN, "get the max ts of next block failed", K(tmp_ret), K(next_block_id)); } // If 'delete_block' or 'get_block_min_scn' failed, need reset 'min_block_scn_' to be invalid. - reset_min_block_info_guarded_by_lock_(next_block_id, next_block_min_scn); + reset_min_block_info_guarded_by_lock_(next_block_id, next_block_max_scn); return ret; } @@ -587,30 +587,50 @@ int LogEngine::get_block_id_range(block_id_t &min_block_id, block_id_t &max_bloc return log_storage_.get_block_id_range(min_block_id, max_block_id); } -int LogEngine::get_min_block_id_and_min_scn(block_id_t &block_id, SCN &scn) +int LogEngine::get_min_block_info_for_gc(block_id_t &block_id, SCN &max_scn) { int ret = OB_SUCCESS; block_id_t max_block_id = LOG_INVALID_BLOCK_ID; block_id_t min_block_id = LOG_INVALID_BLOCK_ID; - SCN min_block_scn; + SCN min_block_max_scn; do { ObSpinLockGuard guard(block_gc_lock_); min_block_id = min_block_id_; - min_block_scn = min_block_scn_; + min_block_max_scn = min_block_max_scn_; } while (0); - if (min_block_scn.is_valid()) { + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(WARN, "LogEngine is not inited", K(ret), KPC(this)); + } else if (min_block_max_scn.is_valid()) { block_id = min_block_id; - scn = min_block_scn; + max_scn = min_block_max_scn; } else if (OB_FAIL(get_block_id_range(min_block_id, max_block_id))) { PALF_LOG(WARN, "get_block_id_range failed", K(ret)); - } else if (OB_FAIL(get_block_min_scn(min_block_id, min_block_scn))) { - PALF_LOG(WARN, "get_block_min_scn failed", K(ret)); + // NB: used next block min_block_ts as the max_scn of current block + } else if (OB_FAIL(get_block_min_scn(min_block_id+1, min_block_max_scn))) { + PALF_LOG(TRACE, "get_block_min_scn failed", K(ret)); } else { - reset_min_block_info_guarded_by_lock_(min_block_id, min_block_scn); + reset_min_block_info_guarded_by_lock_(min_block_id, min_block_max_scn); block_id = min_block_id; - scn = min_block_scn; + max_scn = min_block_max_scn; + } + return ret; +} + +int LogEngine::get_min_block_info(block_id_t &min_block_id, SCN &min_block_scn) const +{ + int ret = OB_SUCCESS; + block_id_t max_block_id; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(WARN, "LogEngine is not inited", K(ret), KPC(this)); + } else if (OB_FAIL(get_block_id_range(min_block_id, max_block_id))) { + PALF_LOG(WARN, "get_block_id_range failed", K(ret), KPC(this)); + } else if (OB_FAIL(get_block_min_scn(min_block_id, min_block_scn))) { + PALF_LOG(WARN, "get_block_min_scn failed", K(ret), KPC(this)); + } else { } return ret; } @@ -1255,11 +1275,11 @@ bool LogEngine::check_last_block_whether_is_integrity_(const block_id_t expected || expected_next_block_id < max_block_id; } -void LogEngine::reset_min_block_info_guarded_by_lock_(const block_id_t min_block_id, const SCN &min_block_scn) +void LogEngine::reset_min_block_info_guarded_by_lock_(const block_id_t min_block_id, const SCN &min_block_max_scn) { ObSpinLockGuard guard(block_gc_lock_); min_block_id_ = min_block_id; - min_block_scn_ = min_block_scn; + min_block_max_scn_ = min_block_max_scn; } } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_engine.h b/src/logservice/palf/log_engine.h index 886e246ef..91732356d 100644 --- a/src/logservice/palf/log_engine.h +++ b/src/logservice/palf/log_engine.h @@ -130,7 +130,7 @@ public: // ==================== Submit aysnc task end ================== // ====================== LogStorage start ===================== - int append_log(const LSN &lsn, const LogWriteBuf &write_buf, const SCN &scn); + int append_log(const LSN &lsn, const LogWriteBuf &write_buf, const share::SCN &scn); int append_log(const LSNArray &lsn, const LogWriteBufArray &write_buf, const SCNArray &scn_array); int read_log(const LSN &lsn, const int64_t in_read_size, @@ -143,7 +143,7 @@ public: const LSN get_begin_lsn() const; int get_block_id_range(block_id_t &min_block_id, block_id_t &max_block_id) const; - int get_block_min_scn(const block_id_t &block_id, SCN &scn) const; + int get_block_min_scn(const block_id_t &block_id, share::SCN &scn) const; // // ====================== LogStorage end ======================= @@ -362,14 +362,15 @@ public: LogMeta get_log_meta() const; const LSN &get_base_lsn_used_for_block_gc() const; // not thread safe - int get_min_block_id_and_min_scn(block_id_t &block_id, SCN &SCN); + int get_min_block_info_for_gc(block_id_t &block_id, share::SCN &max_scn); + int get_min_block_info(block_id_t &block_id, share::SCN &min_scn) const; // // ===================== NetService end ======================== LogStorage *get_log_storage() { return &log_storage_; } LogStorage *get_log_meta_storage() { return &log_meta_storage_; } int get_total_used_disk_space(int64_t &total_used_size_byte) const; virtual int64_t get_palf_epoch() const { return palf_epoch_; } - TO_STRING_KV(K_(palf_id), K_(is_inited), K_(min_block_scn), K_(min_block_id), K_(base_lsn_for_block_gc), + TO_STRING_KV(K_(palf_id), K_(is_inited), K_(min_block_max_scn), K_(min_block_id), K_(base_lsn_for_block_gc), K_(log_meta), K_(log_meta_storage), K_(log_storage), K_(palf_epoch), KP(this)); private: int submit_flush_meta_task_(const FlushMetaCbCtx &flush_meta_cb_ctx, const LogMeta &log_meta); @@ -405,7 +406,8 @@ private: int serialize_log_meta_(const LogMeta &log_meta, char *buf, int64_t buf_len); - void reset_min_block_info_guarded_by_lock_(const block_id_t block_id, const SCN &min_scn); + void reset_min_block_info_guarded_by_lock_(const block_id_t min_block_id, + const share::SCN &min_block_max_scn); private: DISALLOW_COPY_AND_ASSIGN(LogEngine); @@ -413,7 +415,7 @@ private: private: // used for GC mutable ObSpinLock block_gc_lock_; - SCN min_block_scn_; + share::SCN min_block_max_scn_; mutable block_id_t min_block_id_; LSN base_lsn_for_block_gc_; diff --git a/src/logservice/palf/log_entry.h b/src/logservice/palf/log_entry.h index 6c9e452c5..a55e2ad35 100644 --- a/src/logservice/palf/log_entry.h +++ b/src/logservice/palf/log_entry.h @@ -17,8 +17,8 @@ #include "lib/ob_define.h" // Serialization #include "lib/utility/ob_print_utils.h" // Print* #include "lib/utility/ob_macro_utils.h" // DISALLOW_COPY_AND_ASSIGN +#include "share/scn.h" #include "log_define.h" -#include "scn.h" namespace oceanbase { @@ -39,7 +39,7 @@ public: int64_t get_header_size() const { return header_.get_serialize_size(); } int64_t get_payload_offset() const { return header_.get_serialize_size(); } int64_t get_data_len() const { return header_.get_data_len(); } - const SCN get_scn() const { return header_.get_scn(); } + const share::SCN get_scn() const { return header_.get_scn(); } const char *get_data_buf() const { return buf_; } const LogEntryHeader &get_header() const { return header_; } diff --git a/src/logservice/palf/log_entry_header.cpp b/src/logservice/palf/log_entry_header.cpp index cf8b55aa0..187984915 100644 --- a/src/logservice/palf/log_entry_header.cpp +++ b/src/logservice/palf/log_entry_header.cpp @@ -17,6 +17,7 @@ namespace oceanbase { +using namespace share; namespace palf { @@ -67,7 +68,7 @@ bool LogEntryHeader::get_header_parity_check_res_() const bool bool_ret = parity_check(reinterpret_cast(magic_)); bool_ret ^= parity_check(reinterpret_cast(version_)); bool_ret ^= parity_check(reinterpret_cast(log_size_)); - bool_ret ^= parity_check((scn_.get_val_for_lsn_allocator())); + bool_ret ^= parity_check((scn_.get_val_for_logservice())); bool_ret ^= parity_check(reinterpret_cast(data_checksum_)); int64_t tmp_flag = (flag_ & ~(0x1)); bool_ret ^= parity_check(reinterpret_cast(tmp_flag)); diff --git a/src/logservice/palf/log_entry_header.h b/src/logservice/palf/log_entry_header.h index bd9c4b790..268b5b8ee 100644 --- a/src/logservice/palf/log_entry_header.h +++ b/src/logservice/palf/log_entry_header.h @@ -15,7 +15,7 @@ #include "lib/ob_define.h" #include "lib/utility/ob_print_utils.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -29,13 +29,13 @@ public: public: int generate_header(const char *log_data, const int64_t data_len, - const SCN &scn); + const share::SCN &scn); LogEntryHeader& operator=(const LogEntryHeader &header); void reset(); bool is_valid() const; bool check_integrity(const char *buf, const int64_t buf_len) const; int32_t get_data_len() const { return log_size_; } - const SCN get_scn() const { return scn_; } + const share::SCN get_scn() const { return scn_; } int64_t get_data_checksum() const { return data_checksum_; } bool check_header_integrity() const; NEED_SERIALIZE_AND_DESERIALIZE; @@ -58,7 +58,7 @@ private: int16_t magic_; int16_t version_; int32_t log_size_; - SCN scn_; + share::SCN scn_; int64_t data_checksum_; // The lowest bit is used for parity check. int64_t flag_; diff --git a/src/logservice/palf/log_group_entry.cpp b/src/logservice/palf/log_group_entry.cpp index 063331757..497e24612 100644 --- a/src/logservice/palf/log_group_entry.cpp +++ b/src/logservice/palf/log_group_entry.cpp @@ -17,9 +17,9 @@ namespace oceanbase { +using namespace share; namespace palf { -using namespace common; LogGroupEntry::LogGroupEntry() : header_(), buf_(NULL) { } diff --git a/src/logservice/palf/log_group_entry.h b/src/logservice/palf/log_group_entry.h index aeea504ad..43897b42f 100644 --- a/src/logservice/palf/log_group_entry.h +++ b/src/logservice/palf/log_group_entry.h @@ -48,14 +48,14 @@ public: // used for fetch_log, ignore the data len of padding entry int64_t get_group_size_without_padding_data() const { return header_.get_serialize_size() + (header_.is_padding_log() ? 0 : header_.get_data_len()); } - const SCN get_scn() const { return header_.get_max_scn(); } + const share::SCN get_scn() const { return header_.get_max_scn(); } LSN get_committed_end_lsn() const { return header_.get_committed_end_lsn(); } const LogGroupEntryHeader &get_header() const { return header_; } const char *get_data_buf() const { return buf_; } // @brief truncate log group entry the upper_limit_scn, only log entries with scn not bigger than which can reserve // param[in] upper_limit_scn the upper bound to determain which log entries can reserve // param[in] pre_accum_checksum, the accum_checksum of the pre log - int truncate(const SCN &upper_limit_scn, const int64_t pre_accum_checksum); + int truncate(const share::SCN &upper_limit_scn, const int64_t pre_accum_checksum); TO_STRING_KV("LogGroupEntryHeader", header_); NEED_SERIALIZE_AND_DESERIALIZE; diff --git a/src/logservice/palf/log_group_entry_header.cpp b/src/logservice/palf/log_group_entry_header.cpp index 357a56c56..97192f270 100644 --- a/src/logservice/palf/log_group_entry_header.cpp +++ b/src/logservice/palf/log_group_entry_header.cpp @@ -18,9 +18,9 @@ #include "lib/utility/utility.h" // !FALSE_IT #include "lib/oblog/ob_log_module.h" // LOG* #include "share/rc/ob_tenant_base.h" // mtl_malloc +#include "share/scn.h" // SCN #include "log_define.h" // is_valid_log_id... #include "log_writer_utils.h" // LogWriteBuf -#include "scn.h" // SCN namespace oceanbase { @@ -179,7 +179,7 @@ bool LogGroupEntryHeader::get_header_parity_check_res_() const bool_ret ^= parity_check(reinterpret_cast(group_size_)); bool_ret ^= parity_check(reinterpret_cast(proposal_id_)); bool_ret ^= parity_check(committed_end_lsn_.val_); - bool_ret ^= parity_check(max_scn_.get_val_for_lsn_allocator()); + bool_ret ^= parity_check(max_scn_.get_val_for_logservice()); bool_ret ^= parity_check(reinterpret_cast(accumulated_checksum_)); bool_ret ^= parity_check(reinterpret_cast(log_id_)); int64_t tmp_flag = (flag_ & ~(0x1)); diff --git a/src/logservice/palf/log_group_entry_header.h b/src/logservice/palf/log_group_entry_header.h index 29638b2e7..dfde08fc2 100644 --- a/src/logservice/palf/log_group_entry_header.h +++ b/src/logservice/palf/log_group_entry_header.h @@ -16,8 +16,8 @@ #include "lib/ob_define.h" // Serialization #include "lib/ob_name_def.h" #include "lib/utility/ob_print_utils.h" // Print* +#include "share/scn.h" // SCN #include "lsn.h" // LSN -#include "scn.h" // SCN namespace oceanbase { @@ -45,7 +45,7 @@ public: const bool is_padding_log, const LogWriteBuf &log_write_buf, const int64_t data_len, - const SCN &max_scn, + const share::SCN &max_scn, const int64_t log_id, const LSN &committed_end_lsn, const int64_t &log_proposal_id, @@ -55,7 +55,7 @@ public: LogGroupEntryHeader& operator=(const LogGroupEntryHeader &header); int32_t get_data_len() const { return group_size_; } int64_t get_accum_checksum() const { return accumulated_checksum_; } - const SCN &get_max_scn() const { return max_scn_; } + const share::SCN &get_max_scn() const { return max_scn_; } int64_t get_log_id() const { return log_id_; } const int64_t &get_log_proposal_id() const { return proposal_id_; } const LSN &get_committed_end_lsn() const { return committed_end_lsn_; } @@ -86,7 +86,7 @@ public: int truncate(const char *buf, const int64_t data_len, - const SCN &cut_scn, + const share::SCN &cut_scn, const int64_t pre_accum_checksum); NEED_SERIALIZE_AND_DESERIALIZE; @@ -133,7 +133,7 @@ private: // The max committed log offset before this log LSN committed_end_lsn_; // The max scn of this log - SCN max_scn_; + share::SCN max_scn_; // The accumulated checksum before this log, including this log, // not including log header int64_t accumulated_checksum_; diff --git a/src/logservice/palf/log_io_task_cb_utils.cpp b/src/logservice/palf/log_io_task_cb_utils.cpp index 1a0a43816..1e80781e1 100644 --- a/src/logservice/palf/log_io_task_cb_utils.cpp +++ b/src/logservice/palf/log_io_task_cb_utils.cpp @@ -16,6 +16,7 @@ namespace oceanbase { +using namespace share; namespace palf { FlushLogCbCtx::FlushLogCbCtx() diff --git a/src/logservice/palf/log_io_task_cb_utils.h b/src/logservice/palf/log_io_task_cb_utils.h index 19620b42e..f294919f4 100644 --- a/src/logservice/palf/log_io_task_cb_utils.h +++ b/src/logservice/palf/log_io_task_cb_utils.h @@ -15,8 +15,8 @@ #include "lib/oblog/ob_log_print_kv.h" #include "lib/utility/ob_macro_utils.h" #include "lib/utility/ob_print_utils.h" // TO_STRING_KV +#include "share/scn.h" #include "lsn.h" -#include "scn.h" #include "palf_base_info.h" #include "log_meta_info.h" @@ -27,7 +27,7 @@ namespace palf struct FlushLogCbCtx { FlushLogCbCtx(); - FlushLogCbCtx(const int64_t log_id, const SCN &scn, const LSN &lsn, + FlushLogCbCtx(const int64_t log_id, const share::SCN &scn, const LSN &lsn, const int64_t &log_proposal_id, const int64_t total_len, const int64_t &curr_log_proposal_id, const int64_t begine_ts); ~FlushLogCbCtx(); @@ -36,7 +36,7 @@ struct FlushLogCbCtx FlushLogCbCtx &operator=(const FlushLogCbCtx &flush_log_cb_ctx); TO_STRING_KV(K_(log_id), K_(scn), K_(lsn), K_(log_proposal_id), K_(total_len), K_(curr_proposal_id), K_(begin_ts)); int64_t log_id_; - SCN scn_; + share::SCN scn_; LSN lsn_; int64_t log_proposal_id_; int64_t total_len_; diff --git a/src/logservice/palf/log_meta.cpp b/src/logservice/palf/log_meta.cpp index 6d7d447d0..f95597707 100644 --- a/src/logservice/palf/log_meta.cpp +++ b/src/logservice/palf/log_meta.cpp @@ -19,6 +19,7 @@ namespace oceanbase namespace palf { using namespace common; +using namespace share; LogMeta::LogMeta() : version_(-1), log_prepare_meta_(), log_config_meta_(), diff --git a/src/logservice/palf/log_meta_info.cpp b/src/logservice/palf/log_meta_info.cpp index 0969ba9d5..47232c59d 100644 --- a/src/logservice/palf/log_meta_info.cpp +++ b/src/logservice/palf/log_meta_info.cpp @@ -20,6 +20,7 @@ namespace oceanbase namespace palf { using namespace common; +using namespace share; LogVotedFor::LogVotedFor() { diff --git a/src/logservice/palf/log_meta_info.h b/src/logservice/palf/log_meta_info.h index 494bab2ae..c67e19cdb 100644 --- a/src/logservice/palf/log_meta_info.h +++ b/src/logservice/palf/log_meta_info.h @@ -17,8 +17,8 @@ #include "lib/utility/ob_print_utils.h" // Print* #include "common/ob_learner_list.h" // common::GlobalLearnerList #include "common/ob_member_list.h" // ObMemberList +#include "share/scn.h" // SCN #include "lsn.h" // LSN -#include "scn.h" // SCN #include "palf_base_info.h" // LogInfo #include "palf_options.h" // AccessMode #include "log_define.h" // ReplicaType @@ -177,7 +177,7 @@ public: const int64_t proposal_id, const int64_t mode_version, const AccessMode &access_mode, - const SCN &ref_scn); + const share::SCN &ref_scn); bool is_valid() const; void reset(); void operator=(const LogModeMeta &mode_meta); @@ -192,7 +192,7 @@ public: AccessMode access_mode_; // scn lower bound // after switching over, scn of all submitted log should be bigger than ref_scn_ - SCN ref_scn_; + share::SCN ref_scn_; static constexpr int64_t LOG_MODE_META_VERSION = 1; }; diff --git a/src/logservice/palf/log_mode_mgr.cpp b/src/logservice/palf/log_mode_mgr.cpp index 408d5cbfa..3b7208117 100644 --- a/src/logservice/palf/log_mode_mgr.cpp +++ b/src/logservice/palf/log_mode_mgr.cpp @@ -22,6 +22,7 @@ namespace oceanbase { using namespace common; +using namespace share; namespace palf { diff --git a/src/logservice/palf/log_mode_mgr.h b/src/logservice/palf/log_mode_mgr.h index 7499403f7..f8c6d5c53 100644 --- a/src/logservice/palf/log_mode_mgr.h +++ b/src/logservice/palf/log_mode_mgr.h @@ -57,7 +57,7 @@ public: virtual void reset_status(); virtual int get_access_mode(AccessMode &access_mode) const; virtual int get_access_mode(int64_t &mode_version, AccessMode &access_mode) const; - virtual int get_ref_scn(int64_t &mode_version, SCN &ref_scn) const; + virtual int get_ref_scn(int64_t &mode_version, share::SCN &ref_scn) const; bool can_append() const; bool can_raw_write() const; bool is_in_pending_state() const; @@ -68,7 +68,7 @@ public: virtual int reconfirm_mode_meta(); virtual int change_access_mode(const int64_t mode_version, const AccessMode &access_mode, - const SCN &ref_scn); + const share::SCN &ref_scn); virtual int handle_prepare_response(const common::ObAddr &server, const int64_t msg_proposal_id, const int64_t accept_log_proposal_id, @@ -95,7 +95,7 @@ private: bool can_finish_change_mode_() const; bool is_need_retry_() const; int switch_state_(const AccessMode &access_mode, - const SCN &ref_scn, + const share::SCN &ref_scn, const bool is_reconfirm); int submit_prepare_req_(const bool need_inc_pid, const bool need_send_and_handle_prepare); int submit_accept_req_(const int64_t proposal_id, const LogModeMeta &mode_meta); diff --git a/src/logservice/palf/log_sliding_window.cpp b/src/logservice/palf/log_sliding_window.cpp index e2bf3a78e..9041e3cc3 100644 --- a/src/logservice/palf/log_sliding_window.cpp +++ b/src/logservice/palf/log_sliding_window.cpp @@ -31,6 +31,7 @@ #include "log_mode_mgr.h" namespace oceanbase { +using namespace share; namespace palf { LogSlidingWindow::LogSlidingWindow() diff --git a/src/logservice/palf/log_sliding_window.h b/src/logservice/palf/log_sliding_window.h index f99eb40cb..e54555135 100644 --- a/src/logservice/palf/log_sliding_window.h +++ b/src/logservice/palf/log_sliding_window.h @@ -17,12 +17,12 @@ #include "lib/hash/ob_linear_hash_map.h" #include "lib/lock/ob_spin_lock.h" #include "lib/thread/ob_thread_lease.h" +#include "share/scn.h" #include "log_group_entry.h" #include "log_group_buffer.h" #include "log_checksum.h" #include "log_req.h" #include "lsn.h" -#include "scn.h" #include "lsn_allocator.h" #include "log_task.h" #include "fixed_sliding_window.h" @@ -110,7 +110,7 @@ public: const PalfBaseInfo &palf_base_info); virtual int sliding_cb(const int64_t sn, const FixedSlidingWindowSlot *data); virtual int64_t get_max_log_id() const; - virtual const SCN get_max_scn() const; + virtual const share::SCN get_max_scn() const; virtual LSN get_max_lsn() const; virtual int64_t get_start_id() const; virtual int get_committed_end_lsn(LSN &committed_end_lsn) const; @@ -128,9 +128,9 @@ public: // ================= log sync part begin virtual int submit_log(const char *buf, const int64_t buf_len, - const SCN &ref_scn, + const share::SCN &ref_scn, LSN &lsn, - SCN &scn); + share::SCN &scn); virtual int submit_group_log(const LSN &lsn, const char *buf, const int64_t buf_len); @@ -184,9 +184,9 @@ public: virtual int64_t get_last_submit_log_id_() const; virtual int get_last_submit_log_info(LSN &last_submit_lsn, int64_t &log_id, int64_t &log_proposal_id) const; virtual int get_last_slide_end_lsn(LSN &out_end_lsn) const; - virtual const SCN get_last_slide_scn() const; + virtual const share::SCN get_last_slide_scn() const; virtual int try_freeze_last_log(); - virtual int inc_update_scn_base(const SCN &scn); + virtual int inc_update_scn_base(const share::SCN &scn); // location cache will be removed TODO by yunlong virtual int set_location_cache_cb(PalfLocationCacheCb *lc_cb); virtual int reset_location_cache_cb(); @@ -218,7 +218,7 @@ private: int get_prev_log_info_(const int64_t log_id, LSN &prev_lsn, LSN &prev_end_lsn, - SCN &prev_log_iscn, + share::SCN &prev_log_iscn, int64_t &prev_log_pid, int64_t &prev_log_accum_checksum); int inc_update_max_flushed_log_info_(const LSN &lsn, @@ -230,13 +230,13 @@ private: void get_last_slide_end_lsn_(LSN &out_end_lsn) const; int64_t get_last_slide_log_id_() const; void get_last_slide_log_info_(int64_t &log_id, - SCN &scn, + share::SCN &scn, LSN &lsn, LSN &end_lsn, int64_t &log_proposal_id, int64_t &accum_checksum) const; int try_update_last_slide_log_info_(const int64_t log_id, - const SCN &scn, + const share::SCN &scn, const LSN &lsn, const LSN &end_lsn, const int64_t &proposal_id, @@ -254,7 +254,7 @@ private: int try_freeze_last_log_(const int64_t expected_log_id, const LSN &expected_end_lsn, bool &is_need_handle); int generate_new_group_log_(const LSN &lsn, const int64_t log_id, - const SCN &scn, + const share::SCN &scn, const int64_t log_body_size, const LogType &log_type, const char *log_data, @@ -262,7 +262,7 @@ private: bool &is_need_handle); int append_to_group_log_(const LSN &lsn, const int64_t log_id, - const SCN &scn, + const share::SCN &scn, const int64_t log_entry_size, const char *log_data, const int64_t data_len, @@ -288,7 +288,7 @@ private: int wait_group_buffer_ready_(const LSN &lsn, const int64_t data_len); int append_disk_log_to_sw_(const LSN &lsn, const LogGroupEntry &group_entry); int try_update_max_lsn_(const LSN &lsn, const LogGroupEntryHeader &header); - int truncate_lsn_allocator_(const LSN &last_lsn, const int64_t last_log_id, const SCN &last_scn); + int truncate_lsn_allocator_(const LSN &last_lsn, const int64_t last_log_id, const share::SCN &last_scn); bool is_all_committed_log_slided_out_(LSN &prev_lsn, int64_t &prev_log_id, LSN &committed_end_lsn) const; void get_last_fetch_info_(LSN &last_fetch_end_lsn, LSN &last_committed_end_lsn, @@ -307,7 +307,7 @@ private: int get_min_scn_from_buf_(const LogGroupEntryHeader &group_entry_header, const char *buf, const int64_t buf_len, - SCN &min_scn); + share::SCN &min_scn); int leader_get_committed_log_info_(const LSN &committed_end_lsn, int64_t &log_id, int64_t &log_proposal_id); @@ -385,7 +385,7 @@ private: // last_slide_log_pid_: it is used for forward checking when receive log. mutable common::ObSpinLock last_slide_info_lock_; int64_t last_slide_log_id_; // used by clean log - SCN last_slide_scn_; + share::SCN last_slide_scn_; LSN last_slide_lsn_; LSN last_slide_end_lsn_; int64_t last_slide_log_pid_; diff --git a/src/logservice/palf/log_storage.cpp b/src/logservice/palf/log_storage.cpp index 829bafd2b..2a19eb570 100644 --- a/src/logservice/palf/log_storage.cpp +++ b/src/logservice/palf/log_storage.cpp @@ -15,7 +15,7 @@ #include "lib/ob_errno.h" // OB_INVALID_ARGUMENT #include "share/rc/ob_tenant_base.h" // mtl_malloc #include "log_reader_utils.h" // ReadBuf -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { diff --git a/src/logservice/palf/log_storage.h b/src/logservice/palf/log_storage.h index b9fb3d17c..aba7784e9 100644 --- a/src/logservice/palf/log_storage.h +++ b/src/logservice/palf/log_storage.h @@ -29,9 +29,12 @@ namespace common { class ObILogAllocator; } -namespace palf +namespace share { class SCN; +} +namespace palf +{ class ReadBuf; class LogStorage : public ILogStorage { @@ -62,7 +65,7 @@ public: void destroy(); int writev(const LSNArray &lsn_array, const LogWriteBufArray &write_buf_array, const SCNArray &scn_array); - int writev(const LSN &lsn, const LogWriteBuf &write_buf, const SCN &scn); + int writev(const LSN &lsn, const LogWriteBuf &write_buf, const share::SCN &scn); int append_meta(const char *buf, const int64_t buf_len); @@ -92,7 +95,7 @@ public: // OB_ERR_OUT_OF_UPPER_BOUND, 'block_id' is the active block, and there is no data in this // block. OB_ERR_UNEXPECTED, file maybe deleted by human. OB_INVALID_DATA, data has been // corrupted - int get_block_min_scn(const block_id_t &block_id, SCN &min_scn) const; + int get_block_min_scn(const block_id_t &block_id, share::SCN &min_scn) const; const LSN get_begin_lsn() const; const LSN get_end_lsn() const; @@ -115,8 +118,8 @@ private: bool check_read_out_of_lower_bound_(const block_id_t &block_id) const; int inner_switch_block_(); int append_block_header_used_for_meta_storage_(); - int append_block_header_(const LSN &block_min_lsn, const SCN &block_min_scn); - int update_block_header_(const block_id_t block_id, const LSN &block_min_lsn, const SCN &block_min_scn); + int append_block_header_(const LSN &block_min_lsn, const share::SCN &block_min_scn); + int update_block_header_(const block_id_t block_id, const LSN &block_min_lsn, const share::SCN &block_min_scn); bool need_switch_block_() const; // 1. 使用group_entry_iterator迭代最后一个文件, 重建block_header; // 2. 从iterator中获取终点信息； diff --git a/src/logservice/palf/log_task.cpp b/src/logservice/palf/log_task.cpp index 1cc8783cb..9db37ff02 100644 --- a/src/logservice/palf/log_task.cpp +++ b/src/logservice/palf/log_task.cpp @@ -21,6 +21,7 @@ namespace oceanbase { +using namespace share; namespace palf { LogTaskHeaderInfo& LogTaskHeaderInfo::operator=(const LogTaskHeaderInfo &rval) diff --git a/src/logservice/palf/log_task.h b/src/logservice/palf/log_task.h index b02800c86..915487bac 100644 --- a/src/logservice/palf/log_task.h +++ b/src/logservice/palf/log_task.h @@ -14,10 +14,10 @@ #define OCEABASE_LOGSERVICE_LOG_TASK_ #include "lib/ob_define.h" // Serialization #include "lib/utility/ob_print_utils.h" // Print* +#include "share/scn.h" #include "fixed_sliding_window.h" #include "log_define.h" // block_id_t #include "lsn.h" -#include "scn.h" namespace oceanbase { @@ -31,8 +31,8 @@ struct LogTaskHeaderInfo LSN begin_lsn_; LSN end_lsn_; int64_t log_id_; - SCN min_scn_; - SCN max_scn_; + share::SCN min_scn_; + share::SCN max_scn_; int64_t data_len_; // total len without log_group_entry_header int64_t proposal_id_; // leader's proposal_id when generate this log LSN prev_lsn_; @@ -95,7 +95,7 @@ public: void unlock() const { lock_.unlock(); } - void inc_update_max_scn(const SCN &scn); + void inc_update_max_scn(const share::SCN &scn); void update_data_len(const int64_t data_len); void set_end_lsn(const LSN &end_lsn); int try_freeze(const LSN &end_lsn); @@ -120,7 +120,7 @@ public: int64_t get_log_cnt() const { return ATOMIC_LOAD(&log_cnt_); } int set_initial_header_info(const LogTaskHeaderInfo &header_info); int update_header_info(const LSN &committed_end_lsn, const int64_t accum_checksum); - int set_group_header(const LSN &lsn, const SCN &scn, const LogGroupEntryHeader &group_entry_header); + int set_group_header(const LSN &lsn, const share::SCN &scn, const LogGroupEntryHeader &group_entry_header); // update group log data_checksum void set_group_log_checksum(const int64_t data_checksum); void set_prev_lsn(const LSN &prev_lsn); @@ -132,8 +132,8 @@ public: bool is_raw_write() const { return header_.is_raw_write_; } int64_t get_data_len() const { return ATOMIC_LOAD(&(header_.data_len_)); } int64_t get_log_id() const { return header_.log_id_; } - const SCN get_min_scn() const {return header_.min_scn_; } - const SCN get_max_scn() const { return header_.max_scn_; } + const share::SCN get_min_scn() const {return header_.min_scn_; } + const share::SCN get_max_scn() const { return header_.max_scn_; } int64_t get_proposal_id() const { return header_.proposal_id_; } LSN get_begin_lsn() const { return header_.begin_lsn_; } LSN get_end_lsn() const { return header_.end_lsn_; } diff --git a/src/logservice/palf/lsn_allocator.cpp b/src/logservice/palf/lsn_allocator.cpp index 82c51919d..65a8f6d0e 100644 --- a/src/logservice/palf/lsn_allocator.cpp +++ b/src/logservice/palf/lsn_allocator.cpp @@ -15,6 +15,7 @@ namespace oceanbase { +using namespace share; namespace palf { const int64_t LSNAllocator::LOG_ID_DELTA_UPPER_BOUND; @@ -54,7 +55,7 @@ int LSNAllocator::init(const int64_t log_id, PALF_LOG(WARN, "invalid arguments", K(ret), K(log_id), K(scn), K(start_lsn)); } else { log_id_base_ = log_id; - scn_base_ = scn.get_val_for_lsn_allocator(); + scn_base_ = scn.get_val_for_logservice(); lsn_ts_meta_.v128_.lo = 0; lsn_ts_meta_.lsn_val_ = start_lsn.val_; lsn_ts_meta_.is_need_cut_ = 1; @@ -88,7 +89,7 @@ int LSNAllocator::truncate(const LSN &lsn, const int64_t log_id, const SCN &scn) next.is_need_cut_ = 1; if (CAS128(&lsn_ts_meta_, last, next)) { log_id_base_ = log_id; - scn_base_ = scn.get_val_for_lsn_allocator(); + scn_base_ = scn.get_val_for_logservice(); PALF_LOG(INFO, "truncate success", K(lsn), K(log_id), K(scn)); break; } else { @@ -126,7 +127,7 @@ int LSNAllocator::inc_update_last_log_info(const LSN &lsn, const int64_t log_id, break; } else if (CAS128(&lsn_ts_meta_, last, next)) { log_id_base_ = log_id; - scn_base_ = scn.get_val_for_lsn_allocator(); + scn_base_ = scn.get_val_for_logservice(); PALF_LOG(TRACE, "inc_update_last_log_info success", K(lsn), K(scn), K(log_id)); break; } else { @@ -148,7 +149,7 @@ int LSNAllocator::inc_update_scn_base(const SCN &ref_scn) } else { LSNTsMeta last; LSNTsMeta next; - const uint64_t scn = ref_scn.get_val_for_lsn_allocator(); + const uint64_t scn = ref_scn.get_val_for_logservice(); while (true) { WLockGuard guard(lock_); LOAD128(last, &lsn_ts_meta_); @@ -194,8 +195,8 @@ SCN LSNAllocator::get_max_scn() const LOAD128(last, &lsn_ts_meta_); max_scn = scn_base_ + last.scn_delta_; int ret = OB_SUCCESS; - if (OB_FAIL(result.convert_for_lsn_allocator(max_scn))) { - PALF_LOG(ERROR, "failed to convert_for_lsn_allocator", K(max_scn), + if (OB_FAIL(result.convert_for_logservice(max_scn))) { + PALF_LOG(ERROR, "failed to convert_for_logservice", K(max_scn), K(scn_base_), K(last.scn_delta_)); } } @@ -313,7 +314,7 @@ int LSNAllocator::alloc_lsn_scn(const SCN &base_scn, LOAD128(last, &lsn_ts_meta_); const int64_t last_log_id = log_id_base_ + last.log_id_delta_; const uint64_t last_scn = scn_base_ + last.scn_delta_; - const uint64_t new_scn = std::max(base_scn.get_val_for_lsn_allocator(), last_scn); + const uint64_t new_scn = std::max(base_scn.get_val_for_logservice(), last_scn); log_id_base_ = last_log_id; scn_base_ = new_scn; @@ -339,7 +340,7 @@ int LSNAllocator::alloc_lsn_scn(const SCN &base_scn, LOAD128(last, &lsn_ts_meta_); const int64_t last_log_id = log_id_base_ + last.log_id_delta_; const uint64_t last_scn = scn_base_ + last.scn_delta_; - const uint64_t tmp_next_scn = std::max(base_scn.get_val_for_lsn_allocator(), last_scn + 1); + const uint64_t tmp_next_scn = std::max(base_scn.get_val_for_logservice(), last_scn + 1); if ((tmp_next_scn + 1) - scn_base_ >= LOG_TS_DELTA_UPPER_BOUND) { // 对于可能生成的padding log, 也会占用一个scn @@ -456,7 +457,7 @@ int LSNAllocator::alloc_lsn_scn(const SCN &base_scn, } uint64_t scn_val = scn_base_ + output_next_scn_delta; - if (OB_FAIL(scn.convert_for_lsn_allocator(scn_val))) { + if (OB_FAIL(scn.convert_for_logservice(scn_val))) { PALF_LOG(ERROR, "failed to convert scn", K(ret), K(base_scn), K(scn)); } diff --git a/src/logservice/palf/lsn_allocator.h b/src/logservice/palf/lsn_allocator.h index 7ee319437..e5952d3e0 100644 --- a/src/logservice/palf/lsn_allocator.h +++ b/src/logservice/palf/lsn_allocator.h @@ -16,8 +16,8 @@ #include "lib/atomic/atomic128.h" #include "lib/lock/ob_tc_rwlock.h" #include "lib/utility/ob_macro_utils.h" +#include "share/scn.h" #include "lsn.h" -#include "scn.h" namespace oceanbase { @@ -29,7 +29,7 @@ public: LSNAllocator(); ~LSNAllocator(); public: - int init(const int64_t log_id, const SCN &scn, const LSN &start_lsn); + int init(const int64_t log_id, const share::SCN &scn, const LSN &start_lsn); void reset(); int get_log_block_size(const uint64_t block_id, int64_t &block_size) const { @@ -40,7 +40,7 @@ public: return OB_SUCCESS; } int64_t get_max_log_id() const; - SCN get_max_scn() const; + share::SCN get_max_scn() const; int get_curr_end_lsn(LSN &curr_end_lsn) const; int try_freeze_by_time(LSN &last_lsn, int64_t &last_log_id); int try_freeze(LSN &last_lsn, int64_t &last_log_id); @@ -55,20 +55,20 @@ public: // @param [out] need_gen_padding_entry: 是否需要在本条日志之前生成padding_entry // @param [out] padding_len: padding部分的总长度 // - int alloc_lsn_scn(const SCN &base_scn, + int alloc_lsn_scn(const share::SCN &base_scn, const int64_t size, LSN &lsn, int64_t &log_id, - SCN &scn, + share::SCN &scn, bool &is_new_log, bool &need_gen_padding_entry, int64_t &padding_len); // 更新last_lsn和log_timestamp // receive_log/append_disk_log 时调用 - int inc_update_last_log_info(const LSN &lsn, const int64_t log_id, const SCN &scn); + int inc_update_last_log_info(const LSN &lsn, const int64_t log_id, const share::SCN &scn); // inc update scn base, called by change access mode and to leader active - int inc_update_scn_base(const SCN &scn); - int truncate(const LSN &lsn, const int64_t log_id, const SCN &scn); + int inc_update_scn_base(const share::SCN &scn); + int truncate(const LSN &lsn, const int64_t log_id, const share::SCN &scn); // 获取last_lsn和log_timestamp TO_STRING_KV("max_log_id", get_max_log_id(), "max_lsn", lsn_ts_meta_.lsn_val_, "max_scn", get_max_scn()); diff --git a/src/logservice/palf/palf_base_info.h b/src/logservice/palf/palf_base_info.h index a18ed3acb..bd306d28c 100644 --- a/src/logservice/palf/palf_base_info.h +++ b/src/logservice/palf/palf_base_info.h @@ -14,8 +14,8 @@ #define OCEANBASE_LOGSERVICE_PALF_BASE_INFO_ #include "lib/utility/ob_unify_serialize.h" // OB_UNIS_VERSION +#include "share/scn.h" #include "lsn.h" -#include "scn.h" namespace oceanbase { @@ -40,7 +40,7 @@ public: int64_t version_; int64_t log_id_; LSN lsn_; - SCN scn_; + share::SCN scn_; int64_t log_proposal_id_; int64_t accum_checksum_; }; diff --git a/src/logservice/palf/palf_env_impl.cpp b/src/logservice/palf/palf_env_impl.cpp index 9de92be91..1647cbcfb 100644 --- a/src/logservice/palf/palf_env_impl.cpp +++ b/src/logservice/palf/palf_env_impl.cpp @@ -31,6 +31,7 @@ namespace oceanbase { using namespace common; +using namespace share; namespace palf { PalfHandleImpl *PalfHandleImplFactory::alloc() @@ -591,7 +592,7 @@ int PalfEnvImpl::try_freeze_log_for_all() PalfEnvImpl::LogGetRecycableFileCandidate::LogGetRecycableFileCandidate() : id_(-1), min_block_id_(LOG_INVALID_BLOCK_ID), - min_block_id_scn_(), + min_block_max_scn_(), min_using_block_id_(LOG_INVALID_BLOCK_ID), oldest_palf_id_(INVALID_PALF_ID), oldest_block_scn_(), @@ -602,7 +603,7 @@ PalfEnvImpl::LogGetRecycableFileCandidate::~LogGetRecycableFileCandidate() { ret_code_ = OB_SUCCESS; min_using_block_id_ = LOG_INVALID_BLOCK_ID; - min_block_id_scn_.reset(); + min_block_max_scn_.reset(); min_block_id_ = LOG_INVALID_BLOCK_ID; oldest_palf_id_ = INVALID_PALF_ID; oldest_block_scn_.reset(); @@ -620,14 +621,18 @@ bool PalfEnvImpl::LogGetRecycableFileCandidate::operator()(const LSKey &palf_id, const LSN base_lsn = palf_handle_impl->get_base_lsn_used_for_block_gc(); const block_id_t min_using_block_id = lsn_2_block(base_lsn, PALF_BLOCK_SIZE); block_id_t min_block_id = LOG_INVALID_BLOCK_ID; - SCN min_block_id_scn; + SCN min_block_max_scn; + // OB_ENTRY_EXIST means there is not any block; + // OB_NO_SUCH_FILE_OR_DIRECTORY means there is concurrently with rebuild. + // OB_ERR_OUT_OF_UPPER_BOUND means there is one block + auto need_skip_by_ret = [](const int ret ){ + return OB_ENTRY_EXIST == ret || OB_NO_SUCH_FILE_OR_DIRECTORY == ret + || OB_ERR_OUT_OF_UPPER_BOUND == ret; + }; if (false == base_lsn.is_valid()) { PALF_LOG(WARN, "base_lsn is invalid", K(base_lsn), KPC(palf_handle_impl)); - // OB_ENTRY_EXIST means there is not any block; - // OB_NO_SUCH_FILE_OR_DIRECTORY means there is concurrently with rebuild. - } else if (OB_FAIL(palf_handle_impl->get_min_block_id_min_scn(min_block_id, min_block_id_scn)) - && OB_ENTRY_NOT_EXIST != ret - && OB_NO_SUCH_FILE_OR_DIRECTORY != ret) { + } else if (OB_FAIL(palf_handle_impl->get_min_block_info_for_gc(min_block_id, min_block_max_scn)) + && !need_skip_by_ret(ret)) { ret_code_ = ret; bool_ret = false; PALF_LOG(WARN, "LogGetRecycableFileCandidate get_min_block_id_min_scn failed", K(ret), K(palf_id)); @@ -636,26 +641,25 @@ bool PalfEnvImpl::LogGetRecycableFileCandidate::operator()(const LSKey &palf_id, // 2. current palf_handle_impl must have older blocks(at least two blocks). // Always keep there are at least two blocks in range [begin_lsn, base_lsn], because for restart, we will read // first uncommitted log before base_lsn. - } else if (OB_ENTRY_NOT_EXIST == ret - || OB_NO_SUCH_FILE_OR_DIRECTORY == ret + } else if (need_skip_by_ret(ret) || min_using_block_id < min_block_id || min_using_block_id - min_block_id < 2) { PALF_LOG(TRACE, "can not recycle blocks, need keep at least two blocks or has been concurrently" " with rebuild, skip it", K(ret), KPC(palf_handle_impl), K(min_block_id), K(min_using_block_id)); - } else if (min_block_id_scn_.is_valid() && min_block_id_scn_ < min_block_id_scn) { - PALF_LOG(TRACE, "current palf_handle_impl is not older than previous, skip it", K(min_block_id_scn), - K(min_block_id_scn_), KPC(palf_handle_impl), K(min_block_id)); + } else if (min_block_max_scn_.is_valid() && min_block_max_scn_ < min_block_max_scn) { + PALF_LOG(TRACE, "current palf_handle_impl is not older than previous, skip it", K(min_block_max_scn), + K(min_block_max_scn_), KPC(palf_handle_impl), K(min_block_id)); } else { id_ = palf_id.id_; min_block_id_ = min_block_id; - min_block_id_scn_ = min_block_id_scn; + min_block_max_scn_ = min_block_max_scn; min_using_block_id_ = min_using_block_id; PALF_LOG(TRACE, "can be recycable palf_handle_impl", K(id_), K(min_block_id_), K(min_using_block_id_), - K(min_block_id_scn_), K(base_lsn)); + K(min_block_max_scn_), K(base_lsn)); } - if (min_block_id_scn.is_valid() && (!oldest_block_scn_.is_valid() || oldest_block_scn_ > min_block_id_scn)) { - oldest_block_scn_ = min_block_id_scn; + if (min_block_max_scn.is_valid() && (!oldest_block_scn_.is_valid() || oldest_block_scn_ > min_block_max_scn)) { + oldest_block_scn_ = min_block_max_scn; oldest_palf_id_ = palf_id.id_; } } diff --git a/src/logservice/palf/palf_env_impl.h b/src/logservice/palf/palf_env_impl.h index 47f01e146..31dad6528 100644 --- a/src/logservice/palf/palf_env_impl.h +++ b/src/logservice/palf/palf_env_impl.h @@ -22,6 +22,7 @@ #include "lib/utility/ob_macro_utils.h" #include "lib/utility/utility.h" #include "share/ob_occam_timer.h" +#include "share/scn.h" #include "fetch_log_engine.h" #include "log_loop_thread.h" #include "log_define.h" @@ -32,7 +33,6 @@ #include "palf_handle_impl.h" #include "log_io_worker.h" #include "block_gc_timer_task.h" -#include "scn.h" namespace oceanbase { namespace common @@ -253,12 +253,12 @@ private: bool operator() (const LSKey &palf_id, PalfHandleImpl *palf_handle_impl); int64_t id_; block_id_t min_block_id_; - SCN min_block_id_scn_; + share::SCN min_block_max_scn_; block_id_t min_using_block_id_; int64_t oldest_palf_id_; - SCN oldest_block_scn_; + share::SCN oldest_block_scn_; int ret_code_; - TO_STRING_KV(K_(id), K_(min_block_id_scn), K_(min_block_id), K_(min_using_block_id), K_(oldest_palf_id), K_(oldest_block_scn), K_(ret_code)); + TO_STRING_KV(K_(id), K_(min_block_max_scn), K_(min_block_id), K_(min_using_block_id), K_(oldest_palf_id), K_(oldest_block_scn), K_(ret_code)); }; struct GetTotalUsedDiskSpace { @@ -283,7 +283,7 @@ private: int get_disk_usage_(int64_t &used_size_byte, int64_t &palf_id, int64_t &maximum_used_size); - int recycle_blocks_(bool &has_recycled, int64_t &oldest_palf_id, SCN &oldest_scn); + int recycle_blocks_(bool &has_recycled, int64_t &oldest_palf_id, share::SCN &oldest_scn); int wait_until_reference_count_to_zero_(const int64_t palf_id); // check the diskspace whether is enough to hold a new palf instance. bool check_can_create_palf_handle_impl_() const; diff --git a/src/logservice/palf/palf_handle.cpp b/src/logservice/palf/palf_handle.cpp index 9369dcde1..756c5d8cd 100644 --- a/src/logservice/palf/palf_handle.cpp +++ b/src/logservice/palf/palf_handle.cpp @@ -20,6 +20,7 @@ namespace oceanbase { +using namespace share; namespace palf { #define CHECK_VALID if (NULL == palf_handle_impl_) { return OB_NOT_INIT; } diff --git a/src/logservice/palf/palf_handle.h b/src/logservice/palf/palf_handle.h index db5dc8ae8..81c6565f3 100644 --- a/src/logservice/palf/palf_handle.h +++ b/src/logservice/palf/palf_handle.h @@ -21,13 +21,16 @@ #include "palf_iterator.h" namespace oceanbase { +namespace share +{ +class SCN; +} namespace palf { class PalfAppendOptions; class PalfFSCb; class PalfRoleChangeCb; class PalfLocationCacheCb; -class SCN; class PalfHandle { public: @@ -79,9 +82,9 @@ public: int append(const PalfAppendOptions &opts, const void *buffer, const int64_t nbytes, - const SCN &ref_scn, + const share::SCN &ref_scn, LSN &lsn, - SCN &scn); + share::SCN &scn); int raw_write(const PalfAppendOptions &opts, const LSN &lsn, @@ -117,7 +120,7 @@ public: // - OB_ENTRY_NOT_EXIST: there is no log's scn is higher than scn // - OB_ERR_OUT_OF_LOWER_BOUND: scn is too old, log files may have been recycled // - others: bug - int seek(const SCN &scn, PalfGroupBufferIterator &iter); + int seek(const share::SCN &scn, PalfGroupBufferIterator &iter); // @desc: query coarse lsn by scn, that means there is a LogGroupEntry in disk, // its lsn and scn are result_lsn and result_scn, and result_scn <= scn. @@ -131,7 +134,7 @@ public: // - OB_ENTRY_NOT_EXIST: there is no log in disk // - OB_ERR_OUT_OF_LOWER_BOUND: scn is too small, log files may have been recycled // - others: bug - virtual int locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn); + virtual int locate_by_scn_coarsely(const share::SCN &scn, LSN &result_lsn); // @desc: query coarse scn by lsn, that means there is a log in disk, // its lsn and scn are result_lsn and result_scn, and result_lsn <= lsn. @@ -142,7 +145,7 @@ public: // - OB_INVALID_ARGUMENT // - OB_ERR_OUT_OF_LOWER_BOUND: lsn is too small, log files may have been recycled // - others: bug - virtual int locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn); + virtual int locate_by_lsn_coarsely(const LSN &lsn, share::SCN &result_scn); // 开启日志同步 virtual int enable_sync(); @@ -156,7 +159,7 @@ public: // 返回文件中可读的最早日志的位置信息 int get_begin_lsn(LSN &lsn) const; - int get_begin_scn(SCN &scn) const; + int get_begin_scn(share::SCN &scn) const; // PalfBaseInfo include the 'base_lsn' and the 'prev_log_info' of sliding window. // @param[in] const LSN&, base_lsn of ls. @@ -167,9 +170,9 @@ public: // 返回最后一条已确认日志的下一位置 // 在没有新的写入的场景下，返回的end_lsn不可读 virtual int get_end_lsn(LSN &lsn) const; - int get_end_scn(SCN &scn) const; + int get_end_scn(share::SCN &scn) const; int get_max_lsn(LSN &lsn) const; - int get_max_scn(SCN &scn) const; + int get_max_scn(share::SCN &scn) const; int get_last_rebuild_lsn(LSN &last_rebuild_lsn) const; //================= 分布式相关接口 ========================= @@ -369,7 +372,7 @@ public: int change_access_mode(const int64_t proposal_id, const int64_t mode_version, const AccessMode &access_mode, - const SCN &ref_scn); + const share::SCN &ref_scn); // @brief: query the access_mode of palf and it's corresponding mode_version // @param[out] palf::AccessMode &access_mode: current access_mode // @param[out] int64_t &mode_version: mode_version corresponding to AccessMode diff --git a/src/logservice/palf/palf_handle_impl.cpp b/src/logservice/palf/palf_handle_impl.cpp index 18058d695..4a4b31ce3 100644 --- a/src/logservice/palf/palf_handle_impl.cpp +++ b/src/logservice/palf/palf_handle_impl.cpp @@ -28,6 +28,7 @@ namespace oceanbase { using namespace common; +using namespace share; using namespace palf::election; namespace palf { @@ -301,14 +302,13 @@ int PalfHandleImpl::get_begin_scn(SCN &scn) if (IS_NOT_INIT) { ret = OB_NOT_INIT; PALF_LOG(WARN, "PalfHandleImpl not init", K(ret), KPC(this)); - } else if (OB_FAIL(log_engine_.get_min_block_id_and_min_scn(unused_block_id, scn))) { - PALF_LOG(WARN, "LogEngine get_min_block_id_and_min_scn failed", K(ret), KPC(this)); + } else if (OB_FAIL(log_engine_.get_min_block_info(unused_block_id, scn))) { + PALF_LOG(WARN, "LogEngine get_min_block_info failed", K(ret), KPC(this)); } return ret; } -int PalfHandleImpl::get_base_info(const LSN &base_lsn, - PalfBaseInfo &base_info) +int PalfHandleImpl::get_base_info(const LSN &base_lsn, PalfBaseInfo &base_info) { int ret = OB_SUCCESS; LSN curr_end_lsn = get_end_lsn(); @@ -1595,13 +1595,16 @@ int PalfHandleImpl::locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn) return ret; } -int PalfHandleImpl::get_min_block_id_min_scn(block_id_t &block_id, SCN &scn) +int PalfHandleImpl::get_min_block_info_for_gc(block_id_t &min_block_id, SCN &max_scn) { int ret = OB_SUCCESS; - if (OB_FAIL(log_engine_.get_min_block_id_and_min_scn(block_id, scn))) { - PALF_LOG(WARN, "get_min_block_id_and_min_scn failed", K(ret), KPC(this)); +// if (false == end_lsn.is_valid()) { +// ret = OB_ENTRY_NOT_EXIST; +// } + if (OB_FAIL(log_engine_.get_min_block_info_for_gc(min_block_id, max_scn))) { + PALF_LOG(WARN, "get_min_block_info_for_gc failed", K(ret), KPC(this)); } else { - PALF_LOG(TRACE, "get_min_block_id_and_min_scn success", K(ret), KPC(this), K(block_id), K(scn)); + PALF_LOG(TRACE, "get_min_block_info_for_gc success", K(ret), KPC(this), K(min_block_id), K(max_scn)); } return ret; } @@ -3482,7 +3485,7 @@ int PalfHandleImpl::stat(PalfStat &palf_stat) palf_stat.allow_vote_ = state_mgr_.is_allow_vote(); palf_stat.replica_type_ = state_mgr_.get_replica_type(); palf_stat.base_lsn_ = log_engine_.get_log_meta().get_log_snapshot_meta().base_lsn_; - (void)log_engine_.get_min_block_id_and_min_scn(min_block_id, min_block_min_scn); + (void)log_engine_.get_min_block_info(min_block_id, min_block_min_scn); palf_stat.begin_lsn_ = LSN(min_block_id * PALF_BLOCK_SIZE); palf_stat.begin_scn_ = min_block_min_scn; palf_stat.end_lsn_ = get_end_lsn(); diff --git a/src/logservice/palf/palf_handle_impl.h b/src/logservice/palf/palf_handle_impl.h index ccb7e7988..98ade24a3 100644 --- a/src/logservice/palf/palf_handle_impl.h +++ b/src/logservice/palf/palf_handle_impl.h @@ -23,11 +23,11 @@ #include "election/interface/election_msg_handler.h" #include "election/message/election_message.h" #include "election/algorithm/election_impl.h" +#include "share/scn.h" #include "palf_callback_wrapper.h" #include "log_engine.h" // LogEngine #include "log_meta.h" #include "lsn.h" -#include "scn.h" #include "log_config_mgr.h" #include "log_mode_mgr.h" #include "log_reconfirm.h" @@ -74,12 +74,12 @@ struct PalfStat { bool allow_vote_; LogReplicaType replica_type_; LSN begin_lsn_; - SCN begin_scn_; + share::SCN begin_scn_; LSN base_lsn_; LSN end_lsn_; - SCN end_scn_; + share::SCN end_scn_; LSN max_lsn_; - SCN max_scn_; + share::SCN max_scn_; TO_STRING_KV(K_(self), K_(palf_id), K_(role), K_(log_proposal_id), K_(config_version), K_(access_mode), K_(paxos_member_list), K_(paxos_replica_num), K_(allow_vote), K_(replica_type), K_(base_lsn), K_(end_lsn), K_(end_scn), K_(max_lsn)); @@ -182,9 +182,9 @@ public: virtual int submit_log(const PalfAppendOptions &opts, const char *buf, const int64_t buf_len, - const SCN &ref_scn, + const share::SCN &ref_scn, LSN &lsn, - SCN &scn) = 0; + share::SCN &scn) = 0; // 提交group_log到palf // 使用场景：备库leader处理从主库收到的日志 // @param [in] opts, 提交日志的一些可选项参数，具体参见PalfAppendOptions的定义 @@ -403,7 +403,7 @@ public: // - OB_ENTRY_NOT_EXIST: there is no log in disk // - OB_ERR_OUT_OF_LOWER_BOUND: scn is too old, log files may have been recycled // - others: bug - virtual int locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn) = 0; + virtual int locate_by_scn_coarsely(const share::SCN &scn, LSN &result_lsn) = 0; // @desc: query coarse scn by lsn, that means there is a LogGroupEntry in disk, // its lsn and scn are result_lsn and result_scn, and result_lsn <= lsn. @@ -419,12 +419,12 @@ public: // - OB_INVALID_ARGUMENT // - OB_ERR_OUT_OF_LOWER_BOUND: lsn is too small, log files may have been recycled // - others: bug - virtual int locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn) = 0; + virtual int locate_by_lsn_coarsely(const LSN &lsn, share::SCN &result_scn) = 0; virtual int get_begin_lsn(LSN &lsn) const = 0; - virtual int get_begin_scn(SCN &scn) = 0; + virtual int get_begin_scn(share::SCN &scn) = 0; virtual int get_base_info(const LSN &base_lsn, PalfBaseInfo &base_info) = 0; - virtual int get_min_block_id_min_scn(block_id_t &block_id, SCN &scn) = 0; + virtual int get_min_block_info_for_gc(block_id_t &min_block_id, share::SCN &max_scn) = 0; //begin lsn base lsn end lsn // │ │ │ // │ │ │ @@ -440,8 +440,8 @@ public: // return the block length which the previous data was committed virtual const LSN get_end_lsn() const = 0; virtual LSN get_max_lsn() const = 0; - virtual const SCN get_max_scn() const = 0; - virtual const SCN get_end_scn() const = 0; + virtual const share::SCN get_max_scn() const = 0; + virtual const share::SCN get_end_scn() const = 0; virtual int get_last_rebuild_lsn(LSN &last_rebuild_lsn) const = 0; virtual int64_t get_total_used_disk_space() const = 0; virtual const LSN &get_base_lsn_used_for_block_gc() const = 0; @@ -458,7 +458,7 @@ public: virtual int advance_reuse_lsn(const LSN &flush_log_end_lsn) = 0; virtual int inner_append_log(const LSN &lsn, const LogWriteBuf &write_buf, - const SCN &scn) = 0; + const share::SCN &scn) = 0; virtual int inner_append_log(const LSNArray &lsn_array, const LogWriteBufArray &write_buf_array, const SCNArray &scn_array) = 0; @@ -536,7 +536,7 @@ public: virtual int change_access_mode(const int64_t proposal_id, const int64_t mode_version, const AccessMode &access_mode, - const SCN &ref_scn) = 0; + const share::SCN &ref_scn) = 0; virtual int get_access_mode(int64_t &mode_version, AccessMode &access_mode) const = 0; virtual int get_access_mode(AccessMode &access_mode) const = 0; virtual int handle_committed_info(const common::ObAddr &server, @@ -558,7 +558,7 @@ public: PalfBufferIterator &iterator) = 0; virtual int alloc_palf_group_buffer_iterator(const LSN &offset, PalfGroupBufferIterator &iterator) = 0; - virtual int alloc_palf_group_buffer_iterator(const SCN &scn, + virtual int alloc_palf_group_buffer_iterator(const share::SCN &scn, PalfGroupBufferIterator &iterator) = 0; // ===================== Iterator end ======================= @@ -626,9 +626,9 @@ public: int submit_log(const PalfAppendOptions &opts, const char *buf, const int64_t buf_len, - const SCN &ref_scn, + const share::SCN &ref_scn, LSN &lsn, - SCN &scn) override final; + share::SCN &scn) override final; int submit_group_log(const PalfAppendOptions &opts, const LSN &lsn, @@ -677,8 +677,8 @@ public: int disable_sync(); bool is_sync_enabled() const; int advance_base_info(const PalfBaseInfo &palf_base_info, const bool is_rebuild) override final; - int locate_by_scn_coarsely(const SCN &scn, LSN &result_lsn) override final; - int locate_by_lsn_coarsely(const LSN &lsn, SCN &result_scn) override final; + int locate_by_scn_coarsely(const share::SCN &scn, LSN &result_lsn) override final; + int locate_by_lsn_coarsely(const LSN &lsn, share::SCN &result_scn) override final; void set_deleted(); int disable_vote() override final; int enable_vote() override final; @@ -692,13 +692,13 @@ public: int change_access_mode(const int64_t proposal_id, const int64_t mode_version, const AccessMode &access_mode, - const SCN &ref_scn) override final; + const share::SCN &ref_scn) override final; int get_access_mode(int64_t &mode_version, AccessMode &access_mode) const override final; int get_access_mode(AccessMode &access_mode) const override final; // =========================== Iterator start ============================ int alloc_palf_buffer_iterator(const LSN &offset, PalfBufferIterator &iterator) override final; int alloc_palf_group_buffer_iterator(const LSN &offset, PalfGroupBufferIterator &iterator) override final; - int alloc_palf_group_buffer_iterator(const SCN &scn, PalfGroupBufferIterator &iterator) override final; + int alloc_palf_group_buffer_iterator(const share::SCN &scn, PalfGroupBufferIterator &iterator) override final; // =========================== Iterator end ============================ // ==================== Callback start ====================== @@ -715,9 +715,9 @@ public: // ==================== Callback end ======================== public: int get_begin_lsn(LSN &lsn) const; - int get_begin_scn(SCN &scn); + int get_begin_scn(share::SCN &scn); int get_base_info(const LSN &base_lsn, PalfBaseInfo &base_info); - int get_min_block_id_min_scn(block_id_t &block_id, SCN &scn) override final; + int get_min_block_info_for_gc(block_id_t &min_block_id, share::SCN &max_scn) override final; // return the block length which the previous data was committed const LSN get_end_lsn() const override final { @@ -731,12 +731,12 @@ public: return sw_.get_max_lsn(); } - const SCN get_max_scn() const override final + const share::SCN get_max_scn() const override final { return sw_.get_max_scn(); } - const SCN get_end_scn() const override final + const share::SCN get_end_scn() const override final { // 基于实现复杂度考虑，直接用last_slide_scn作为end_scn // 否则需要在match_lsn_map中额外维护scn @@ -757,7 +757,7 @@ public: int advance_reuse_lsn(const LSN &flush_log_end_lsn); int inner_append_log(const LSN &lsn, const LogWriteBuf &write_buf, - const SCN &scn) override final; + const share::SCN &scn) override final; int inner_append_log(const LSNArray &lsn_array, const LogWriteBufArray &write_buf_array, const SCNArray &scn_array); @@ -890,11 +890,11 @@ private: const int64_t accepted_mode_pid); int try_update_proposal_id_(const common::ObAddr &server, const int64_t &proposal_id); - int get_binary_search_range_(const SCN &scn, + int get_binary_search_range_(const share::SCN &scn, block_id_t &min_block_id, block_id_t &max_block_id, block_id_t &result_block_id); - void inc_update_last_locate_block_scn_(const block_id_t &block_id, const SCN &scn); + void inc_update_last_locate_block_scn_(const block_id_t &block_id, const share::SCN &scn); int can_change_config_(const LogConfigChangeArgs &args, int64_t &proposal_id); int check_args_and_generate_config_(const LogConfigChangeArgs &args, bool &is_already_finished, @@ -993,7 +993,7 @@ private: PalfLocationCacheCb *lc_cb_; // ======optimization for locate_by_scn_coarsely========= mutable SpinLock last_locate_lock_; - SCN last_locate_scn_; + share::SCN last_locate_scn_; block_id_t last_locate_block_; // ======optimization for locate_by_scn_coarsely========= int64_t cannot_recv_log_warn_time_; diff --git a/src/logservice/palf/palf_iterator.h b/src/logservice/palf/palf_iterator.h index 4d2d6faec..3de7b315e 100644 --- a/src/logservice/palf/palf_iterator.h +++ b/src/logservice/palf/palf_iterator.h @@ -16,9 +16,12 @@ #include "log_iterator_storage.h" // LogIteratorStorage namespace oceanbase { -namespace palf +namespace share { class SCN; +} +namespace palf +{ template class PalfIterator { @@ -88,7 +91,7 @@ public: } return ret; } - int get_entry(const char *&buffer, int64_t &nbytes, SCN &scn, LSN &lsn, bool &is_raw_write) + int get_entry(const char *&buffer, int64_t &nbytes, share::SCN &scn, LSN &lsn, bool &is_raw_write) { int ret = OB_SUCCESS; LogEntryType entry; @@ -105,7 +108,7 @@ public: } return ret; } - int get_entry(const char *&buffer, int64_t &nbytes, SCN &scn, LSN &lsn) + int get_entry(const char *&buffer, int64_t &nbytes, share::SCN &scn, LSN &lsn) { int ret = OB_SUCCESS; LogEntryType entry; diff --git a/src/logservice/palf_handle_guard.h b/src/logservice/palf_handle_guard.h index 80e398144..2f2ebad52 100644 --- a/src/logservice/palf_handle_guard.h +++ b/src/logservice/palf_handle_guard.h @@ -91,7 +91,7 @@ public: return palf_handle_.seek(start_lsn, iter); } - int seek(const palf::SCN &scn, + int seek(const share::SCN &scn, PalfGroupBufferIterator &iter) { return palf_handle_.seek(scn, iter); diff --git a/src/logservice/replayservice/ob_log_replay_service.cpp b/src/logservice/replayservice/ob_log_replay_service.cpp index 5c96d00c7..e877bdf05 100644 --- a/src/logservice/replayservice/ob_log_replay_service.cpp +++ b/src/logservice/replayservice/ob_log_replay_service.cpp @@ -15,7 +15,7 @@ #include "logservice/ob_log_base_header.h" #include "logservice/ob_ls_adapter.h" #include "logservice/palf/palf_env.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_thread_mgr.h" #include "share/rc/ob_tenant_base.h" #include "storage/tx_storage/ob_tenant_freezer.h" @@ -572,7 +572,7 @@ int ObLogReplayService::is_replay_done(const share::ObLSID &id, return ret; } -int ObLogReplayService::get_min_unreplayed_scn(const share::ObLSID &id, palf::SCN &scn) +int ObLogReplayService::get_min_unreplayed_scn(const share::ObLSID &id, SCN &scn) { int ret = OB_SUCCESS; ObReplayStatus *replay_status = NULL; @@ -728,7 +728,8 @@ bool ObLogReplayService::is_tenant_out_of_memory_() const total_memstore_used, memstore_freeze_trigger, memstore_limit, - freeze_cnt)))) { + freeze_cnt, + false)))) { CLOG_LOG(WARN, "get_tenant_memstore_cond failed", K(ret)); } else { // Estimate the size of memstore based on 16 times expansion @@ -907,13 +908,9 @@ int ObLogReplayService::check_can_submit_log_replay_task_(ObLogReplayTask *repla } else { is_wait_barrier = true; } - } else if (replay_status->need_check_memstore(replay_task->lsn_)) { - if (OB_UNLIKELY(is_tenant_out_of_memory_())) { - ret = OB_EAGAIN; - is_tenant_out_of_mem = true; - } else { - replay_status->set_last_check_memstore_lsn(replay_task->lsn_); - } + } else if (OB_UNLIKELY(is_tenant_out_of_memory_())) { + ret = OB_EAGAIN; + is_tenant_out_of_mem = true; } if (OB_EAGAIN == ret && REACH_TIME_INTERVAL(5 * 1000 * 1000)) { CLOG_LOG(INFO, "submit replay task need retry", K(ret), KPC(replay_status), KPC(replay_task), K(current_replayable_point), diff --git a/src/logservice/replayservice/ob_log_replay_service.h b/src/logservice/replayservice/ob_log_replay_service.h index 569323268..ec282a47f 100644 --- a/src/logservice/replayservice/ob_log_replay_service.h +++ b/src/logservice/replayservice/ob_log_replay_service.h @@ -17,14 +17,17 @@ #include "logservice/ob_log_base_header.h" #include "lib/thread/thread_mgr_interface.h" #include "lib/hash/ob_linear_hash_map.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { +namespace share +{ +class SCN; +} namespace palf { class PalfEnv; -class SCN; } namespace logservice { @@ -122,7 +125,7 @@ public: int remove_ls(const share::ObLSID &id); int enable(const share::ObLSID &id, const palf::LSN &base_lsn, - const palf::SCN &base_scn); + const share::SCN &base_scn); int disable(const share::ObLSID &id); int is_enabled(const share::ObLSID &id, bool &is_enabled); int set_submit_log_pending(const share::ObLSID &id); @@ -134,9 +137,9 @@ public: const palf::LSN &end_lsn, bool &is_done); int get_min_unreplayed_scn(const share::ObLSID &id, - palf::SCN &scn); + share::SCN &scn); int submit_task(ObReplayServiceTask *task); - int update_replayable_point(const palf::SCN &replayable_scn); + int update_replayable_point(const share::SCN &replayable_scn); int stat_for_each(const common::ObFunction &func); int stat_all_ls_replay_process(int64_t &replayed_log_size, int64_t &unreplayed_log_size); void inc_pending_task_size(const int64_t log_size); @@ -159,7 +162,7 @@ private: int fetch_and_submit_single_log_(ObReplayStatus &replay_status, ObReplayServiceSubmitTask *submit_task, palf::LSN &cur_lsn, - palf::SCN &cur_log_submit_scn, + share::SCN &cur_log_submit_scn, int64_t &log_size); int fetch_pre_barrier_log_(ObReplayStatus &replay_status, ObReplayServiceSubmitTask *submit_task, @@ -167,7 +170,7 @@ private: const ObLogBaseHeader &header, const char *log_buf, const palf::LSN &cur_lsn, - const palf::SCN &cur_log_submit_scn, + const share::SCN &cur_log_submit_scn, const int64_t log_size, const bool is_raw_write); bool is_tenant_out_of_memory_() const; @@ -213,7 +216,7 @@ private: ObLSAdapter *ls_adapter_; palf::PalfEnv *palf_env_; ObILogAllocator *allocator_; - palf::SCN replayable_point_; + share::SCN replayable_point_; // 考虑到迁出迁入场景, 不能只通过map管理replay status的生命周期 common::ObLinearHashMap replay_status_map_; int64_t pending_replay_log_size_; diff --git a/src/logservice/replayservice/ob_replay_handler.cpp b/src/logservice/replayservice/ob_replay_handler.cpp index 0e8973b1e..4afab0ac9 100644 --- a/src/logservice/replayservice/ob_replay_handler.cpp +++ b/src/logservice/replayservice/ob_replay_handler.cpp @@ -17,6 +17,7 @@ namespace oceanbase { using namespace palf; +using namespace share; namespace logservice { ObReplayHandler::ObReplayHandler(storage::ObLS *ls) diff --git a/src/logservice/replayservice/ob_replay_handler.h b/src/logservice/replayservice/ob_replay_handler.h index 08cca469d..14cc62701 100644 --- a/src/logservice/replayservice/ob_replay_handler.h +++ b/src/logservice/replayservice/ob_replay_handler.h @@ -24,7 +24,7 @@ namespace storage { class ObLS; } -namespace palf +namespace share { class SCN; } @@ -46,7 +46,7 @@ public: const void *buffer, const int64_t nbytes, const palf::LSN &lsn, - const palf::SCN &scn); + const share::SCN &scn); private: typedef common::RWLock::WLockGuard WLockGuard; typedef common::RWLock::RLockGuard RLockGuard; diff --git a/src/logservice/replayservice/ob_replay_status.cpp b/src/logservice/replayservice/ob_replay_status.cpp index aa7a023b4..8ec8edd71 100644 --- a/src/logservice/replayservice/ob_replay_status.cpp +++ b/src/logservice/replayservice/ob_replay_status.cpp @@ -21,6 +21,7 @@ namespace oceanbase { using namespace common; using namespace palf; +using namespace share; namespace logservice { //---------------ObReplayServiceTask---------------// @@ -94,7 +95,7 @@ bool ObReplayServiceTask::need_replay_immediately() const //---------------ObReplayServiceSubmitTask---------------// int ObReplayServiceSubmitTask::init(const palf::LSN &base_lsn, - const palf::SCN &base_scn, + const SCN &base_scn, PalfHandle *palf_handle, ObReplayStatus *replay_status) { diff --git a/src/logservice/replayservice/ob_replay_status.h b/src/logservice/replayservice/ob_replay_status.h index 779ea70f6..111901c45 100644 --- a/src/logservice/replayservice/ob_replay_status.h +++ b/src/logservice/replayservice/ob_replay_status.h @@ -17,7 +17,7 @@ #include "logservice/ob_log_base_header.h" #include "logservice/ob_log_base_type.h" #include "logservice/palf/lsn.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "logservice/palf/palf_callback.h" #include "logservice/palf/palf_iterator.h" #include "logservice/palf/palf_handle.h" @@ -64,7 +64,7 @@ struct LSReplayStat palf::LSN end_lsn_; bool enabled_; palf::LSN unsubmitted_lsn_; - palf::SCN unsubmitted_scn_; + share::SCN unsubmitted_scn_; int64_t pending_cnt_; TO_STRING_KV(K(ls_id_), @@ -112,7 +112,7 @@ public: int init(const share::ObLSID &ls_id, const ObLogBaseHeader &header, const palf::LSN &lsn, - const palf::SCN &scn, + const share::SCN &scn, const int64_t log_size, const bool is_raw_write, void *log_buf); @@ -123,7 +123,7 @@ public: share::ObLSID ls_id_; ObLogBaseType log_type_; palf::LSN lsn_; - palf::SCN scn_; + share::SCN scn_; bool is_pre_barrier_; bool is_post_barrier_; int64_t log_size_; @@ -253,7 +253,7 @@ public: destroy(); } int init(const palf::LSN &base_lsn, - const palf::SCN &base_scn, + const share::SCN &base_scn, palf::PalfHandle *palf_handle, ObReplayStatus *replay_status); void reset() override; @@ -263,15 +263,15 @@ public: // 迭代器是否迭代到终点 bool has_remained_submit_log(); // 不允许回退 - int update_next_to_submit_log_info(const palf::LSN &lsn, const palf::SCN &scn); + int update_next_to_submit_log_info(const palf::LSN &lsn, const share::SCN &scn); int update_next_to_submit_lsn(const palf::LSN &lsn); - int update_next_to_submit_scn_allow_equal(const palf::SCN &scn); + int update_next_to_submit_scn_allow_equal(const share::SCN &scn); int update_committed_end_offset(const palf::LSN &lsn); - int get_next_to_submit_log_info(palf::LSN &lsn, palf::SCN &scn) const; + int get_next_to_submit_log_info(palf::LSN &lsn, share::SCN &scn) const; int get_committed_end_lsn(palf::LSN &lsn) const; int get_base_lsn(palf::LSN &lsn) const; - int get_base_scn(palf::SCN &scn) const; - int need_skip(const palf::SCN &scn, bool &need_skip); + int get_base_scn(share::SCN &scn) const; + int need_skip(const share::SCN &scn, bool &need_skip); bool is_cached_replay_task_exist() const { return NULL != cache_replay_task_; @@ -292,7 +292,7 @@ public: //释放内存,在废弃cache_replay_task时调用 void revert_cached_replay_task(); - int get_log(const char *&buffer, int64_t &nbytes, palf::SCN &scn, palf::LSN &offset, bool &is_raw_write); + int get_log(const char *&buffer, int64_t &nbytes, share::SCN &scn, palf::LSN &offset, bool &is_raw_write); int next_log(); // 以当前的终点作为新起点重置迭代器 int reset_iterator(palf::PalfHandle &palf_handle, @@ -306,24 +306,24 @@ public: K(base_scn_)); private: int update_next_to_submit_lsn_(const palf::LSN &lsn); - int update_next_to_submit_scn_(const palf::SCN &scn); - int update_next_to_submit_scn_allow_equal_(const palf::SCN &scn); + int update_next_to_submit_scn_(const share::SCN &scn); + int update_next_to_submit_scn_allow_equal_(const share::SCN &scn); int update_committed_end_lsn_(const palf::LSN &lsn); - void set_next_to_submit_log_info_(const palf::LSN &lsn, const palf::SCN &scn); - int get_next_to_submit_log_info_(palf::LSN &lsn, palf::SCN &scn) const; + void set_next_to_submit_log_info_(const palf::LSN &lsn, const share::SCN &scn); + int get_next_to_submit_log_info_(palf::LSN &lsn, share::SCN &scn) const; int get_base_lsn_(palf::LSN &lsn) const; - int get_base_scn_(palf::SCN &scn) const; + int get_base_scn_(share::SCN &scn) const; private: //location of next log after the last log that has already been submit to replay, consider as left side of iterator palf::LSN next_to_submit_lsn_; //location of the last log that need submit to replay, consider as right side of iterator palf::LSN committed_end_lsn_; - palf::SCN next_to_submit_scn_; + share::SCN next_to_submit_scn_; //initial log lsn when enable replay, for stat replay process palf::LSN base_lsn_; //initial log scn when enable replay, logs which scn small than this value should skip replay - palf::SCN base_scn_; + share::SCN base_scn_; //for unittest, should be a member not pointer palf::PalfBufferIterator iterator_; //缓存需要重试的log replay task @@ -365,7 +365,7 @@ public: queue_.push(p); } int get_min_unreplayed_log_info(palf::LSN &lsn, - palf::SCN &scn, + share::SCN &scn, bool &is_queue_empty); private: Link *pop_() @@ -439,7 +439,7 @@ public: void destroy(); public: int enable(const palf::LSN &base_lsn, - const palf::SCN &base_scn); + const share::SCN &base_scn); int disable(); // if is_enabled_ is false, // means log stream will bedestructed and no logs need to replayed any more. @@ -493,9 +493,9 @@ public: int get_ls_id(share::ObLSID &id); int get_min_unreplayed_lsn(palf::LSN &lsn); - int get_min_unreplayed_scn(palf::SCN &scn); + int get_min_unreplayed_scn(share::SCN &scn); int get_min_unreplayed_log_info(palf::LSN &lsn, - palf::SCN &scn); + share::SCN &scn); int get_replay_process(int64_t &replayed_log_size, int64_t &unreplayed_log_size); //提交日志检查barrier状态 int check_submit_barrier(); @@ -547,11 +547,11 @@ public: K(submit_log_task_)); private: - void set_next_to_submit_log_info_(const palf::LSN &lsn, const palf::SCN &scn); + void set_next_to_submit_log_info_(const palf::LSN &lsn, const share::SCN &scn); int submit_task_to_replay_service_(ObReplayServiceTask &task); // 注册回调并提交当前初始化的submit_log_task int enable_(const palf::LSN &base_lsn, - const palf::SCN &base_scn); + const share::SCN &base_scn); // 注销回调并清空任务 int disable_(); diff --git a/src/logservice/restoreservice/ob_fetch_log_task.cpp b/src/logservice/restoreservice/ob_fetch_log_task.cpp index bb7b6ab1d..d5ec15207 100644 --- a/src/logservice/restoreservice/ob_fetch_log_task.cpp +++ b/src/logservice/restoreservice/ob_fetch_log_task.cpp @@ -20,6 +20,7 @@ namespace oceanbase { using namespace palf; +using namespace share; namespace logservice { ObFetchLogTask::ObFetchLogTask(const share::ObLSID &id, diff --git a/src/logservice/restoreservice/ob_fetch_log_task.h b/src/logservice/restoreservice/ob_fetch_log_task.h index d904254e6..ed2621034 100644 --- a/src/logservice/restoreservice/ob_fetch_log_task.h +++ b/src/logservice/restoreservice/ob_fetch_log_task.h @@ -16,7 +16,7 @@ #include "lib/utility/ob_print_utils.h" #include "share/ob_ls_id.h" // ObLSID #include "logservice/palf/lsn.h" // LSN -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // SCN #include namespace oceanbase { @@ -40,7 +40,7 @@ public: public: ObFetchLogTask(const share::ObLSID &id, - const palf::SCN &pre_scn, + const share::SCN &pre_scn, const palf::LSN &lsn, const int64_t size, const int64_t proposal_id); @@ -49,7 +49,7 @@ public: bool is_finish() const { return Status::FINISH == status_; } bool is_to_end() const { return Status::TO_END == status_; } void mark_to_end() { status_ = Status::TO_END; } - int update_cur_lsn_scn(const palf::LSN &lsn, const palf::SCN &max_submit_scn, const palf::SCN &max_fetch_scn); + int update_cur_lsn_scn(const palf::LSN &lsn, const share::SCN &max_submit_scn, const share::SCN &max_fetch_scn); void set_to_end() { status_ = Status::TO_END; } void set_stale() { status_ = Status::STALE; } TO_STRING_KV(K_(id), K_(proposal_id), K_(pre_scn), K_(start_lsn), K_(cur_lsn), K_(end_lsn), @@ -59,12 +59,12 @@ public: share::ObLSID id_; // to distinguish stale tasks which is generated in previous leader int64_t proposal_id_; - palf::SCN pre_scn_; // heuristic log scn to locate piece, may be imprecise one + share::SCN pre_scn_; // heuristic log scn to locate piece, may be imprecise one palf::LSN start_lsn_; palf::LSN cur_lsn_; palf::LSN end_lsn_; - palf::SCN max_fetch_scn_; // 拉取日志最大log scn - palf::SCN max_submit_scn_; // 提交日志最大log cn + share::SCN max_fetch_scn_; // 拉取日志最大log scn + share::SCN max_submit_scn_; // 提交日志最大log cn Status status_; }; diff --git a/src/logservice/restoreservice/ob_log_archive_piece_mgr.cpp b/src/logservice/restoreservice/ob_log_archive_piece_mgr.cpp index 7d0cd5506..e8cdddb48 100644 --- a/src/logservice/restoreservice/ob_log_archive_piece_mgr.cpp +++ b/src/logservice/restoreservice/ob_log_archive_piece_mgr.cpp @@ -51,7 +51,7 @@ void ObLogArchivePieceContext::RoundContext::reset() bool ObLogArchivePieceContext::RoundContext::is_valid() const { - return ((RoundContext::State::ACTIVE == state_ && end_scn_ == palf::SCN::max_scn()) + return ((RoundContext::State::ACTIVE == state_ && end_scn_ == SCN::max_scn()) || (RoundContext::State::STOP == state_ && end_scn_ > start_scn_)) && round_id_ > 0 && start_scn_.is_valid() @@ -195,7 +195,7 @@ int ObLogArchivePieceContext::init(const share::ObLSID &id, return ret; } -int ObLogArchivePieceContext::get_piece(const palf::SCN &pre_scn, +int ObLogArchivePieceContext::get_piece(const SCN &pre_scn, const palf::LSN &start_lsn, int64_t &dest_id, int64_t &round_id, @@ -266,7 +266,7 @@ int ObLogArchivePieceContext::update_file_info(const int64_t dest_id, return ret; } -int ObLogArchivePieceContext::get_round_(const palf::SCN &start_scn) +int ObLogArchivePieceContext::get_round_(const SCN &start_scn) { int ret = OB_SUCCESS; int64_t round_id = 0; @@ -385,7 +385,7 @@ int ObLogArchivePieceContext::load_round_(const int64_t round_id, RoundContext & return ret; } -int ObLogArchivePieceContext::get_piece_(const palf::SCN &scn, +int ObLogArchivePieceContext::get_piece_(const SCN &scn, const palf::LSN &lsn, const int64_t file_id, int64_t &dest_id, @@ -417,7 +417,7 @@ int ObLogArchivePieceContext::get_piece_(const palf::SCN &scn, return ret; } -int ObLogArchivePieceContext::switch_round_if_need_(const palf::SCN &scn, const palf::LSN &lsn) +int ObLogArchivePieceContext::switch_round_if_need_(const SCN &scn, const palf::LSN &lsn) { int ret = OB_SUCCESS; RoundOp op = RoundOp::NONE; @@ -662,7 +662,7 @@ int ObLogArchivePieceContext::check_round_exist_(const int64_t round_id, bool &e return ret; } -int ObLogArchivePieceContext::switch_piece_if_need_(const int64_t file_id, const palf::SCN &scn, const palf::LSN &lsn) +int ObLogArchivePieceContext::switch_piece_if_need_(const int64_t file_id, const SCN &scn, const palf::LSN &lsn) { int ret = OB_SUCCESS; PieceOp op = PieceOp::NONE; @@ -760,7 +760,7 @@ void ObLogArchivePieceContext::check_if_switch_piece_(const int64_t file_id, // 1. inner_piece_context.round_id == round_context.round_id 说明依然消费当前round, 如果当前piece_id有效, 并且该piece依然active, 则继续刷新该piece // 2. inner_piece_context.round_id == round_context.round_id 并且piece状态为FROZEN或者EMPTY, 非预期错误 // 3. inner_piece_context.round_id != round_context.round_id, 需要由scn以及round piece范围, 决定piece id -int ObLogArchivePieceContext::get_cur_piece_info_(const palf::SCN &scn, const palf::LSN &lsn) +int ObLogArchivePieceContext::get_cur_piece_info_(const SCN &scn, const palf::LSN &lsn) { int ret = OB_SUCCESS; int64_t piece_id = 0; @@ -781,7 +781,7 @@ int ObLogArchivePieceContext::get_cur_piece_info_(const palf::SCN &scn, const pa return ret; } -int ObLogArchivePieceContext::cal_load_piece_id_(const palf::SCN &scn, int64_t &piece_id) +int ObLogArchivePieceContext::cal_load_piece_id_(const SCN &scn, int64_t &piece_id) { int ret = OB_SUCCESS; const int64_t base_piece_id = cal_piece_id_(scn); @@ -955,7 +955,7 @@ int ObLogArchivePieceContext::backward_piece_() return ret; } -int64_t ObLogArchivePieceContext::cal_piece_id_(const palf::SCN &scn) const +int64_t ObLogArchivePieceContext::cal_piece_id_(const SCN &scn) const { return share::ObArchivePiece(scn, round_context_.piece_switch_interval_, round_context_.base_piece_scn_, round_context_.base_piece_id_) diff --git a/src/logservice/restoreservice/ob_log_archive_piece_mgr.h b/src/logservice/restoreservice/ob_log_archive_piece_mgr.h index 87f7b3dae..56e9d0fc7 100644 --- a/src/logservice/restoreservice/ob_log_archive_piece_mgr.h +++ b/src/logservice/restoreservice/ob_log_archive_piece_mgr.h @@ -17,7 +17,7 @@ #include "lib/ob_errno.h" #include "lib/utility/ob_print_utils.h" #include "logservice/palf/lsn.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/backup/ob_backup_struct.h" #include "share/ob_ls_id.h" #include "ob_log_restore_define.h" @@ -47,7 +47,7 @@ public: void reset(); - int get_piece(const palf::SCN &pre_scn, + int get_piece(const share::SCN &pre_scn, const palf::LSN &start_lsn, int64_t &dest_id, int64_t &round_id, @@ -101,14 +101,14 @@ private: State state_; int64_t round_id_; - palf::SCN start_scn_; // 该轮次最小日志SCN - palf::SCN end_scn_; // 该轮次最大日志SCN, 非STOP状态该时间为SCN_MAX + share::SCN start_scn_; // 该轮次最小日志SCN + share::SCN end_scn_; // 该轮次最大日志SCN, 非STOP状态该时间为SCN_MAX int64_t min_piece_id_; // 该round最小piece, 随着归档数据回收, 该值可能增大 int64_t max_piece_id_; // 该round最大piece, ACTIVE round该值可能增大 int64_t base_piece_id_; int64_t piece_switch_interval_;//us - palf::SCN base_piece_scn_; + share::SCN base_piece_scn_; RoundContext() { reset(); } ~RoundContext() { reset(); } @@ -169,7 +169,7 @@ private: }; private: - int get_piece_(const palf::SCN &scn, + int get_piece_(const share::SCN &scn, const palf::LSN &lsn, const int64_t file_id, int64_t &dest_id, @@ -185,10 +185,10 @@ private: virtual int get_round_range_(); // 根据时间戳定位round - virtual int get_round_(const palf::SCN &scn); + virtual int get_round_(const share::SCN &scn); // 如果round不满足数据需求, 支持切round - int switch_round_if_need_(const palf::SCN &scn, const palf::LSN &lsn); + int switch_round_if_need_(const share::SCN &scn, const palf::LSN &lsn); void check_if_switch_round_(const palf::LSN &lsn, RoundOp &op); bool is_max_round_done_(const palf::LSN &lsn) const; @@ -210,11 +210,11 @@ private: int backward_round_(); // 当piece不匹配, 需要切piece - int switch_piece_if_need_(const int64_t file_id, const palf::SCN &scn, const palf::LSN &lsn); + int switch_piece_if_need_(const int64_t file_id, const share::SCN &scn, const palf::LSN &lsn); void check_if_switch_piece_(const int64_t file_id, const palf::LSN &lsn, PieceOp &op); // load当前piece信息, 包括piece内文件范围, LSN范围 - int get_cur_piece_info_(const palf::SCN &scn, const palf::LSN &lsn); + int get_cur_piece_info_(const share::SCN &scn, const palf::LSN &lsn); virtual int get_piece_meta_info_(const int64_t piece_id); int get_ls_inner_piece_info_(const share::ObLSID &id, const int64_t dest_id, const int64_t round_id, const int64_t piece_id, palf::LSN &min_lsn, palf::LSN &max_lsn, bool &exist); @@ -222,9 +222,9 @@ private: int forward_piece_(); int backward_piece_(); - int cal_load_piece_id_(const palf::SCN &scn, int64_t &piece_id); + int cal_load_piece_id_(const share::SCN &scn, int64_t &piece_id); - int64_t cal_piece_id_(const palf::SCN &scn) const; + int64_t cal_piece_id_(const share::SCN &scn) const; virtual int get_min_lsn_in_piece_(); int64_t cal_archive_file_id_(const palf::LSN &lsn) const; diff --git a/src/logservice/restoreservice/ob_log_restore_handler.h b/src/logservice/restoreservice/ob_log_restore_handler.h index f541aa27c..1b5adc062 100644 --- a/src/logservice/restoreservice/ob_log_restore_handler.h +++ b/src/logservice/restoreservice/ob_log_restore_handler.h @@ -71,15 +71,15 @@ public: // @param[in], proposal_id, global monotonically increasing id virtual void switch_role(const common::ObRole &role, const int64_t proposal_id) override; // interface only for restore_and_standby branch, TODO should delete after no cut log by shuning.tsn - int get_upper_limit_scn(palf::SCN &scn) const; + int get_upper_limit_scn(share::SCN &scn) const; // get the log ts of the max restored log - int get_max_restore_scn(palf::SCN &scn) const; + int get_max_restore_scn(share::SCN &scn) const; // @brief add log fetch source of phyiscal backups - int add_source(share::DirArray &array, const palf::SCN &end_scn); + int add_source(share::DirArray &array, const share::SCN &end_scn); // for stale TODO delete after log archive source ready - int add_source(logservice::DirArray &array, const palf::SCN &end_scn); - int add_source(share::ObBackupDest &dest, const palf::SCN &end_scn); - int add_source(const ObAddr &addr, const palf::SCN &end_scn); + int add_source(logservice::DirArray &array, const share::SCN &end_scn); + int add_source(share::ObBackupDest &dest, const share::SCN &end_scn); + int add_source(const ObAddr &addr, const share::SCN &end_scn); // @brief As restore handler maybe destroyed, log source should be copied out void deep_copy_source(ObRemoteSourceGuard &guard); // @brief raw write logs to the pointed offset of palf @@ -99,7 +99,7 @@ public: // @param[in] is_to_end, log fetch is to end, only for physical restore // @param[out] is_stale, fetch log task is stale due to cur_proposal_id is changed int update_fetch_log_progress(const int64_t id, const int64_t proposal_id, const LSN &max_fetch_lsn, - const palf::SCN &max_submit_scn, const bool is_finish, const bool is_to_end, bool &is_stale); + const share::SCN &max_submit_scn, const bool is_finish, const bool is_to_end, bool &is_stale); int update_location_info(ObRemoteLogParent *source); // @brief check if restore finish // return true only if in restore state and all replicas have restore and replay finish @@ -108,7 +108,7 @@ public: // param[in], const share::ObLSID, the identification of LS // param[in], int64_t, the commit ts of LS // @param[out], int64_t, the restore upper ts if restore to the end - int get_restore_sync_scn(const share::ObLSID &id, palf::SCN &scn); + int get_restore_sync_scn(const share::ObLSID &id, share::SCN &scn); void mark_error(share::ObTaskId &trace_id, const int ret_code); int get_restore_error(share::ObTaskId &trace_id, int &ret_code, bool &error_exist); TO_STRING_KV(K_(is_inited), K_(is_in_stop_state), K_(id), K_(proposal_id), K_(role), KP_(parent), K_(context)); @@ -116,8 +116,8 @@ public: private: bool is_valid() const; void alloc_source(const share::ObLogArchiveSourceType &type); - int check_replay_done_(const palf::SCN &scn, bool &done); - int check_replica_replay_done_(const palf::SCN &scn, common::ObMemberList &member_list, bool &done); + int check_replay_done_(const share::SCN &scn, bool &done); + int check_replica_replay_done_(const share::SCN &scn, common::ObMemberList &member_list, bool &done); int check_member_list_change_(common::ObMemberList &member_list, bool &member_list_change); private: diff --git a/src/logservice/restoreservice/ob_remote_data_generator.h b/src/logservice/restoreservice/ob_remote_data_generator.h index 1a2dcad19..d30bba5df 100644 --- a/src/logservice/restoreservice/ob_remote_data_generator.h +++ b/src/logservice/restoreservice/ob_remote_data_generator.h @@ -26,7 +26,7 @@ #include "logservice/palf/log_iterator_storage.h" // MemoryStorage #include "logservice/palf/palf_iterator.h" // MemPalfGroupBufferIterator #include "logservice/palf/lsn.h" // LSN -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // SCN #include "ob_remote_log_source.h" // Parent #include "ob_log_restore_rpc_define.h" // ObRemoteFetchLogResponse @@ -73,7 +73,7 @@ public: const ObLSID &id, const LSN &start_lsn, const LSN &end_lsn, - const palf::SCN &end_scn); + const share::SCN &end_scn); virtual ~RemoteDataGenerator(); public: @@ -92,7 +92,7 @@ protected: ObLSID id_; LSN start_lsn_; LSN next_fetch_lsn_; - palf::SCN end_scn_; + share::SCN end_scn_; LSN end_lsn_; bool to_end_; LSN max_consumed_lsn_; @@ -108,7 +108,7 @@ public: const ObLSID &id, const LSN &start_lsn, const LSN &end_lsn, - const palf::SCN &end_scn, + const share::SCN &end_scn, const ObAddr &server); virtual ~ServiceDataGenerator(); @@ -134,11 +134,11 @@ class LocationDataGenerator : public RemoteDataGenerator static const int64_t MAX_DATA_BUF_LEN = 128 * 1024 * 1024L; // 128M public: LocationDataGenerator(const uint64_t tenant_id, - const palf::SCN &pre_scn, + const share::SCN &pre_scn, const ObLSID &id, const LSN &start_lsn, const LSN &end_lsn, - const palf::SCN &end_scn, + const share::SCN &end_scn, share::ObBackupDest *dest, ObLogArchivePieceContext *piece_context); ~LocationDataGenerator(); @@ -157,7 +157,7 @@ private: palf::LSN &lsn, share::ObBackupPath &piece_path); private: - palf::SCN pre_scn_; + share::SCN pre_scn_; palf::LSN base_lsn_; int64_t data_len_; char data_[MAX_DATA_BUF_LEN]; @@ -185,7 +185,7 @@ public: const LSN &start_lsn, const LSN &end_lsn, const DirArray &array, - const palf::SCN &end_scn, + const share::SCN &end_scn, const int64_t piece_index, const int64_t min_file_id, const int64_t max_file_id); diff --git a/src/logservice/restoreservice/ob_remote_fetch_log.h b/src/logservice/restoreservice/ob_remote_fetch_log.h index 83945b0fc..5de92b49e 100644 --- a/src/logservice/restoreservice/ob_remote_fetch_log.h +++ b/src/logservice/restoreservice/ob_remote_fetch_log.h @@ -21,6 +21,7 @@ namespace oceanbase namespace share { class ObLSID; +class SCN; } namespace storage @@ -32,7 +33,6 @@ class ObLSService; namespace palf { struct LSN; -class SCN; } namespace logservice @@ -61,9 +61,9 @@ private: int check_need_schedule_(ObLS &ls, bool &need_schedule, int64_t &proposal_id, LSN &lsn, int64_t &last_fetch_ts); int get_fetch_log_max_lsn_(ObLS &ls, palf::LSN &max_lsn); int get_fetch_log_base_lsn_(ObLS &ls, const LSN &max_fetch_lsn, const int64_t last_fetch_ts, - palf::SCN &scn, LSN &lsn, int64_t &size); - int get_palf_base_lsn_scn_(ObLS &ls, LSN &lsn, palf::SCN &scn); - int submit_fetch_log_task_(ObLS &ls, const palf::SCN &scn, const LSN &lsn, const int64_t size, const int64_t proposal_id); + share::SCN &scn, LSN &lsn, int64_t &size); + int get_palf_base_lsn_scn_(ObLS &ls, LSN &lsn, share::SCN &scn); + int submit_fetch_log_task_(ObLS &ls, const share::SCN &scn, const LSN &lsn, const int64_t size, const int64_t proposal_id); private: bool inited_; diff --git a/src/logservice/restoreservice/ob_remote_fetch_log_worker.cpp b/src/logservice/restoreservice/ob_remote_fetch_log_worker.cpp index faf9a5d6d..2ccdac56c 100644 --- a/src/logservice/restoreservice/ob_remote_fetch_log_worker.cpp +++ b/src/logservice/restoreservice/ob_remote_fetch_log_worker.cpp @@ -27,7 +27,7 @@ #include "logservice/palf/log_group_entry.h" // LogGroupEntry #include "logservice/palf/lsn.h" // LSN #include "ob_log_restore_service.h" // ObLogRestoreService -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // SCN #include "ob_fetch_log_task.h" // ObFetchLogTask #include "ob_remote_log_source.h" // ObRemoteLogParent #include "ob_remote_log_iterator.h" // ObRemoteLogIterator diff --git a/src/logservice/restoreservice/ob_remote_fetch_log_worker.h b/src/logservice/restoreservice/ob_remote_fetch_log_worker.h index d82c72242..dcea8eb2e 100644 --- a/src/logservice/restoreservice/ob_remote_fetch_log_worker.h +++ b/src/logservice/restoreservice/ob_remote_fetch_log_worker.h @@ -25,11 +25,15 @@ namespace storage class ObLSService; } +namespace share +{ +class SCN; +} + namespace palf { struct LSN; class LogGroupEntry; -class SCN; } namespace logservice @@ -65,12 +69,12 @@ private: void run1(); void do_thread_task_(); int handle(ObFetchLogTask &task); - int get_upper_limit_scn_(const ObLSID &id, palf::SCN &scn); - int submit_entries_(const ObLSID &id, const palf::SCN &upper_limit_scn, ObRemoteLogIterator &iter, palf::SCN &max_submit_scn); - int cut_group_log_(const ObLSID &id, const LSN &lsn, const palf::SCN &cut_scn, palf::LogGroupEntry &entry); + int get_upper_limit_scn_(const ObLSID &id, share::SCN &scn); + int submit_entries_(const ObLSID &id, const share::SCN &upper_limit_scn, ObRemoteLogIterator &iter, share::SCN &max_submit_scn); + int cut_group_log_(const ObLSID &id, const LSN &lsn, const share::SCN &cut_scn, palf::LogGroupEntry &entry); int get_pre_accum_checksum_(const ObLSID &id, const LSN &lsn, int64_t &pre_accum_checksum); int submit_log_(const ObLSID &id, const LSN &lsn, char *buf, const int64_t buf_size); - void mark_if_to_end_(ObFetchLogTask &task, const palf::SCN &upper_limit_scn, const palf::SCN &scn); + void mark_if_to_end_(ObFetchLogTask &task, const share::SCN &upper_limit_scn, const share::SCN &scn); int try_retire_(ObFetchLogTask *&task); void try_update_location_info_(const ObFetchLogTask &task, ObRemoteLogIterator &iter); diff --git a/src/logservice/restoreservice/ob_remote_location_adaptor.cpp b/src/logservice/restoreservice/ob_remote_location_adaptor.cpp index c722d810a..48218b7bb 100644 --- a/src/logservice/restoreservice/ob_remote_location_adaptor.cpp +++ b/src/logservice/restoreservice/ob_remote_location_adaptor.cpp @@ -18,7 +18,7 @@ #include "share/ob_ls_id.h" // ObLSID #include "common/ob_role.h" // ObRole #include "logservice/palf_handle_guard.h" // PalfHandleGuard -#include "logservice/palf/scn.h" // palf::SCN +#include "share/scn.h" // SCN #include "share/restore/ob_ls_restore_status.h" // ObLSRestoreStatus #include "share/restore/ob_log_archive_source.h" // ObLogArchiveSourceItem #include "share/restore/ob_log_archive_source_mgr.h" // ObLogArchiveSourceMgr diff --git a/src/logservice/restoreservice/ob_remote_log_iterator.cpp b/src/logservice/restoreservice/ob_remote_log_iterator.cpp index ee7b5c635..6d84cbc81 100644 --- a/src/logservice/restoreservice/ob_remote_log_iterator.cpp +++ b/src/logservice/restoreservice/ob_remote_log_iterator.cpp @@ -75,7 +75,7 @@ ObRemoteLogIterator::~ObRemoteLogIterator() int ObRemoteLogIterator::init(const uint64_t tenant_id, const ObLSID &id, - const palf::SCN &pre_scn, + const SCN &pre_scn, const LSN &start_lsn, const LSN &end_lsn) { diff --git a/src/logservice/restoreservice/ob_remote_log_iterator.h b/src/logservice/restoreservice/ob_remote_log_iterator.h index ecda3009d..cdc627b2d 100644 --- a/src/logservice/restoreservice/ob_remote_log_iterator.h +++ b/src/logservice/restoreservice/ob_remote_log_iterator.h @@ -22,7 +22,7 @@ #include "ob_remote_log_source.h" // ObRemoteLogParent #include "logservice/palf/log_group_entry.h" // LogGroupEntry #include "logservice/palf/lsn.h" // LSN -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // SCN #include "logservice/palf/log_iterator_storage.h" // MemoryStorage #include "ob_remote_data_generator.h" // RemoteDataBuffer @@ -67,7 +67,7 @@ public: // iteraotr should be inited again to follow this change int init(const uint64_t tenant_id, const ObLSID &id, - const palf::SCN &pre_scn, + const share::SCN &pre_scn, const LSN &start_lsn, const LSN &end_lsn); // = [](share::ObBackupDest &dest) -> int { return OB_NOT_SUPPORTED; }); @@ -79,17 +79,17 @@ public: // @param[out] entry 迭代的LogGroupEntry // @param[out] lsn LogGroupEntry在日志流起始offset int next(LogGroupEntry &entry, LSN &lsn, char *&buf, int64_t &buf_size); - int get_cur_lsn_scn(LSN &lsn, palf::SCN &scn) const; + int get_cur_lsn_scn(LSN &lsn, share::SCN &scn) const; TO_STRING_KV(K_(inited), K_(tenant_id), K_(id), K_(start_lsn), K_(cur_lsn), K_(end_lsn), K_(gen)); private: - int build_data_generator_(const palf::SCN &pre_scn, + int build_data_generator_(const share::SCN &pre_scn, ObRemoteLogParent *source, const std::function &refresh_storage_info_func); int build_service_data_generator_(ObRemoteSerivceParent *source); - int build_dest_data_generator_(const palf::SCN &pre_scn, ObRemoteRawPathParent *source); - int build_location_data_generator_(const palf::SCN &pre_scn, + int build_dest_data_generator_(const share::SCN &pre_scn, ObRemoteRawPathParent *source); + int build_location_data_generator_(const share::SCN &pre_scn, ObRemoteLocationParent *source, const std::function &refresh_storage_info_func); int next_entry_(LogGroupEntry &entry, LSN &lsn, char *&buf, int64_t &buf_size); @@ -105,7 +105,7 @@ private: ObLSID id_; LSN start_lsn_; LSN cur_lsn_; // 迭代最新一条日志所对应终点LSN - palf::SCN cur_scn_; // 迭代最新一条日志scn + share::SCN cur_scn_; // 迭代最新一条日志scn LSN end_lsn_; ObRemoteSourceGuard source_guard_; RemoteDataBuffer data_buffer_; diff --git a/src/logservice/restoreservice/ob_remote_log_source.cpp b/src/logservice/restoreservice/ob_remote_log_source.cpp index bafa0dd46..4fd1c6793 100644 --- a/src/logservice/restoreservice/ob_remote_log_source.cpp +++ b/src/logservice/restoreservice/ob_remote_log_source.cpp @@ -23,6 +23,7 @@ namespace oceanbase { using namespace palf; +using namespace share; namespace logservice { // =========================== ObRemoteLogParent ==============================// diff --git a/src/logservice/restoreservice/ob_remote_log_source.h b/src/logservice/restoreservice/ob_remote_log_source.h index 7962bb81e..1ba0adc5f 100644 --- a/src/logservice/restoreservice/ob_remote_log_source.h +++ b/src/logservice/restoreservice/ob_remote_log_source.h @@ -20,7 +20,7 @@ #include "lib/utility/ob_macro_utils.h" #include "lib/utility/ob_print_utils.h" #include "logservice/palf/lsn.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/backup/ob_backup_struct.h" // ObBackupPathString #include "share/ob_define.h" #include "share/ob_ls_id.h" @@ -47,9 +47,9 @@ public: virtual int64_t to_string(char *buf, const int64_t buf_len) const = 0; virtual int update_locate_info(ObRemoteLogParent &source) = 0; bool to_end() const { return to_end_; } - void set_to_end(const bool is_to_end, const palf::SCN &scn); - void get_end_scn(palf::SCN &scn) const { scn = end_fetch_scn_;} - void get_upper_limit_scn(palf::SCN &scn) const { scn = upper_limit_scn_; } + void set_to_end(const bool is_to_end, const share::SCN &scn); + void get_end_scn(share::SCN &scn) const { scn = end_fetch_scn_;} + void get_upper_limit_scn(share::SCN &scn) const { scn = upper_limit_scn_; } ObLogArchiveSourceType get_source_type() const { return type_; } const char *get_source_type_str(const ObLogArchiveSourceType &type) const; void mark_error(share::ObTaskId &trace_id, const int ret_code); @@ -62,9 +62,9 @@ protected: protected: share::ObLSID ls_id_; ObLogArchiveSourceType type_; - palf::SCN upper_limit_scn_; + share::SCN upper_limit_scn_; bool to_end_; - palf::SCN end_fetch_scn_; + share::SCN end_fetch_scn_; palf::LSN end_lsn_; ObLogRestoreErrorContext error_context_; // 记录该source的错误信息, 仅leader有效 @@ -79,8 +79,8 @@ public: virtual ~ObRemoteSerivceParent(); public: - int set(const ObAddr &addr, const palf::SCN &end_scn); - void get(ObAddr &addr, palf::SCN &end_scn); + int set(const ObAddr &addr, const share::SCN &end_scn); + void get(ObAddr &addr, share::SCN &end_scn); int deep_copy_to(ObRemoteLogParent &other) override; bool is_valid() const override; int update_locate_info(ObRemoteLogParent &source) override { UNUSED(source); return OB_SUCCESS; } @@ -101,8 +101,8 @@ public: virtual ~ObRemoteLocationParent(); public: - void get(share::ObBackupDest *&dest, ObLogArchivePieceContext *&piece_context, palf::SCN &end_scn); - int set(const share::ObBackupDest &dest, const palf::SCN &end_scn); + void get(share::ObBackupDest *&dest, ObLogArchivePieceContext *&piece_context, share::SCN &end_scn); + int set(const share::ObBackupDest &dest, const share::SCN &end_scn); int deep_copy_to(ObRemoteLogParent &other) override; bool is_valid() const override; int update_locate_info(ObRemoteLogParent &source) override; @@ -125,11 +125,11 @@ public: virtual ~ObRemoteRawPathParent(); public: - void get(DirArray &array, palf::SCN &end_scn); + void get(DirArray &array, share::SCN &end_scn); int set(const int64_t cluster_id, const ObAddr &addr); int deep_copy_to(ObRemoteLogParent &other) override; bool is_valid() const override; - int set(DirArray &array, const palf::SCN &end_scn); + int set(DirArray &array, const share::SCN &end_scn); int update_locate_info(ObRemoteLogParent &source) override { UNUSED(source); return OB_NOT_SUPPORTED; } void get_locate_info(int64_t &piece_index, int64_t &min_file_id, int64_t &max_file_id) const; diff --git a/src/objit/CMakeLists.txt b/src/objit/CMakeLists.txt index 2ce5a8aeb..d215a7cb8 100644 --- a/src/objit/CMakeLists.txt +++ b/src/objit/CMakeLists.txt @@ -28,7 +28,7 @@ add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0 -g -O2 -frtti) if( ${ARCHITECTURE} STREQUAL "x86_64" ) LLVM_MAP_COMPONENTS_TO_LIBNAMES(llvm_libs Support Core IRReader ExecutionEngine OrcJit McJit X86CodeGen X86AsmParser runtimedyld bitreader bitwriter object objectyaml target DebugInfoDWARF Symbolize) else() - LLVM_MAP_COMPONENTS_TO_LIBNAMES(llvm_libs Support Core IRReader ExecutionEngine OrcJit McJit AArch64CodeGen AArch64AsmParser runtimedyld bitreader bitwriter object objectyaml target DebugInfoDWARF) + LLVM_MAP_COMPONENTS_TO_LIBNAMES(llvm_libs Support Core IRReader ExecutionEngine OrcJit McJit AArch64CodeGen AArch64AsmParser runtimedyld bitreader bitwriter object objectyaml target DebugInfoDWARF Symbolize) endif() set(CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}") include(${LLVM_CMAKE_DIR}/HandleLLVMOptions.cmake) diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index 0c2596a49..489236d91 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -35,7 +35,6 @@ ob_set_subtarget(ob_server common ob_srv_xlator_primary.cpp ob_srv_xlator_rootserver.cpp ob_srv_xlator_storage.cpp - ob_tablet_checksum_updater.cpp ob_tenant_duty_task.cpp ob_uniq_task_queue.cpp ) diff --git a/src/observer/mysql/ob_query_driver.cpp b/src/observer/mysql/ob_query_driver.cpp index c8b57c534..97f846b78 100644 --- a/src/observer/mysql/ob_query_driver.cpp +++ b/src/observer/mysql/ob_query_driver.cpp @@ -156,7 +156,8 @@ int ObQueryDriver::response_query_result(ObResultSet &result, ObObj& value = row->get_cell(i); if (result.is_ps_protocol() && !is_packed) { if (value.get_type() != fields->at(i).type_.get_type()) { - ObCastCtx cast_ctx(&result.get_mem_pool(), NULL, CM_WARN_ON_FAIL, CS_TYPE_INVALID); + ObCastCtx cast_ctx(&result.get_mem_pool(), NULL, CM_WARN_ON_FAIL, + fields->at(i).type_.get_collation_type()); if (OB_FAIL(common::ObObjCaster::to_type(fields->at(i).type_.get_type(), cast_ctx, value, diff --git a/src/observer/mysql/obmp_base.cpp b/src/observer/mysql/obmp_base.cpp index 0697869ce..fd4c15d01 100644 --- a/src/observer/mysql/obmp_base.cpp +++ b/src/observer/mysql/obmp_base.cpp @@ -475,39 +475,46 @@ int ObMPBase::response_row(ObSQLSessionInfo &session, ret = OB_INVALID_ARGUMENT; LOG_WARN("fields is null", K(ret), KP(fields)); } - - for (int64_t i = 0; OB_SUCC(ret) && i < row.get_count(); ++i) { - ObObj &value = row.get_cell(i); - // need at ps mode - if (value.get_type() != fields->at(i).type_.get_type()) { - ObCastCtx cast_ctx(&THIS_WORKER.get_sql_arena_allocator(), - NULL, CM_WARN_ON_FAIL, CS_TYPE_INVALID); - if (OB_FAIL(common::ObObjCaster::to_type(fields->at(i).type_.get_type(), - cast_ctx, - value, - value))) { - LOG_WARN("failed to cast object", K(ret), K(value), K(i), - K(value.get_type()), K(fields->at(i).type_.get_type())); + lib::ContextParam param; + param.set_mem_attr(session.get_effective_tenant_id(), + ObModIds::OB_SQL_EXECUTOR, ObCtxIds::DEFAULT_CTX_ID) + .set_properties(lib::USE_TL_PAGE_OPTIONAL) + .set_page_size(OB_MALLOC_NORMAL_BLOCK_SIZE) + .set_ablock_size(lib::INTACT_MIDDLE_AOBJECT_SIZE); + CREATE_WITH_TEMP_CONTEXT(param) { + for (int64_t i = 0; OB_SUCC(ret) && i < row.get_count(); ++i) { + ObObj &value = row.get_cell(i); + // need at ps mode + if (value.get_type() != fields->at(i).type_.get_type()) { + ObCastCtx cast_ctx(&THIS_WORKER.get_sql_arena_allocator(), + NULL, CM_WARN_ON_FAIL, fields->at(i).type_.get_collation_type()); + if (OB_FAIL(common::ObObjCaster::to_type(fields->at(i).type_.get_type(), + cast_ctx, + value, + value))) { + LOG_WARN("failed to cast object", K(ret), K(value), K(i), + K(value.get_type()), K(fields->at(i).type_.get_type())); + } + } + ObCharsetType charset_type = CHARSET_INVALID; + if (OB_FAIL(ret)) { + } else if (OB_FAIL(session.get_character_set_results(charset_type))) { + LOG_WARN("fail to get result charset", K(ret)); + } else if (ob_is_string_type(value.get_type()) + && CS_TYPE_INVALID != value.get_collation_type() + && OB_FAIL(value.convert_string_value_charset(charset_type, + THIS_WORKER.get_sql_arena_allocator()))) { + LOG_WARN("convert string value charset failed", K(ret), K(value)); + } else if (value.is_clob_locator() + && OB_FAIL(ObQueryDriver::convert_lob_value_charset(value, charset_type, + THIS_WORKER.get_sql_arena_allocator()))) { + LOG_WARN("convert lob value charset failed", K(ret)); + } + if (OB_SUCC(ret) && OB_FAIL(ObQueryDriver::convert_lob_locator_to_longtext(value, + session.is_client_use_lob_locator(), + &THIS_WORKER.get_sql_arena_allocator()))) { + LOG_WARN("convert lob locator to longtext failed", K(ret)); } - } - ObCharsetType charset_type = CHARSET_INVALID; - if (OB_FAIL(ret)) { - } else if (OB_FAIL(session.get_character_set_results(charset_type))) { - LOG_WARN("fail to get result charset", K(ret)); - } else if (ob_is_string_type(value.get_type()) - && CS_TYPE_INVALID != value.get_collation_type() - && OB_FAIL(value.convert_string_value_charset(charset_type, - THIS_WORKER.get_sql_arena_allocator()))) { - LOG_WARN("convert string value charset failed", K(ret), K(value)); - } else if (value.is_clob_locator() - && OB_FAIL(ObQueryDriver::convert_lob_value_charset(value, charset_type, - THIS_WORKER.get_sql_arena_allocator()))) { - LOG_WARN("convert lob value charset failed", K(ret)); - } - if (OB_SUCC(ret) && OB_FAIL(ObQueryDriver::convert_lob_locator_to_longtext(value, - session.is_client_use_lob_locator(), - &THIS_WORKER.get_sql_arena_allocator()))) { - LOG_WARN("convert lob locator to longtext failed", K(ret)); } } diff --git a/src/observer/mysql/obmp_stmt_execute.cpp b/src/observer/mysql/obmp_stmt_execute.cpp index 4fc516fde..915b7ee26 100644 --- a/src/observer/mysql/obmp_stmt_execute.cpp +++ b/src/observer/mysql/obmp_stmt_execute.cpp @@ -2171,7 +2171,11 @@ int ObMPStmtExecute::parse_basic_param_value(ObIAllocator &allocator, if (is_oracle_mode() && !is_complex_element) { param.set_char(dst); } else { - param.set_varchar(dst); + if (is_complex_element && dst.length()== 0) { + param.set_null(); + } else { + param.set_varchar(dst); + } } } } diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index c4f934eaf..7c5da4fa3 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -48,7 +48,7 @@ #include "share/sequence/ob_sequence_cache.h" #include "logservice/ob_log_service.h" #include "logservice/ob_log_handler.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/high_availability/ob_storage_ha_service.h" #include "storage/tx_table/ob_tx_table.h" #include "storage/slog_ckpt/ob_server_checkpoint_slog_handler.h" @@ -1277,7 +1277,7 @@ int ObRpcGetLSAccessModeP::process() palf::AccessMode mode; int64_t mode_version = palf::INVALID_PROPOSAL_ID; int64_t second_proposal_id = 0; - const palf::SCN ref_scn = palf::SCN::min_scn(); + const SCN ref_scn = SCN::min_scn(); if (OB_FAIL(log_handler->get_access_mode(mode_version, mode))) { LOG_WARN("failed to get access mode", KR(ret), K(ls_id)); } else if (OB_FAIL(result_.init(tenant_id, ls_id, mode_version, mode, ref_scn))) { @@ -1925,7 +1925,7 @@ int ObRpcRemoteWriteDDLPrepareLogP::process() LOG_WARN("init sstable redo writer", K(ret), K(table_key)); } else if (FALSE_IT(sstable_redo_writer.set_start_scn(arg_.start_scn_))) { } else { - palf::SCN prepare_scn; + SCN prepare_scn; if (OB_FAIL(sstable_redo_writer.write_prepare_log(table_key, arg_.table_id_, arg_.execution_id_, diff --git a/src/observer/ob_service.cpp b/src/observer/ob_service.cpp index 066e6bf3e..751231122 100644 --- a/src/observer/ob_service.cpp +++ b/src/observer/ob_service.cpp @@ -62,7 +62,7 @@ #include "storage/ls/ob_ls.h" #include "logservice/ob_log_service.h" // ObLogService #include "logservice/palf_handle_guard.h" // PalfHandleGuard -#include "logservice/palf/scn.h" // PalfHandleGuard +#include "share/scn.h" // PalfHandleGuard #include "storage/backup/ob_backup_handler.h" #include "storage/backup/ob_ls_backup_clean_mgr.h" #include "storage/ob_file_system_router.h" @@ -166,7 +166,7 @@ ObService::ObService(const ObGlobalContext &gctx) lease_state_mgr_(), heartbeat_process_(gctx, schema_updater_, lease_state_mgr_), gctx_(gctx), server_trace_task_(), schema_release_task_(), schema_status_task_(), remote_master_rs_update_task_(gctx), ls_table_updater_(), - tablet_table_updater_(), tablet_checksum_updater_(), meta_table_checker_() + tablet_table_updater_(), meta_table_checker_() { } @@ -207,8 +207,6 @@ int ObService::init(common::ObMySQLProxy &sql_proxy, FLOG_WARN("init tablet table updater failed", KR(ret)); } else if (OB_FAIL(ls_table_updater_.init())) { FLOG_WARN("init log stream table updater failed", KR(ret)); - } else if (OB_FAIL(tablet_checksum_updater_.init(*this))) { - FLOG_WARN("init tablet checksum updater failed", KR(ret)); } else if (OB_FAIL(meta_table_checker_.init( gctx_.lst_operator_, gctx_.tablet_operator_, @@ -302,10 +300,6 @@ void ObService::stop() tablet_table_updater_.stop(); FLOG_INFO("tablet table updater stopped"); - FLOG_INFO("begin to stop tablet checksum updater"); - tablet_checksum_updater_.stop(); - FLOG_INFO("tablet checksum updater stopped"); - FLOG_INFO("begin to stop meta table checker"); meta_table_checker_.stop(); FLOG_INFO("meta table checker stopped"); @@ -331,10 +325,6 @@ void ObService::wait() tablet_table_updater_.wait(); FLOG_INFO("wait tablet table updater success"); - FLOG_INFO("begin to wait tablet checksum updater"); - tablet_checksum_updater_.wait(); - FLOG_INFO("wait tablet checksum updater success"); - FLOG_INFO("begin to wait meta table checker"); meta_table_checker_.wait(); FLOG_INFO("wait meta table checker success"); @@ -432,23 +422,6 @@ int ObService::submit_tablet_update_task( return ret; } -int ObService::submit_tablet_checksums_task( - const uint64_t tenant_id, - const ObLSID &ls_id, - const ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (!ls_id.is_valid_with_tenant(tenant_id) || !tablet_id.is_valid_with_tenant(tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(tablet_checksum_updater_.async_update(tenant_id, ls_id, tablet_id))) { - LOG_WARN("fail to async update tablet checksum", KR(ret), K(tenant_id), K(ls_id), K(tablet_id)); - } - return ret; -} int ObService::submit_async_refresh_schema_task( const uint64_t tenant_id, @@ -474,7 +447,7 @@ int ObService::check_frozen_scn(const obrpc::ObCheckFrozenScnArg &arg) { LOG_INFO("receive check frozen SCN request", K(arg)); int ret = OB_SUCCESS; - palf::SCN last_merged_scn = palf::SCN::min_scn(); + SCN last_merged_scn = SCN::min_scn(); if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); @@ -664,8 +637,8 @@ int ObService::backup_completing_log(const obrpc::ObBackupComplLogArg &arg) ObBackupSetDesc backup_set_desc; backup_set_desc.backup_set_id_ = arg.backup_set_id_; backup_set_desc.backup_type_.type_ = arg.backup_type_; - palf::SCN start_scn = arg.start_scn_; - palf::SCN end_scn = arg.end_scn_; + SCN start_scn = arg.start_scn_; + SCN end_scn = arg.end_scn_; ObLSID ls_id = arg.ls_id_; ObMySQLProxy *sql_proxy = GCTX.sql_proxy_; if (!arg.is_valid() || OB_ISNULL(sql_proxy)) { @@ -743,7 +716,7 @@ int ObService::backup_meta(const obrpc::ObBackupMetaArg &arg) const ObLSID &ls_id = arg.ls_id_; const int64_t turn_id = arg.turn_id_; const int64_t retry_id = arg.retry_id_; - const palf::SCN start_scn = arg.start_scn_; + const SCN start_scn = arg.start_scn_; ObMySQLProxy *sql_proxy = GCTX.sql_proxy_; if (!arg.is_valid() || OB_ISNULL(sql_proxy)) { ret = OB_INVALID_ARGUMENT; @@ -1034,7 +1007,7 @@ int ObService::check_modify_time_elapsed( } else { MTL_SWITCH(arg.tenant_id_) { ObLSHandle ls_handle; - palf::SCN tmp_scn; + SCN tmp_scn; transaction::ObTransService *txs = MTL(transaction::ObTransService *); ObLSService *ls_service = MTL(ObLSService *); if (OB_FAIL(ls_service->get_ls(ObLSID(arg.ls_id_), ls_handle, ObLSGetMod::OBSERVER_MOD))) { @@ -1946,11 +1919,13 @@ int ObService::write_ddl_sstable_commit_log(const ObDDLWriteSSTableCommitLogArg return ret; } -int ObService::inner_fill_tablet_replica_( +int ObService::inner_fill_tablet_info_( const int64_t tenant_id, const ObTabletID &tablet_id, storage::ObLS *ls, - ObTabletReplica &tablet_replica) + ObTabletReplica &tablet_replica, + share::ObTabletReplicaChecksumItem &tablet_checksum, + const bool need_checksum) { ObLSHandle ls_handle; ObTabletHandle tablet_handle; @@ -1982,9 +1957,9 @@ int ObService::inner_fill_tablet_replica_( const ObLSID &ls_id = ls->get_ls_id(); int64_t data_size = 0; int64_t required_size = 0; - ObArray column_checksums; // param palceholder + ObArray column_checksums; if (OB_FAIL(tablet_handle.get_obj()->get_tablet_report_info(column_checksums, data_size, - required_size, false/*need_checksum*/))) { + required_size, need_checksum))) { LOG_WARN("fail to get tablet report info from tablet", KR(ret), K(tenant_id), K(tablet_id)); } else if (OB_FAIL(tablet_replica.init( tenant_id, @@ -1996,70 +1971,30 @@ int ObService::inner_fill_tablet_replica_( required_size))) { LOG_WARN("fail to init a tablet replica", KR(ret), K(tenant_id), K(tablet_id), K(tablet_replica)); - } - } - return ret; -} - -int ObService::inner_fill_tablet_replica_checksum_item_( - const int64_t tenant_id, - const ObTabletID &tablet_id, - storage::ObLS *ls, - share::ObTabletReplicaChecksumItem &item) -{ - int ret = OB_SUCCESS; - ObTabletHandle tablet_handle; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("service not inited", KR(ret)); - } else if (OB_UNLIKELY(!tablet_id.is_valid()) - || OB_INVALID_TENANT_ID == tenant_id - || OB_ISNULL(ls)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument or nullptr", KR(ret), K(tablet_id), K(tenant_id)); - } else if (OB_ISNULL(ls->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get_tablet_svr is null", KR(ret), K(tenant_id), K(tablet_id)); - } else if (OB_FAIL(ls->get_tablet_svr()->get_tablet( - tablet_id, - tablet_handle, - ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) { - if (OB_TABLET_NOT_EXIST != ret) { - LOG_WARN("get tablet failed", KR(ret), K(tenant_id), K(tablet_id)); - } - } else if (OB_ISNULL(gctx_.config_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("gctx_.config_ is null", KR(ret), K(tenant_id), K(tablet_id)); - } else { - item.tenant_id_ = tenant_id; - item.ls_id_ = ls->get_ls_id(); - item.tablet_id_ = tablet_id; - item.server_ = gctx_.self_addr(); - item.row_count_ = tablet_handle.get_obj()->get_tablet_meta().report_status_.row_count_; - item.data_checksum_ = tablet_handle.get_obj()->get_tablet_meta().report_status_.data_checksum_; - const uint64_t compaction_scn_val = (uint64_t)( - tablet_handle.get_obj()->get_tablet_meta().report_status_.merge_snapshot_version_); - if (OB_FAIL(item.compaction_scn_.convert_for_inner_table_field(compaction_scn_val))) { - LOG_WARN("fail to convert val to SCN", KR(ret), K(compaction_scn_val)); - } - - int64_t data_size = 0; - int64_t required_size = 0; - ObArray column_checksums; - if (FAILEDx(tablet_handle.get_obj()->get_tablet_report_info(column_checksums, data_size, required_size))) { - LOG_WARN("fail to get column checksums from tablet", KR(ret), K(tenant_id), K(tablet_id), K(item)); - } else if (OB_FAIL(item.column_meta_.init(column_checksums))) { + } else if (!need_checksum) { + } else if (OB_FAIL(tablet_checksum.column_meta_.init(column_checksums))) { LOG_WARN("fail to init report column meta with column_checksums", KR(ret), K(column_checksums)); + } else if (OB_FAIL(tablet_checksum.compaction_scn_.convert_for_tx(snapshot_version))) { + LOG_WARN("failed to convert scn", KR(ret), K(snapshot_version)); + } else { + tablet_checksum.tenant_id_ = tenant_id; + tablet_checksum.ls_id_ = ls->get_ls_id(); + tablet_checksum.tablet_id_ = tablet_id; + tablet_checksum.server_ = gctx_.self_addr(); + tablet_checksum.row_count_ = tablet_handle.get_obj()->get_tablet_meta().report_status_.row_count_; + tablet_checksum.data_checksum_ = tablet_handle.get_obj()->get_tablet_meta().report_status_.data_checksum_; } } return ret; } -int ObService::fill_tablet_replica( +int ObService::fill_tablet_report_info( const uint64_t tenant_id, const ObLSID &ls_id, const ObTabletID &tablet_id, - ObTabletReplica &tablet_replica) + ObTabletReplica &tablet_replica, + share::ObTabletReplicaChecksumItem &tablet_checksum, + const bool need_checksum) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!inited_)) { @@ -2085,62 +2020,18 @@ int ObService::fill_tablet_replica( LOG_TRACE("log stream not exist in this tenant", KR(ret), K(tenant_id), K(ls_id)); } } else if (FALSE_IT(ls = ls_handle.get_ls())) { - } else if (OB_FAIL(inner_fill_tablet_replica_(tenant_id, - tablet_id, - ls, - tablet_replica))) { + } else if (OB_FAIL(inner_fill_tablet_info_(tenant_id, + tablet_id, + ls, + tablet_replica, + tablet_checksum, + need_checksum))) { if (OB_TABLET_NOT_EXIST != ret) { LOG_WARN("fail to inner fill tenant's tablet replica", KR(ret), - K(tenant_id), K(tablet_id), K(ls), K(tablet_replica)); + K(tenant_id), K(tablet_id), K(ls), K(tablet_replica), K(tablet_checksum), K(need_checksum)); } else { LOG_TRACE("tablet not exist in this log stream", KR(ret), - K(tenant_id), K(tablet_id), K(ls), K(tablet_replica)); - } - } - } - } - return ret; -} - -int ObService::fill_tablet_replica_checksum_item( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const ObTabletID &tablet_id, - share::ObTabletReplicaChecksumItem &item) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("service not inited", KR(ret)); - } else if (!tablet_id.is_valid() || !ls_id.is_valid() || OB_INVALID_TENANT_ID == tenant_id) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tablet_id), K(ls_id), K(tenant_id)); - } else { - MTL_SWITCH(tenant_id) { - ObTabletHandle tablet_handle; - ObLSHandle ls_handle; - storage::ObLS *ls = nullptr; - ObLSService* ls_svr = nullptr; - if (OB_ISNULL(ls_svr = MTL(ObLSService*))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("MTL ObLSService is null", KR(ret), K(tenant_id)); - } else if (OB_FAIL(ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::OBSERVER_MOD))) { - if (OB_LS_NOT_EXIST != ret) { - LOG_WARN("fail to get log_stream's ls_handle", KR(ret), K(tenant_id), K(ls_id)); - } else { - LOG_TRACE("log stream not exist in this tenant", KR(ret), K(tenant_id), K(ls_id)); - } - } else if (FALSE_IT(ls = ls_handle.get_ls())) { - } else if (OB_FAIL(inner_fill_tablet_replica_checksum_item_(tenant_id, - tablet_id, - ls, - item))) { - if (OB_TABLET_NOT_EXIST != ret) { - LOG_WARN("fail to inner fill tenant's tablet checksum item", KR(ret), - K(tenant_id), K(tablet_id), K(ls), K(item)); - } else { - LOG_TRACE("tablet not exist in this log stream", KR(ret), - K(tenant_id), K(tablet_id), K(ls), K(item)); + K(tenant_id), K(tablet_id), K(ls), K(tablet_replica), K(tablet_checksum), K(need_checksum)); } } } diff --git a/src/observer/ob_service.h b/src/observer/ob_service.h index 8ad638996..06fb33d46 100644 --- a/src/observer/ob_service.h +++ b/src/observer/ob_service.h @@ -18,7 +18,6 @@ #include "share/ob_all_server_tracer.h" #include "observer/ob_lease_state_mgr.h" #include "observer/ob_heartbeat.h" -#include "observer/ob_tablet_checksum_updater.h" #include "observer/ob_server_schema_updater.h" #include "observer/ob_rpc_processor_simple.h" #include "observer/ob_uniq_task_queue.h" @@ -92,18 +91,17 @@ public: // @params[in] ls_id: tablet belongs to which log stream // @params[in] tablet_id: the tablet to build // @params[out] tablet_replica: infos about this tablet replica + // @params[out] tablet_checksum: infos about this tablet data/column checksum + // @params[in] need_checksum: whether to fill tablet_checksum // ATTENTION: If ls not exist, then OB_LS_NOT_EXIST // If tablet not exist on that ls, then OB_TABLET_NOT_EXIST - int fill_tablet_replica( + int fill_tablet_report_info( const uint64_t tenant_id, const share::ObLSID &ls_id, const ObTabletID &tablet_id, - share::ObTabletReplica &tablet_replica); - int fill_tablet_replica_checksum_item( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const ObTabletID &tablet_id, - share::ObTabletReplicaChecksumItem &item); + share::ObTabletReplica &tablet_replica, + share::ObTabletReplicaChecksumItem &tablet_checksum, + const bool need_checksum = true); int detect_master_rs_ls(const obrpc::ObDetectMasterRsArg &arg, obrpc::ObDetectMasterRsLSResult &result); @@ -120,10 +118,6 @@ public: const uint64_t tenant_id, const share::ObLSID &ls_id, const ObTabletID &tablet_id) override; - virtual int submit_tablet_checksums_task( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const ObTabletID &tablet_id) override; //////////////////////////////////////////////////////////////// int check_frozen_scn(const obrpc::ObCheckFrozenScnArg &arg); @@ -227,16 +221,13 @@ public: int renew_in_zone_hb(const share::ObInZoneHbRequest &arg, share::ObInZoneHbResponse &result); private: - int inner_fill_tablet_replica_( + int inner_fill_tablet_info_( const int64_t tenant_id, const ObTabletID &tablet_id, storage::ObLS *ls, - share::ObTabletReplica &tablet_replica); - int inner_fill_tablet_replica_checksum_item_( - const int64_t tenant_id, - const ObTabletID &tablet_id, - storage::ObLS *ls, - share::ObTabletReplicaChecksumItem &item); + share::ObTabletReplica &tablet_replica, + share::ObTabletReplicaChecksumItem &tablet_checksum, + const bool need_checksum); int register_self(); int check_server_empty(const obrpc::ObCheckServerEmptyArg &arg, const bool wait_log_scan, bool &server_empty); @@ -262,7 +253,6 @@ private: // report ObLSTableUpdater ls_table_updater_; ObTabletTableUpdater tablet_table_updater_; - ObTabletChecksumUpdater tablet_checksum_updater_; ObServerMetaTableChecker meta_table_checker_; }; diff --git a/src/observer/ob_srv_xlator_rootserver.cpp b/src/observer/ob_srv_xlator_rootserver.cpp index db927b8d6..1797898f9 100644 --- a/src/observer/ob_srv_xlator_rootserver.cpp +++ b/src/observer/ob_srv_xlator_rootserver.cpp @@ -209,6 +209,7 @@ void oceanbase::observer::init_srv_xlator_for_rootserver(ObSrvRpcXlator *xlator) RPC_PROCESSOR(rootserver::ObRpcRunJobP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcAdminRefreshIOCalibrationP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcRunUpgradeJobP, *gctx_.root_service_); + RPC_PROCESSOR(rootserver::ObRpcUpgradeTableSchemaP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcAdminFlushCacheP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcAdminUpgradeCmdP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcAdminRollingUpgradeCmdP, *gctx_.root_service_); diff --git a/src/observer/ob_tablet_checksum_updater.cpp b/src/observer/ob_tablet_checksum_updater.cpp deleted file mode 100644 index b98abe1c1..000000000 --- a/src/observer/ob_tablet_checksum_updater.cpp +++ /dev/null @@ -1,508 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#define USING_LOG_PREFIX SERVER - -#include "ob_tablet_checksum_updater.h" -#include "observer/ob_service.h" -#include "share/ob_tablet_replica_checksum_operator.h" - -namespace oceanbase -{ -using namespace common; -using namespace share; - -namespace observer -{ - -ObTabletChecksumUpdateTask::ObTabletChecksumUpdateTask() - : tenant_id_(OB_INVALID_TENANT_ID), - ls_id_(), - tablet_id_(), - add_timestamp_(OB_INVALID_TIMESTAMP) -{ -} - -ObTabletChecksumUpdateTask::ObTabletChecksumUpdateTask( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - const int64_t add_timestamp) - : tenant_id_(tenant_id), - ls_id_(ls_id), - tablet_id_(tablet_id), - add_timestamp_(add_timestamp) -{ -} - -ObTabletChecksumUpdateTask::~ObTabletChecksumUpdateTask() -{ - reset(); -} - -void ObTabletChecksumUpdateTask::reset() -{ - tenant_id_ = OB_INVALID_TENANT_ID; - ls_id_.reset(); - tablet_id_.reset(); - add_timestamp_ = OB_INVALID_TIMESTAMP; -} - -bool ObTabletChecksumUpdateTask::is_valid() const -{ - return OB_INVALID_TENANT_ID != tenant_id_ - && ls_id_.is_valid_with_tenant(tenant_id_) - && tablet_id_.is_valid_with_tenant(tenant_id_) - && 0 < add_timestamp_; -} - -int ObTabletChecksumUpdateTask::init( - const uint64_t tenant_id, - const ObLSID &ls_id, - const ObTabletID &tablet_id, - const int64_t add_timestamp) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ls_id.is_valid_with_tenant(tenant_id) - || !tablet_id.is_valid_with_tenant(tenant_id) - || 0 >= add_timestamp)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("get invalid arguments", KR(ret), K(tenant_id), K(ls_id), K(tablet_id), K(add_timestamp)); - } else { - tenant_id_ = tenant_id; - ls_id_ = ls_id; - tablet_id_ = tablet_id; - add_timestamp_ = add_timestamp; - } - return ret; -} - -int64_t ObTabletChecksumUpdateTask::hash() const -{ - uint64_t hash_val = 0; - hash_val = murmurhash(&tenant_id_, sizeof(tenant_id_), hash_val); - hash_val = murmurhash(&ls_id_, sizeof(ls_id_), hash_val); - hash_val = murmurhash(&tablet_id_, sizeof(tablet_id_), hash_val); - return hash_val; -} - -void ObTabletChecksumUpdateTask::check_task_status() const -{ - // TODO @danling.fjk ignore this error log temp - // int64_t now = ObTimeUtility::current_time(); - // const int64_t safe_interval = TABLET_CHECK_INTERVAL; - // // print an error log if this task is not executed correctly since two mins ago - // if (now - add_timestamp_ > safe_interval) { - // LOG_ERROR("tablet table update task cost too much time to execute", - // K(*this), K(safe_interval), "cost_time", now - add_timestamp_); - // } -} - -int ObTabletChecksumUpdateTask::assign(const ObTabletChecksumUpdateTask &other) -{ - int ret = OB_SUCCESS; - if (this != &other) { - tenant_id_ = other.tenant_id_; - ls_id_ = other.ls_id_; - tablet_id_ = other.tablet_id_; - add_timestamp_ = other.add_timestamp_; - } - return ret; -} - -bool ObTabletChecksumUpdateTask::operator==(const ObTabletChecksumUpdateTask &other) const -{ - bool bret = false; - if (this == &other) { - bret = true; - } else { - bret = (tenant_id_ == other.tenant_id_ - && ls_id_ == other.ls_id_ - && tablet_id_ == other.tablet_id_); - } - return bret; -} - -bool ObTabletChecksumUpdateTask::operator!=(const ObTabletChecksumUpdateTask &other) const -{ - return !(operator==(other)); -} - -bool ObTabletChecksumUpdateTask::compare_without_version( - const ObTabletChecksumUpdateTask &other) const -{ - return (operator==(other)); -} - - -ObTabletChecksumUpdater::ObTabletChecksumUpdater() - : inited_(false), - stopped_(true), - ob_service_(nullptr), - task_queue_() -{ -} - -ObTabletChecksumUpdater::~ObTabletChecksumUpdater() -{ - destroy(); -} - -int ObTabletChecksumUpdater::init(ObService &ob_service) -{ - int ret = OB_SUCCESS; - const int64_t task_queue_size = !lib::is_mini_mode() - ? TASK_QUEUE_SIZE - : MINI_MODE_TASK_QUEUE_SIZE; - const int64_t task_thread_cnt = !lib::is_mini_mode() - ? UPDATE_TASK_THREAD_CNT - : MINI_MODE_UPDATE_TASK_THREAD_CNT; - if (OB_UNLIKELY(inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("inited twice", KR(ret)); - } else if (OB_FAIL(task_queue_.init(this, - task_thread_cnt, task_queue_size, "TbltCksUp"))) { - LOG_WARN("failed to init tablet checksum updater task queue", KR(ret), - "thread_count", task_thread_cnt, - "queue_size", task_queue_size); - } else { - ob_service_ = &ob_service; - inited_ = true; - stopped_ = false; - LOG_INFO("success to init ObTabletChecksumUpdater"); - } - return ret; -} - -void ObTabletChecksumUpdater::stop() -{ - if (inited_) { - stopped_ = true; - task_queue_.stop(); - LOG_INFO("stop ObTabletChecksumUpdater success"); - } -} - -void ObTabletChecksumUpdater::wait() -{ - if (inited_) { - task_queue_.wait(); - LOG_INFO("wait ObTabletChecksumUpdater"); - } -} - -void ObTabletChecksumUpdater::destroy() -{ - stop(); - wait(); - inited_ = false; - stopped_ = true; - ob_service_ = nullptr; -} - -int ObTabletChecksumUpdater::async_update( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - int64_t add_timestamp = ObTimeUtility::current_time(); - ObTabletChecksumUpdateTask task; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("ObTabletChecksumUpdater not inited", KR(ret)); - } else if (tablet_id.is_reserved_tablet() || is_virtual_tenant_id(tenant_id)) { - LOG_TRACE("no need to report virtual tenant's tablet and reserved tablet", - KR(ret), K(tablet_id), K(tenant_id)); - } else if (OB_INVALID_TENANT_ID == tenant_id - || !ls_id.is_valid() - || !ls_id.is_valid_with_tenant(tenant_id) - || !tablet_id.is_valid() - || !tablet_id.is_valid_with_tenant(tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(task.init(tenant_id, - ls_id, - tablet_id, - add_timestamp))) { - LOG_WARN("set update task failed", KR(ret), K(tenant_id), K(ls_id), K(tablet_id), - K(add_timestamp)); - } else if (OB_FAIL(add_task_(task))){ - LOG_WARN("fail to add task", KR(ret), K(tenant_id), K(ls_id), K(tablet_id), - K(add_timestamp)); - } - return ret; -} - -int ObTabletChecksumUpdater::add_task_(const ObTabletChecksumUpdateTask &task) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", KR(ret)); - } else if (!task.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid task", KR(ret), K(task)); - } else if (OB_FAIL(task_queue_.add(task))){ - // TODO: deal with barrier-tasks when execute - if (OB_EAGAIN == ret) { - LOG_TRACE("tablet table update task exist", K(task)); - ret = OB_SUCCESS; - } else { - LOG_WARN("add tablet table update task failed", KR(ret), K(task)); - } - } - return ret; -} - -int ObTabletChecksumUpdater::reput_to_queue_(const ObIArray &tasks) -{ - int ret = OB_SUCCESS; - // try to push task back to queue, ignore ret code - for (int64_t i = 0; i < tasks.count(); i++) { - const ObTabletChecksumUpdateTask &task = tasks.at(i); - if (!task.is_valid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get invalid task", KR(ret), K(task)); - } else if (OB_FAIL(add_task_(task))) { - LOG_ERROR("fail to reput to queue", KR(ret), K(task)); - } - } - return ret; -} - -int ObTabletChecksumUpdater::process_barrier( - const ObTabletChecksumUpdateTask &task, - bool &stopped) -{ - int ret = OB_NOT_SUPPORTED; - UNUSEDx(task, stopped); - return ret; -} - -int ObTabletChecksumUpdater::batch_process_tasks( - const common::ObIArray &tasks, - bool &stopped) -{ - int ret = OB_SUCCESS; - UNUSED(stopped); - int tmp_ret = OB_SUCCESS; - const int64_t start_time = ObTimeUtility::current_time(); - ObArray update_tablet_items; - UpdateTabletChecksumTaskList update_tasks; - RemoveTabletChecksumTaskList remove_tasks; - ObCurTraceId::init(GCONF.self_addr_); - - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("ObTabletChecksumUpdater is not inited", KR(ret)); - } else if (OB_FAIL(generate_tasks_(tasks, update_tablet_items, update_tasks, remove_tasks))){ - if (OB_TENANT_NOT_IN_SERVER != ret && OB_NOT_RUNNING != ret) { - LOG_ERROR("generate_tasks failed", KR(ret), "tasks count", tasks.count(), - "update_tablet_items", update_tablet_items.count(), - "update_tasks", update_tasks.count(), - "remove_tasks", remove_tasks.count()); - } else { - LOG_WARN("Tenant/LogStream has been stopped, skip to process tasks", KR(ret)); - } - } else { - tmp_ret = do_batch_update_(start_time, update_tasks, update_tablet_items); - if (OB_SUCCESS != tmp_ret) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("do_batch_update_ failed", KR(tmp_ret), K(start_time), K(update_tasks), - K(update_tablet_items)); - } else { - ObTaskController::get().allow_next_syslog(); - LOG_INFO("REPORT: success to update tablets", KR(tmp_ret), K(update_tablet_items)); - } - tmp_ret = do_batch_remove_(start_time, remove_tasks); - if (OB_SUCCESS != tmp_ret) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("do_batch_remove_ failed", KR(tmp_ret), K(start_time), K(remove_tasks)); - } else { - ObTaskController::get().allow_next_syslog(); - LOG_INFO("REPORT: success to remove tablets", KR(tmp_ret), K(remove_tasks)); - } - } - return ret; -} - -int ObTabletChecksumUpdater::generate_tasks_( - const ObIArray &tasks, - ObIArray &update_tablet_items, - UpdateTabletChecksumTaskList &update_tasks, - RemoveTabletChecksumTaskList &remove_tasks) -{ - int ret = OB_SUCCESS; - int64_t tenant_id = OB_INVALID_TENANT_ID; - int64_t count = UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM; - if (OB_UNLIKELY(!inited_)) { - ret = OB_NOT_INIT; - LOG_WARN("ObTabletChecksumUpdater is not inited", KR(ret)); - } else if (OB_ISNULL(ob_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid argument", KR(ret), KP_(ob_service)); - } else if (OB_UNLIKELY(tasks.count() <= 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tasks count <= 0", KR(ret), "tasks_count", tasks.count()); - } else if (FALSE_IT(tenant_id = tasks.at(0).get_tenant_id())) { - // shall never be here - } else { - ObTabletReplicaChecksumItem item; - FOREACH_CNT_X(task, tasks, OB_SUCC(ret)) { - // split tasks into remove and update - if (OB_ISNULL(task)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid task", KR(ret), K(task)); - } else if (tenant_id != task->get_tenant_id()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tenant_id not the same", KR(ret), K(tenant_id), K(task)); - } else { - task->check_task_status(); - item.reset(); - if (OB_FAIL(ob_service_->fill_tablet_replica_checksum_item(task->get_tenant_id(), - task->get_ls_id(), task->get_tablet_id(), item))) { - if (OB_TABLET_NOT_EXIST == ret || OB_LS_NOT_EXIST == ret) { - LOG_INFO("try update a not exist or invalid tablet, turn to remove tablet table", - KR(ret), "tenant_id", task->get_tenant_id(), - "ls_id", task->get_ls_id(), - "tablet_id", task->get_tablet_id()); - ret = OB_SUCCESS; - if (OB_FAIL(remove_tasks.reserve(count))) { - // reserve() is reentrant, do not have to check whether first time - LOG_WARN("fail to reserver remove_tasks", KR(ret), K(count)); - } else if (OB_FAIL(remove_tasks.push_back(*task))) { - LOG_WARN("fail to push back remove task", KR(ret), KPC(task)); - } - } else { - LOG_WARN("fail to fill tablet item", KR(ret), "tenant_id", task->get_tenant_id(), - "ls_id", task->get_ls_id(), "tablet_id", task->get_tablet_id()); - } - } else { - LOG_TRACE("fill tablet checksum item success", K(task), K(item)); - if (OB_FAIL(update_tablet_items.reserve(count))) { - // reserve() is reentrant, do not have to check whether first time - LOG_WARN("fail to reserve update_tablet_items", KR(ret), K(count)); - } else if (OB_FAIL(update_tasks.reserve(count))) { - // reserve() is reentrant, do not have to check whether first time - LOG_WARN("fail to reserve update_tasks", KR(ret), K(count)); - } else if (OB_FAIL(update_tablet_items.push_back(item))) { - LOG_WARN("fail to push back item", KR(ret), K(item)); - } else if (OB_FAIL(update_tasks.push_back(*task))) { - LOG_WARN("fail to push back task", KR(ret), KPC(task)); - } - } - } - } //FOREACH - - if (OB_SUCC(ret) - && (update_tasks.count() != update_tablet_items.count() - || update_tasks.count() + remove_tasks.count() != tasks.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("tablet task count and item count not match", KR(ret), - "tablet_update_tasks count", update_tasks.count(), - "tablet_update_items count", update_tablet_items.count(), - "tablet_remove_tasks count", remove_tasks.count(), - "tasks count", tasks.count()); - } - } - return ret; -} - -int ObTabletChecksumUpdater::do_batch_update_( - const int64_t start_time, - const ObIArray &tasks, - const ObIArray &items) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - int64_t tenant_id = OB_INVALID_TENANT_ID; - if (tasks.count() != items.count() || 0 == tasks.count()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("tasks num not match", KR(ret), "task_cnt", tasks.count(), "item_cnt", items.count()); - } else if (FALSE_IT(tenant_id = tasks.at(0).get_tenant_id())) { - } else if (OB_FAIL(ObTabletReplicaChecksumOperator::batch_update(tenant_id, items, *GCTX.sql_proxy_))) { - LOG_WARN("do tablet table update failed, try to reput to queue", KR(ret), - "escape time", ObTimeUtility::current_time() - start_time); - (void) throttle_(ret, ObTimeUtility::current_time() - start_time); - if (OB_SUCCESS != (tmp_ret = reput_to_queue_(tasks))) { - LOG_ERROR("fail to reput update task to queue", KR(tmp_ret), K(tasks.count())); - } else { - LOG_INFO("reput update task to queue success", K(tasks.count())); - } - } else { - LOG_INFO("batch process update success", KR(ret), K(items.count()), - "use_time", ObTimeUtility::current_time() - start_time); - } - return ret; -} - -int ObTabletChecksumUpdater::do_batch_remove_( - const int64_t start_time, - const ObIArray &tasks) -{ - int ret = OB_SUCCESS; - int tmp_ret = OB_SUCCESS; - int64_t tenant_id = OB_INVALID_TENANT_ID; - if (0 == tasks.count()) { - LOG_INFO("no need to remove task", KR(ret), "task_cnt", tasks.count()); - } else if (FALSE_IT(tenant_id = tasks.at(0).get_tenant_id())) { - } else { - ObArray ls_ids; - ObArray tablet_ids; - for (int64_t i = 0; OB_SUCC(ret) && i < tasks.count(); ++i) { - if (OB_FAIL(ls_ids.push_back(tasks.at(i).ls_id_))) { - LOG_WARN("failed to add ls id", KR(ret)); - } else if (OB_FAIL(tablet_ids.push_back(tasks.at(i).tablet_id_))) { - LOG_WARN("failed to add tablet id", KR(ret)); - } - } - if (OB_SUCC(ret) && OB_FAIL(ObTabletReplicaChecksumOperator::batch_remove(tenant_id, ls_ids, tablet_ids, *GCTX.sql_proxy_))) { - LOG_WARN("do tablet checksum remove failed, try to reput to queue", KR(ret), - "escape time", ObTimeUtility::current_time() - start_time); - (void) throttle_(ret, ObTimeUtility::current_time() - start_time); - if (OB_SUCCESS != (tmp_ret = reput_to_queue_(tasks))) { - LOG_ERROR("fail to reput remove task to queue", KR(tmp_ret), K(tasks.count())); - } else { - LOG_INFO("reput remove task to queue success", K(tasks.count())); - } - } else { - LOG_INFO("batch process remove success", KR(ret), K(tasks.count()), - "use_time", ObTimeUtility::current_time() - start_time); - } - } - return ret; -} - -int ObTabletChecksumUpdater::throttle_( - const int return_code, - const int64_t execute_time_us) -{ - int ret = OB_SUCCESS; - int64_t sleep_us = 0; - if (OB_SUCCESS != return_code) { - sleep_us = 2l * 1000 * 1000; // 2s - } else if (execute_time_us > 20 * 1000 * 1000) { // 20s - sleep_us = MIN(1L * 1000 * 1000, (execute_time_us - 20 * 1000 * 1000)); - LOG_WARN("detected slow update, may be too many concurrent updating", K(sleep_us)); - } - const static int64_t sleep_step_us = 20 * 1000; // 20ms - for (; !stopped_ && sleep_us > 0; - sleep_us -= sleep_step_us) { - ob_usleep(static_cast(std::min(sleep_step_us, sleep_us))); - } - return ret; -} - - -} // observer -} // oceanbase diff --git a/src/observer/ob_tablet_checksum_updater.h b/src/observer/ob_tablet_checksum_updater.h deleted file mode 100644 index 1c5042b5d..000000000 --- a/src/observer/ob_tablet_checksum_updater.h +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_OBSERVER_OB_TABLET_CHECKSUM_UPDATER_H_ -#define OCEANBASE_OBSERVER_OB_TABLET_CHECKSUM_UPDATER_H_ - -#include "observer/ob_uniq_task_queue.h" -#include "common/ob_tablet_id.h" -#include "share/ob_ls_id.h" - -namespace oceanbase -{ -namespace common -{ -class ObAddr; -class ObTabletID; -} -namespace share -{ -class ObLSID; -struct ObTabletReplicaChecksumItem; -} - -namespace observer -{ -class ObService; -class ObTabletChecksumUpdater; -class ObTabletChecksumUpdateTask : public ObIUniqTaskQueueTask -{ -public: - friend class ObTabletChecksumUpdater; - - ObTabletChecksumUpdateTask(); - ObTabletChecksumUpdateTask( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - const int64_t add_timestamp); - virtual ~ObTabletChecksumUpdateTask(); - int init( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - const int64_t add_timestamp); - void reset(); - bool is_valid() const; - int64_t hash() const; - void check_task_status() const; - int assign(const ObTabletChecksumUpdateTask &other); - virtual bool operator==(const ObTabletChecksumUpdateTask &other) const; - virtual bool operator!=(const ObTabletChecksumUpdateTask &other) const; - inline int64_t get_tenant_id() const { return tenant_id_; } - inline const share::ObLSID &get_ls_id() const { return ls_id_; } - inline const common::ObTabletID &get_tablet_id() const { return tablet_id_; } - inline int64_t get_add_timestamp() const { return add_timestamp_; } - bool compare_without_version(const ObTabletChecksumUpdateTask& other) const; - uint64_t get_group_id() const { return tenant_id_; } - bool need_process_alone() const { return false; } - virtual bool need_assign_when_equal() const { return false; } - inline int assign_when_equal(const ObTabletChecksumUpdateTask &other) - { - UNUSED(other); - return common::OB_NOT_SUPPORTED; - } - bool is_barrier() const { return false; } - TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(tablet_id), K_(add_timestamp)); -private: - const int64_t TABLET_CHECK_INTERVAL = 120l * 1000 * 1000; //2 minutes - int64_t tenant_id_; - share::ObLSID ls_id_; - common::ObTabletID tablet_id_; - int64_t add_timestamp_; -}; - - -// TODO impl Checker to clean the abandoned task -typedef ObUniqTaskQueue ObTabletChecksumTaskQueue; -typedef ObArray UpdateTabletChecksumTaskList; -typedef ObArray RemoveTabletChecksumTaskList; - -// TODO impl MTL Module -class ObTabletChecksumUpdater -{ -public: - ObTabletChecksumUpdater(); - virtual ~ObTabletChecksumUpdater(); - int init(ObService &ob_service); - inline bool is_inited() const { return inited_; } - void stop(); - void wait(); - void destroy(); - int async_update( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id); - int batch_process_tasks( - const common::ObIArray &tasks, - bool &stopped); - int process_barrier(const ObTabletChecksumUpdateTask &task, bool &stopped); -private: - int add_task_(const ObTabletChecksumUpdateTask &task); - int reput_to_queue_(const ObIArray &tasks); - int generate_tasks_( - const ObIArray &tasks, - ObIArray &update_tablet_items, - UpdateTabletChecksumTaskList &update_tasks, - RemoveTabletChecksumTaskList &remove_tasks); - int do_batch_update_( - const int64_t start_time, - const ObIArray &tasks, - const ObIArray &items); - int do_batch_remove_( - const int64_t start_time, - const ObIArray &tasks); - - int throttle_(const int return_code, const int64_t execute_time_us); -private: - const int64_t MINI_MODE_UPDATE_TASK_THREAD_CNT = 1; - const int64_t UPDATE_TASK_THREAD_CNT = 7; - const int64_t MINI_MODE_TASK_QUEUE_SIZE = 20 * 10000; - const int64_t TASK_QUEUE_SIZE = 100 * 10000; - bool inited_; - bool stopped_; - ObService *ob_service_; - ObTabletChecksumTaskQueue task_queue_; - DISALLOW_COPY_AND_ASSIGN(ObTabletChecksumUpdater); -}; - - -} // observer -} // oceanbase -#endif // OCEANBASE_OBSERVER_OB_TABLET_CHECKSUM_UPDATER_H_ \ No newline at end of file diff --git a/src/observer/report/ob_i_meta_report.h b/src/observer/report/ob_i_meta_report.h index 258aedf12..01d9a906c 100644 --- a/src/observer/report/ob_i_meta_report.h +++ b/src/observer/report/ob_i_meta_report.h @@ -37,10 +37,6 @@ public: const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObTabletID &tablet_id) = 0; - virtual int submit_tablet_checksums_task( - const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id) = 0; }; } // end namespace observer diff --git a/src/observer/report/ob_tablet_table_updater.cpp b/src/observer/report/ob_tablet_table_updater.cpp index 8a1eeade6..ebaf2502d 100644 --- a/src/observer/report/ob_tablet_table_updater.cpp +++ b/src/observer/report/ob_tablet_table_updater.cpp @@ -17,6 +17,9 @@ #include "share/tablet/ob_tablet_info.h" // for ObTabletInfo #include "share/tablet/ob_tablet_table_operator.h" // for ObTabletOperator #include "observer/ob_service.h" // for is_mini_mode +#include "share/ob_tablet_replica_checksum_operator.h" // for ObTabletReplicaChecksumItem +#include "lib/mysqlclient/ob_mysql_transaction.h" // ObMySQLTransaction +#include "lib/mysqlclient/ob_mysql_proxy.h" namespace oceanbase { @@ -287,6 +290,7 @@ int ObTabletTableUpdater::generate_tasks_( const ObIArray &batch_tasks, ObArray &update_tablet_replicas, ObArray &remove_tablet_replicas, + ObArray &update_tablet_checksums, UpdateTaskList &update_tablet_tasks, RemoveTaskList &remove_tablet_tasks) { @@ -307,6 +311,7 @@ int ObTabletTableUpdater::generate_tasks_( // shall never be here } else { ObTabletReplica replica; + ObTabletReplicaChecksumItem checksum_item; FOREACH_CNT_X(task, batch_tasks, OB_SUCC(ret)) { // split tasks into remove and update if (OB_ISNULL(task)) { @@ -318,10 +323,12 @@ int ObTabletTableUpdater::generate_tasks_( } else { task->check_task_status(); replica.reset(); - if (OB_FAIL(ob_service_->fill_tablet_replica(task->get_tenant_id(), - task->get_ls_id(), - task->get_tablet_id(), - replica))) { + checksum_item.reset(); + if (OB_FAIL(ob_service_->fill_tablet_report_info(task->get_tenant_id(), + task->get_ls_id(), + task->get_tablet_id(), + replica, + checksum_item))) { if (OB_TABLET_NOT_EXIST == ret || OB_LS_NOT_EXIST == ret) { ret = OB_SUCCESS; // fill primary keys of the replica for removing @@ -353,11 +360,16 @@ int ObTabletTableUpdater::generate_tasks_( if (OB_FAIL(update_tablet_replicas.reserve(count))) { // reserve() is reentrant, do not have to check whether first time LOG_WARN("fail to reserve update_tablet_replicas", KR(ret), K(count)); + } else if (OB_FAIL(update_tablet_checksums.reserve(count))) { + // reserve() is reentrant, do not have to check whether first time + LOG_WARN("fail to reserve update_tablet_checksums", KR(ret), K(count)); } else if (OB_FAIL(update_tablet_tasks.reserve(count))) { // reserve() is reentrant, do not have to check whether first time LOG_WARN("fail to reserve update_tablet_tasks", KR(ret), K(count)); } else if (OB_FAIL(update_tablet_replicas.push_back(replica))) { LOG_WARN("fail to push back replica", KR(ret), K(replica)); + } else if (OB_FAIL(update_tablet_checksums.push_back(checksum_item))) { + LOG_WARN("fail to push back checksum item", KR(ret), K(checksum_item)); } else if (OB_FAIL(update_tablet_tasks.push_back(*task))) { LOG_WARN("fail to push back task", KR(ret), KPC(task)); } @@ -367,12 +379,14 @@ int ObTabletTableUpdater::generate_tasks_( if (OB_SUCC(ret) && (update_tablet_tasks.count() != update_tablet_replicas.count() + || update_tablet_tasks.count() != update_tablet_checksums.count() || update_tablet_tasks.count() + remove_tablet_tasks.count() != batch_tasks.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet task count and replica count not match", KR(ret), "tablet_update_tasks count", update_tablet_tasks.count(), "tablet_update_replicas count", update_tablet_replicas.count(), + "tablet_update_checksums count", update_tablet_checksums.count(), "tablet_remove_tasks count", remove_tablet_tasks.count(), "batch_tasks count", batch_tasks.count()); } @@ -390,6 +404,7 @@ int ObTabletTableUpdater::batch_process_tasks( const int64_t start_time = ObTimeUtility::current_time(); ObArray update_tablet_replicas; ObArray remove_tablet_replicas; + ObArray update_tablet_checksums; UpdateTaskList update_tablet_tasks; RemoveTaskList remove_tablet_tasks; ObCurTraceId::init(GCONF.self_addr_); @@ -424,19 +439,21 @@ int ObTabletTableUpdater::batch_process_tasks( batch_tasks, update_tablet_replicas, remove_tablet_replicas, + update_tablet_checksums, update_tablet_tasks, remove_tablet_tasks))) { LOG_ERROR("generate_tasks failed", KR(ret), "batch_tasks count", batch_tasks.count(), "update_tablet_replicas", update_tablet_replicas.count(), "remove_tablet_replicas", remove_tablet_replicas.count(), + "update_tablet_checksums", update_tablet_checksums.count(), "update_tablet_tasks", update_tablet_tasks.count(), "remove_tablet_tasks", remove_tablet_tasks.count()); } else { - tmp_ret = do_batch_update_(start_time, update_tablet_tasks, update_tablet_replicas); + tmp_ret = do_batch_update_(start_time, update_tablet_tasks, update_tablet_replicas, update_tablet_checksums); if (OB_SUCCESS != tmp_ret) { ret = OB_SUCC(ret) ? tmp_ret : ret; LOG_WARN("do_batch_update_ failed", KR(tmp_ret), K(start_time), K(update_tablet_tasks), - K(update_tablet_replicas)); + K(update_tablet_replicas), K(update_tablet_checksums)); } else { ObTaskController::get().allow_next_syslog(); LOG_INFO("REPORT: success to update tablets", KR(tmp_ret), K(update_tablet_replicas)); @@ -472,18 +489,37 @@ int ObTabletTableUpdater::do_batch_remove_( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid tasks count", KR(ret), K(tasks_count)); } else if (FALSE_IT(tenant_id = tasks.at(0).get_tenant_id())) { - } else if (OB_FAIL(tablet_operator_->batch_remove(tenant_id, replicas))) { - LOG_WARN("do tablet table remove failed, try to reput to queue", KR(ret), - "escape time", ObTimeUtility::current_time() - start_time); - (void) throttle_(ret, ObTimeUtility::current_time() - start_time); - if (OB_SUCCESS != (tmp_ret = reput_to_queue_(tasks))) { - LOG_ERROR("fail to reput remove task to queue", KR(tmp_ret), K(tasks_count)); - } else { - LOG_INFO("reput remove task to queue success", K(tasks_count)); - } } else { - LOG_INFO("batch process remove success", KR(ret), K(tasks_count), - "use_time", ObTimeUtility::current_time() - start_time); + common::ObMySQLTransaction trans; + const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id); + if (OB_FAIL(trans.start(GCTX.sql_proxy_, meta_tenant_id))) { + LOG_WARN("fail to start transaction", KR(ret), K(tenant_id), K(meta_tenant_id)); + } else if (OB_FAIL(tablet_operator_->batch_remove(trans, tenant_id, replicas))) { + LOG_WARN("do tablet table remove failed, try to reput to queue", KR(ret), + "escape time", ObTimeUtility::current_time() - start_time); + } else if (OB_FAIL(ObTabletReplicaChecksumOperator::batch_remove_with_trans(trans, tenant_id, replicas))) { + LOG_WARN("do tablet table checksum remove failed, try to reput to queue", KR(ret), + "escape time", ObTimeUtility::current_time() - start_time); + } + + if (trans.is_started()) { + int trans_ret = trans.end(OB_SUCCESS == ret); + if (OB_SUCCESS != trans_ret) { + LOG_WARN("fail to end transaction", KR(trans_ret)); + ret = ((OB_SUCCESS == ret) ? trans_ret : ret); + } + } + if (OB_FAIL(ret)) { + (void) throttle_(ret, ObTimeUtility::current_time() - start_time); + if (OB_SUCCESS != (tmp_ret = reput_to_queue_(tasks))) { + LOG_ERROR("fail to reput remove task to queue", KR(tmp_ret), K(tasks_count)); + } else { + LOG_INFO("reput remove task to queue success", K(tasks_count)); + } + } else { + LOG_INFO("batch process remove success", KR(ret), K(tasks_count), + "use_time", ObTimeUtility::current_time() - start_time); + } } return ret; } @@ -491,30 +527,51 @@ int ObTabletTableUpdater::do_batch_remove_( int ObTabletTableUpdater::do_batch_update_( const int64_t start_time, const ObIArray &tasks, - const ObIArray &replicas) + const ObIArray &replicas, + const ObIArray &checksums) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; int64_t tenant_id = OB_INVALID_TENANT_ID; if (tasks.count() != replicas.count() + || tasks.count() != checksums.count() || OB_ISNULL(tablet_operator_) || 0 == tasks.count()) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("tasks num not match or invalid tablet_operator", - KR(ret), "task_cnt", tasks.count(), "replica_cnt", replicas.count()); + LOG_WARN("tasks num not match or invalid tablet_operator", KR(ret), "task_cnt", tasks.count(), + "replica_cnt", replicas.count(), "checksum_cnt", checksums.count()); } else if (FALSE_IT(tenant_id = tasks.at(0).get_tenant_id())) { - } else if (OB_FAIL(tablet_operator_->batch_update(tenant_id, replicas))) { - LOG_WARN("do tablet table update failed, try to reput to queue", KR(ret), - "escape time", ObTimeUtility::current_time() - start_time); - (void) throttle_(ret, ObTimeUtility::current_time() - start_time); - if (OB_SUCCESS != (tmp_ret = reput_to_queue_(tasks))) { - LOG_ERROR("fail to reput update task to queue", KR(tmp_ret), K(tasks.count())); - } else { - LOG_INFO("reput update task to queue success", K(tasks.count())); - } } else { - LOG_INFO("batch process update success", KR(ret), K(replicas.count()), + common::ObMySQLTransaction trans; + const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id); + if (OB_FAIL(trans.start(GCTX.sql_proxy_, meta_tenant_id))) { + LOG_WARN("fail to start transaction", KR(ret), K(tenant_id), K(meta_tenant_id)); + } else if (OB_FAIL(tablet_operator_->batch_update(trans, tenant_id, replicas))) { + LOG_WARN("do tablet table update failed, try to reput to queue", KR(ret), + "escape time", ObTimeUtility::current_time() - start_time); + } else if (OB_FAIL(ObTabletReplicaChecksumOperator::batch_update_with_trans(trans, tenant_id, checksums))) { + LOG_WARN("do tablet table checksum update failed, try to reput to queue", KR(ret), + "escape time", ObTimeUtility::current_time() - start_time); + } + + if (trans.is_started()) { + int trans_ret = trans.end(OB_SUCCESS == ret); + if (OB_SUCCESS != trans_ret) { + LOG_WARN("fail to end transaction", KR(trans_ret)); + ret = ((OB_SUCCESS == ret) ? trans_ret : ret); + } + } + if (OB_FAIL(ret)) { + (void) throttle_(ret, ObTimeUtility::current_time() - start_time); + if (OB_SUCCESS != (tmp_ret = reput_to_queue_(tasks))) { + LOG_ERROR("fail to reput update task to queue", KR(tmp_ret), K(tasks.count())); + } else { + LOG_INFO("reput update task to queue success", K(tasks.count())); + } + } else { + LOG_INFO("batch process update success", KR(ret), K(replicas.count()), "use_time", ObTimeUtility::current_time() - start_time); + } } return ret; } diff --git a/src/observer/report/ob_tablet_table_updater.h b/src/observer/report/ob_tablet_table_updater.h index a39139377..8be00f37f 100644 --- a/src/observer/report/ob_tablet_table_updater.h +++ b/src/observer/report/ob_tablet_table_updater.h @@ -29,6 +29,7 @@ namespace share class ObLSID; class ObTabletReplica; class ObTabletTableOperator; +struct ObTabletReplicaChecksumItem; } namespace observer { @@ -143,12 +144,14 @@ private: // @parma [in] batch_tasks, input tasks // @parma [out] update_tablet_replicas, generated update replicas // @parma [out] remove_tablet_replicas, generated remove replicas + // @parma [out] update_tablet_checksums, generated update tablet checksums // @parma [out] update_tablet_tasks, generated update tasks // @parma [out] remove_tablet_tasks, generated remove tasks int generate_tasks_( const ObIArray &batch_tasks, ObArray &update_tablet_replicas, ObArray &remove_tablet_replicas, + ObArray &update_tablet_checksums, UpdateTaskList &update_tablet_tasks, RemoveTaskList &remove_tablet_tasks); @@ -156,10 +159,12 @@ private: // @parma [in] start_time, the time to start this execution // @parma [in] tasks, batch of tasks to execute // @parma [in] replicas, related replica to each task + // @parma [in] checksums, related checksum to each task int do_batch_update_( const int64_t start_time, const ObIArray &tasks, - const ObIArray &replicas); + const ObIArray &replicas, + const ObIArray &checksums); // do_batch_remove - the real action to remove a batch of tasks // @parma [in] start_time, the time to start this execution diff --git a/src/observer/report/ob_tenant_meta_checker.cpp b/src/observer/report/ob_tenant_meta_checker.cpp index 0ea73e7d7..55e9796bf 100644 --- a/src/observer/report/ob_tenant_meta_checker.cpp +++ b/src/observer/report/ob_tenant_meta_checker.cpp @@ -22,6 +22,7 @@ #include "share/tablet/ob_tablet_table_iterator.h" // ObTenantTabletTableIterator #include "storage/tx_storage/ob_ls_service.h" // ObLSService, ObLSIterator #include "storage/tx_storage/ob_ls_handle.h" // ObLSHandle +#include "share/ob_tablet_replica_checksum_operator.h" // ObTabletReplicaChecksumItem namespace oceanbase { @@ -621,6 +622,8 @@ int ObTenantMetaChecker::check_report_replicas_( ObTabletID tablet_id; ObTabletReplica local_replica; // replica from local ObTabletReplica table_replica; // replica from meta table + share::ObTabletReplicaChecksumItem tablet_checksum; // TODO(@donglou.zl) check tablet_replica_checksum + const bool need_checksum = false; const ObLSID &ls_id = ls->get_ls_id(); while (OB_SUCC(ret)) { if (OB_FAIL(tablet_iter.get_next_tablet(tablet_handle))) { @@ -650,11 +653,13 @@ int ObTenantMetaChecker::check_report_replicas_( LOG_WARN("get replica from hashmap failed", KR(ret), K_(tenant_id), K(ls_id), K(tablet_id)); } - } else if (OB_FAIL(GCTX.ob_service_->fill_tablet_replica( + } else if (OB_FAIL(GCTX.ob_service_->fill_tablet_report_info( tenant_id_, ls_id, tablet_id, - local_replica))) { + local_replica, + tablet_checksum, + need_checksum))) { LOG_WARN("fail to fill tablet replica", KR(ret), K_(tenant_id), K(ls_id), K(tablet_id)); } else if (table_replica.is_equal_for_report(local_replica)) { continue; diff --git a/src/observer/table/ob_table_rpc_processor.cpp b/src/observer/table/ob_table_rpc_processor.cpp index 91b9814fe..831bd844f 100644 --- a/src/observer/table/ob_table_rpc_processor.cpp +++ b/src/observer/table/ob_table_rpc_processor.cpp @@ -411,7 +411,7 @@ int ObTableApiProcessorBase::setup_tx_snapshot_(transaction::ObTxDesc &trans_des LOG_WARN("fail to get global read snapshot", K(ret)); } } else { - palf::SCN weak_read_snapshot; + SCN weak_read_snapshot; if (OB_FAIL(txs->get_weak_read_snapshot_version(weak_read_snapshot))) { LOG_WARN("fail to get weak read snapshot", K(ret)); } else { diff --git a/src/observer/virtual_table/ob_all_virtual_id_service.h b/src/observer/virtual_table/ob_all_virtual_id_service.h index a19bcf921..062b6555a 100644 --- a/src/observer/virtual_table/ob_all_virtual_id_service.h +++ b/src/observer/virtual_table/ob_all_virtual_id_service.h @@ -63,8 +63,8 @@ private: int64_t cur_tenant_id_; int64_t last_id_; int64_t limit_id_; - palf::SCN rec_log_ts_; - palf::SCN latest_log_ts_; + share::SCN rec_log_ts_; + share::SCN latest_log_ts_; int64_t pre_allocated_range_; int64_t submit_log_ts_; bool is_master_; diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp b/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp index 4e2ee2cee..c985b373c 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tablet_info.cpp @@ -13,7 +13,7 @@ #include "observer/ob_server.h" #include "observer/virtual_table/ob_all_virtual_tablet_info.h" #include "storage/tx_storage/ob_ls_service.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" using namespace oceanbase::common; using namespace oceanbase::storage; diff --git a/src/observer/virtual_table/ob_all_virtual_tx_data_table.h b/src/observer/virtual_table/ob_all_virtual_tx_data_table.h index 322e22ead..23734dd7f 100644 --- a/src/observer/virtual_table/ob_all_virtual_tx_data_table.h +++ b/src/observer/virtual_table/ob_all_virtual_tx_data_table.h @@ -26,8 +26,8 @@ private: struct RowData { const char *state_; int64_t tx_data_count_; - palf::SCN min_tx_scn_; - palf::SCN max_tx_scn_; + share::SCN min_tx_scn_; + share::SCN max_tx_scn_; RowData() : state_(""), tx_data_count_(-1), min_tx_scn_(), max_tx_scn_() {} }; diff --git a/src/observer/virtual_table/ob_virtual_ash.cpp b/src/observer/virtual_table/ob_virtual_ash.cpp index 48f2408dd..bb94003f7 100644 --- a/src/observer/virtual_table/ob_virtual_ash.cpp +++ b/src/observer/virtual_table/ob_virtual_ash.cpp @@ -241,6 +241,10 @@ int ObVirtualASH::convert_node_to_row(const ActiveSessionStat &node, ObNewRow *& #endif break; } + case PLAN_ID: { + cells[cell_idx].set_int(node.plan_id_); + break; + } default: { ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "invalid column id", K(column_id), K(cell_idx), diff --git a/src/observer/virtual_table/ob_virtual_ash.h b/src/observer/virtual_table/ob_virtual_ash.h index f818aba50..9e78cf39f 100644 --- a/src/observer/virtual_table/ob_virtual_ash.h +++ b/src/observer/virtual_table/ob_virtual_ash.h @@ -67,7 +67,8 @@ private: MODULE, ACTION, CLIENT_ID, - BACKTRACE + BACKTRACE, + PLAN_ID }; DISALLOW_COPY_AND_ASSIGN(ObVirtualASH); share::ObActiveSessHistList::Iterator iterator_; diff --git a/src/pl/ob_pl_resolver.cpp b/src/pl/ob_pl_resolver.cpp index 0ae363455..52771e8b1 100644 --- a/src/pl/ob_pl_resolver.cpp +++ b/src/pl/ob_pl_resolver.cpp @@ -7136,7 +7136,7 @@ int ObPLResolver::add_pl_integer_checker_expr(ObRawExprFactory &expr_factory, if (OB_SUCC(ret)) { // 对于溢出的检查只关心双目运算符的计算 if (2 == expr->get_param_count() && !IS_COMMON_COMPARISON_OP(expr->get_expr_type()) - && !LOGIC_EXPR(expr) && CHECK_RES_TYPE(expr)) { + && !LOGIC_EXPR(expr) && expr->get_expr_type() != T_FUN_SYS_POWER && CHECK_RES_TYPE(expr)) { const ObRawExpr *left = ObRawExprUtils::skip_implicit_cast(expr->get_param_expr(0)); const ObRawExpr *right = ObRawExprUtils::skip_implicit_cast(expr->get_param_expr(1)); ObPLIntegerType left_pl_integer = PL_INTEGER_INVALID, right_pl_integer = PL_INTEGER_INVALID; diff --git a/src/pl/parser/parse_stmt_node.h b/src/pl/parser/parse_stmt_node.h index 4b06fbac6..11dec8275 100644 --- a/src/pl/parser/parse_stmt_node.h +++ b/src/pl/parser/parse_stmt_node.h @@ -66,7 +66,7 @@ typedef struct _ObParseCtx int comp_mode_; bool is_not_utf8_connection_; const struct ObCharsetInfo *charset_info_; - int64_t last_well_formed_len_; //解析quoted string时的一个临时变量，处理连接gbk字符集时遇到的转义字符问题 + int64_t last_escape_check_pos_; //解析quoted string时的一个临时变量，处理连接gbk字符集时遇到的转义字符问题 int connection_collation_; bool mysql_compatible_comment_; //whether the parser is parsing "/*! xxxx */" struct diff --git a/src/rootserver/backup/ob_backup_data_scheduler.cpp b/src/rootserver/backup/ob_backup_data_scheduler.cpp index 71501869f..8cc848517 100644 --- a/src/rootserver/backup/ob_backup_data_scheduler.cpp +++ b/src/rootserver/backup/ob_backup_data_scheduler.cpp @@ -646,7 +646,7 @@ int ObBackupDataScheduler::start_tenant_backup_data_(const ObBackupJobAttr &job_ return ret; } -int ObBackupDataScheduler::get_scn(common::ObISQLClient &sql_proxy, const uint64_t tenant_id, palf::SCN &scn) +int ObBackupDataScheduler::get_scn(common::ObISQLClient &sql_proxy, const uint64_t tenant_id, SCN &scn) { int ret = OB_SUCCESS; ObAllTenantInfo tenant_info; @@ -660,7 +660,7 @@ int ObBackupDataScheduler::get_scn(common::ObISQLClient &sql_proxy, const uint64 // for backup and restore, we keep the scn round up to microseconds that keep the conversion accuracy is consistent. // meanwhile, in order to solve that boundary is not included in the restore, scn + 1; // 1658475549197665190 --> 1658475549197666000 - palf::SCN tmp_scn; + SCN tmp_scn; int64_t ts; ts = tenant_info.get_standby_scn().convert_to_ts(); if (OB_FAIL(tmp_scn.convert_from_ts(ts))) { @@ -1411,8 +1411,8 @@ int ObUserTenantBackupJobMgr::insert_backup_set_task_(common::ObISQLClient &sql_ backup_set_task.start_ts_ = job_attr_->start_ts_; backup_set_task.meta_turn_id_ = 1; backup_set_task.data_turn_id_ = 1; - backup_set_task.end_scn_ = palf::SCN::min_scn(); - backup_set_task.user_ls_start_scn_ = palf::SCN::min_scn(); + backup_set_task.end_scn_ = SCN::min_scn(); + backup_set_task.user_ls_start_scn_ = SCN::min_scn(); if (OB_FAIL(lease_service_->check_lease())) { LOG_WARN("fail to check leader", K(ret)); } else if (OB_FAIL(ObBackupTaskOperator::insert_backup_task(sql_proxy, backup_set_task))) { @@ -1480,8 +1480,8 @@ int ObUserTenantBackupJobMgr::fill_backup_set_desc_( backup_set_desc.file_status_ = ObBackupFileStatus::BACKUP_FILE_COPYING; backup_set_desc.result_ = job_attr.result_; backup_set_desc.encryption_mode_ = job_attr.encryption_mode_; - backup_set_desc.start_replay_scn_ = palf::SCN::min_scn(); - backup_set_desc.min_restore_scn_ = palf::SCN::min_scn(); + backup_set_desc.start_replay_scn_ = SCN::min_scn(); + backup_set_desc.min_restore_scn_ = SCN::min_scn(); backup_set_desc.backup_compatible_ = ObBackupSetFileDesc::Compatible::COMPATIBLE_VERSION_1; backup_set_desc.tenant_compatible_ = ObClusterVersion::get_instance().get_cluster_version(); backup_set_desc.plus_archivelog_ = job_attr.plus_archivelog_; diff --git a/src/rootserver/backup/ob_backup_data_scheduler.h b/src/rootserver/backup/ob_backup_data_scheduler.h index dc39f2d50..534c2c63a 100644 --- a/src/rootserver/backup/ob_backup_data_scheduler.h +++ b/src/rootserver/backup/ob_backup_data_scheduler.h @@ -53,7 +53,7 @@ public: virtual int get_need_reload_task(common::ObIAllocator &allocator, common::ObIArray &tasks) override; public: // common func used by backup - static int get_scn(common::ObISQLClient &sql_proxy, const uint64_t tenant_id, palf::SCN &scn); + static int get_scn(common::ObISQLClient &sql_proxy, const uint64_t tenant_id, share::SCN &scn); static int check_tenant_status(share::schema::ObMultiVersionSchemaService &schema_service, uint64_t tenant_id, bool &is_valid); static int get_backup_path(common::ObISQLClient &sql_proxy, const uint64_t tenant_id, share::ObBackupPathString &backup_path); static int get_next_job_id(common::ObISQLClient &trans, const uint64_t tenant_id, int64_t &job_id); diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp index 861950e58..c7eaf7ed4 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp @@ -110,7 +110,7 @@ int ObBackupSetTaskMgr::advance_status_( ObMySQLTransaction &trans, const ObBackupStatus &next_status, const int result, - const palf::SCN &scn, + const SCN &scn, const int64_t end_ts) { int ret = OB_SUCCESS; @@ -544,7 +544,7 @@ int ObBackupSetTaskMgr::merge_tablet_to_ls_info_(const ObIArray ls_map; share::ObBackupDataTabletToLSDesc tablet_to_ls_desc; const int64_t OB_BACKUP_MAX_LS_BUCKET = 1024; - palf::SCN max_backup_scn; + SCN max_backup_scn; if (ls_tasks.empty()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("[DATA_BACKUP]invalid argument", K(ret), K(ls_tasks)); @@ -566,7 +566,7 @@ int ObBackupSetTaskMgr::merge_tablet_to_ls_info_(const ObIArrayfirst; const int64_t retry_id = iter->second->retry_id_; share::ObBackupDataTabletToLSInfo tablet_to_ls_info; - palf::SCN backup_scn = palf::SCN::min_scn(); + SCN backup_scn = SCN::min_scn(); ObArray latest_tablet_id; if (OB_FAIL(get_extern_tablet_info_(ls_id, retry_id, tablet_to_ls_info, backup_scn))) { LOG_WARN("fail to get extern tablet info", K(ret), K(ls_id), K(retry_id)); @@ -605,7 +605,7 @@ int ObBackupSetTaskMgr::merge_tablet_to_ls_info_(const ObIArray &cur_tablet_ids, const ObIArray &user_tablet_ids, - const palf::SCN &backup_scn) + const SCN &backup_scn) { int ret = OB_SUCCESS; ObHashSet tablet_set; @@ -686,7 +686,7 @@ int ObBackupSetTaskMgr::check_tablets_match_( int ObBackupSetTaskMgr::do_check_inc_tablets_( const ObLSID &ls_id, const ObIArray &inc_tablets, - const palf::SCN &backup_scn) + const SCN &backup_scn) { int ret = OB_SUCCESS; ObAddr dst_server; @@ -854,7 +854,7 @@ int ObBackupSetTaskMgr::backup_data_() } else if (OB_FAIL(do_backup_data_(ls_task, finish_cnt, build_index_attr))) { LOG_WARN("[DATA_BACKUP]failed to do backup ls task", K(ret), K(ls_task)); } else if (ls_task.count() == finish_cnt) { - palf::SCN end_scn = palf::SCN::min_scn(); + SCN end_scn = SCN::min_scn(); bool need_change_turn = false; bool finish_build_index = false; ObSArray tablets_to_ls; @@ -1566,17 +1566,17 @@ int ObBackupSetTaskMgr::write_backup_set_info_( return ret; } -int ObBackupSetTaskMgr::calculate_start_replay_scn_(palf::SCN &start_replay_scn) +int ObBackupSetTaskMgr::calculate_start_replay_scn_(SCN &start_replay_scn) { int ret = OB_SUCCESS; ObBackupLSMetaInfosDesc ls_meta_infos; - palf::SCN tmp_start_replay_scn = set_task_attr_.start_scn_; + SCN tmp_start_replay_scn = set_task_attr_.start_scn_; if (OB_FAIL(store_.read_ls_meta_infos(ls_meta_infos))) { LOG_WARN("fail to read ls meta infos", K(ret)); } else { ARRAY_FOREACH_X(ls_meta_infos.ls_meta_packages_, i, cnt, OB_SUCC(ret)) { const palf::PalfBaseInfo &palf_base_info = ls_meta_infos.ls_meta_packages_.at(i).palf_meta_; - tmp_start_replay_scn = palf::SCN::min(tmp_start_replay_scn, palf_base_info.prev_log_info_.scn_); + tmp_start_replay_scn = SCN::min(tmp_start_replay_scn, palf_base_info.prev_log_info_.scn_); } if (OB_SUCC(ret)) { start_replay_scn = tmp_start_replay_scn; @@ -1640,8 +1640,8 @@ int ObBackupSetTaskMgr::write_extern_diagnose_info_( int ObBackupSetTaskMgr::write_backup_set_placeholder(const bool is_start) { int ret = OB_SUCCESS; - palf::SCN start_replay_scn = job_attr_->plus_archivelog_ ? set_task_attr_.end_scn_ : set_task_attr_.start_scn_; - palf::SCN min_restore_scn = set_task_attr_.end_scn_; + SCN start_replay_scn = job_attr_->plus_archivelog_ ? set_task_attr_.end_scn_ : set_task_attr_.start_scn_; + SCN min_restore_scn = set_task_attr_.end_scn_; bool is_inner = is_start ? false : true; bool is_success = OB_SUCCESS == job_attr_->result_ ? true : false; if (is_start) { diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.h b/src/rootserver/backup/ob_backup_data_set_task_mgr.h index 445b76074..af2916ba5 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.h +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.h @@ -60,11 +60,11 @@ private: int construct_ls_task_map_(const ObIArray &ls_tasks, hash::ObHashMap &ls_map); int get_extern_tablet_info_(const share::ObLSID &ls_id, const int64_t &retry_cnt, - share::ObBackupDataTabletToLSInfo &tablet_to_ls_info, palf::SCN &backup_scn); + share::ObBackupDataTabletToLSInfo &tablet_to_ls_info, share::SCN &backup_scn); int check_tablets_match_(const share::ObLSID &ls_id, const ObIArray &cur_tablet_ids, - const ObIArray &user_tablet_ids, const palf::SCN &backup_scn); + const ObIArray &user_tablet_ids, const share::SCN &backup_scn); int do_check_inc_tablets_(const share::ObLSID &ls_id, const ObIArray &inc_tablets, - const palf::SCN &backup_scn); + const share::SCN &backup_scn); int get_dst_server_(const share::ObLSID &ls_id, ObAddr &dst); int merge_ls_meta_infos_(const ObIArray &ls_tasks); @@ -93,7 +93,7 @@ private: int backup_completing_log_(); int do_backup_completing_log_(ObArray &ls_task, int64_t &finish_cnt); - int calculate_start_replay_scn_(palf::SCN &start_replay_scn); + int calculate_start_replay_scn_(share::SCN &start_replay_scn); int do_cancel_(); @@ -110,7 +110,7 @@ private: int set_backup_set_files_failed_(ObMySQLTransaction &trans); int advance_status_(ObMySQLTransaction &trans, const share::ObBackupStatus &next_status, const int result = OB_SUCCESS, - const palf::SCN &scn = palf::SCN::min_scn(), const int64_t end_ts = 0); + const share::SCN &scn = share::SCN::min_scn(), const int64_t end_ts = 0); int get_next_status_(const share::ObBackupStatus &cur_status, share::ObBackupStatus &next_status); private: bool is_inited_; diff --git a/src/rootserver/backup/ob_backup_schedule_task.h b/src/rootserver/backup/ob_backup_schedule_task.h index 844c64d55..32bccf373 100644 --- a/src/rootserver/backup/ob_backup_schedule_task.h +++ b/src/rootserver/backup/ob_backup_schedule_task.h @@ -296,7 +296,7 @@ protected: share::ObLSID ls_id_; int64_t turn_id_; int64_t retry_id_; - palf::SCN start_scn_; + share::SCN start_scn_; share::ObBackupPathString backup_path_; share::ObBackupStatus backup_status_; private: @@ -328,8 +328,8 @@ private: share::ObBackupType backup_type_; int64_t backup_date_; share::ObLSID ls_id_; - palf::SCN start_scn_; - palf::SCN end_scn_; + share::SCN start_scn_; + share::SCN end_scn_; share::ObBackupPathString backup_path_; share::ObBackupStatus backup_status_; private: diff --git a/src/rootserver/backup/ob_tenant_archive_scheduler.cpp b/src/rootserver/backup/ob_tenant_archive_scheduler.cpp index 31d0cfc2c..4ced79db5 100644 --- a/src/rootserver/backup/ob_tenant_archive_scheduler.cpp +++ b/src/rootserver/backup/ob_tenant_archive_scheduler.cpp @@ -25,7 +25,7 @@ #include "share/backup/ob_archive_store.h" #include "share/backup/ob_backup_connectivity.h" #include "share/ls/ob_ls_i_life_manager.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_debug_sync.h" using namespace oceanbase; @@ -652,7 +652,7 @@ int ObArchiveHandler::do_checkpoint_(share::ObTenantArchiveRoundAttr &round_info int64_t since_piece_id = 0; ObDestRoundSummary summary; ObDestRoundCheckpointer checkpointer; - palf::SCN max_checkpoint_scn = palf::SCN::min_scn(); + SCN max_checkpoint_scn = SCN::min_scn(); if (OB_FAIL(ObTenantArchiveMgr::decide_piece_id(round_info.start_scn_, round_info.base_piece_id_, round_info.piece_switch_interval_, round_info.checkpoint_scn_, since_piece_id))) { LOG_WARN("failed to calc since piece id", K(ret), K(round_info)); } else if (OB_FAIL(archive_table_op_.get_dest_round_summary(*sql_proxy_, round_info.dest_id_, round_info.round_id_, since_piece_id, summary))) { @@ -678,7 +678,7 @@ int ObArchiveHandler::notify_(const ObTenantArchiveRoundAttr &round) return ret; } -int ObArchiveHandler::get_max_checkpoint_scn_(const uint64_t tenant_id, palf::SCN &max_checkpoint_scn) const +int ObArchiveHandler::get_max_checkpoint_scn_(const uint64_t tenant_id, SCN &max_checkpoint_scn) const { int ret = OB_SUCCESS; ObAllTenantInfo tenant_info; @@ -686,7 +686,7 @@ int ObArchiveHandler::get_max_checkpoint_scn_(const uint64_t tenant_id, palf::SC if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, sql_proxy_, for_update, tenant_info))) { LOG_WARN("failed to get tenant info", K(ret), K(tenant_id)); } else if (OB_FALSE_IT(max_checkpoint_scn = tenant_info.get_standby_scn())) { - } else if (palf::SCN::base_scn() >= max_checkpoint_scn) { + } else if (SCN::base_scn() >= max_checkpoint_scn) { ret = OB_ERR_UNEXPECTED; LOG_WARN("max_checkpoint_scn not valid", K(ret), K(tenant_info)); } diff --git a/src/rootserver/backup/ob_tenant_archive_scheduler.h b/src/rootserver/backup/ob_tenant_archive_scheduler.h index 19ebce0b7..d8778263a 100644 --- a/src/rootserver/backup/ob_tenant_archive_scheduler.h +++ b/src/rootserver/backup/ob_tenant_archive_scheduler.h @@ -67,7 +67,7 @@ private: int notify_(const share::ObTenantArchiveRoundAttr &round_attr); int do_checkpoint_(share::ObTenantArchiveRoundAttr &round_info); int check_archive_dest_validity_(const int64_t dest_no); - int get_max_checkpoint_scn_(const uint64_t tenant_id, palf::SCN &max_checkpoint_scn) const; + int get_max_checkpoint_scn_(const uint64_t tenant_id, share::SCN &max_checkpoint_scn) const; private: bool is_inited_; diff --git a/src/rootserver/ddl_task/ob_constraint_task.cpp b/src/rootserver/ddl_task/ob_constraint_task.cpp index 617b158b5..c318a64e2 100644 --- a/src/rootserver/ddl_task/ob_constraint_task.cpp +++ b/src/rootserver/ddl_task/ob_constraint_task.cpp @@ -18,7 +18,7 @@ #include "share/ob_ddl_common.h" #include "rootserver/ob_root_service.h" #include "rootserver/ob_snapshot_info_manager.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" using namespace oceanbase::common; using namespace oceanbase::share; @@ -601,7 +601,7 @@ int ObConstraintTask::hold_snapshot(const int64_t snapshot_version) int ret = OB_SUCCESS; ObDDLService &ddl_service = root_service_->get_ddl_service(); ObSEArray tablet_ids; - palf::SCN snapshot_scn; + SCN snapshot_scn; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObConstraintTask has not been inited", K(ret)); diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp index 62d7e1784..9b9ab2969 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp @@ -23,7 +23,7 @@ #include "share/ob_ddl_checksum.h" #include "storage/tablelock/ob_table_lock_service.h" #include "storage/tablelock/ob_table_lock_rpc_client.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" using namespace oceanbase::lib; using namespace oceanbase::common; using namespace oceanbase::common::hash; @@ -313,7 +313,7 @@ int ObDDLRedefinitionTask::hold_snapshot(const int64_t snapshot_version) int ret = OB_SUCCESS; ObRootService *root_service = GCTX.root_service_; ObSEArray tablet_ids; - palf::SCN snapshot_scn; + SCN snapshot_scn; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObDDLRedefinitionTask has not been inited", K(ret)); diff --git a/src/rootserver/ddl_task/ob_ddl_task.cpp b/src/rootserver/ddl_task/ob_ddl_task.cpp index dc542c0ac..98e32e166 100644 --- a/src/rootserver/ddl_task/ob_ddl_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_task.cpp @@ -320,6 +320,8 @@ int ObDDLTask::switch_status(ObDDLTaskStatus new_status, const int ret_code) // task failed marked by user real_new_status = FAIL; ret_code_ = OB_CANCELED; + } else if (table_task_status == SUCCESS && old_status != table_task_status) { + real_new_status = SUCCESS; } else if (old_status == new_status) { // do nothing. } else if (OB_FAIL(ObDDLTaskRecordOperator::update_task_status( @@ -519,7 +521,7 @@ int ObDDLTask::batch_release_snapshot( bool need_commit = false; ObMySQLTransaction trans; ObRootService *root_service = GCTX.root_service_; - palf::SCN snapshot_scn; + SCN snapshot_scn; if (OB_ISNULL(root_service)) { ret = OB_ERR_SYS; LOG_WARN("error sys, root service must not be nullptr", K(ret)); @@ -842,7 +844,7 @@ int ObDDLWaitTransEndCtx::get_snapshot(int64_t &snapshot_version) ObFreezeInfoProxy freeze_info_proxy(tenant_id_); ObSimpleFrozenStatus frozen_status; const int64_t timeout = 10 * 1000 * 1000;// 10s - palf::SCN curr_ts; + SCN curr_ts; bool is_external_consistent = false; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -881,7 +883,7 @@ int ObDDLWaitTransEndCtx::get_snapshot(int64_t &snapshot_version) int tmp_ret = OB_SUCCESS; snapshot_version = max(max_snapshot, curr_ts.get_val_for_tx() - INDEX_SNAPSHOT_VERSION_DIFF); if (OB_SUCCESS != (tmp_ret = freeze_info_proxy.get_freeze_info( - root_service->get_sql_proxy(), palf::SCN::min_scn(), frozen_status))) { + root_service->get_sql_proxy(), SCN::min_scn(), frozen_status))) { LOG_WARN("get freeze info failed", K(ret)); } else { const int64_t frozen_scn_val = frozen_status.frozen_scn_.get_val_for_tx(); @@ -1537,7 +1539,7 @@ int ObDDLTaskRecordOperator::fill_task_record( EXTRACT_VARCHAR_FIELD_MYSQL(*result_row, "message_unhex", task_message); EXTRACT_VARCHAR_FIELD_MYSQL(*result_row, "ddl_stmt_str_unhex", ddl_stmt_str); if (OB_SUCC(ret)) { - palf::SCN check_snapshot_version; + SCN check_snapshot_version; if (OB_FAIL(check_snapshot_version.convert_for_tx(task_record.snapshot_version_))) { LOG_WARN("convert for inner table field failed", K(ret), K(task_record.snapshot_version_)); } else if (!check_snapshot_version.is_valid()) { diff --git a/src/rootserver/ddl_task/ob_index_build_task.cpp b/src/rootserver/ddl_task/ob_index_build_task.cpp index 460c6d214..ee32ca18d 100644 --- a/src/rootserver/ddl_task/ob_index_build_task.cpp +++ b/src/rootserver/ddl_task/ob_index_build_task.cpp @@ -17,7 +17,7 @@ #include "share/ob_ddl_checksum.h" #include "share/ob_ddl_error_message_table_operator.h" #include "rootserver/ob_root_service.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" using namespace oceanbase::rootserver; using namespace oceanbase::common; @@ -524,7 +524,7 @@ int ObIndexBuildTask::wait_trans_end() int ObIndexBuildTask::hold_snapshot(const int64_t snapshot) { int ret = OB_SUCCESS; - palf::SCN snapshot_scn; + SCN snapshot_scn; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); diff --git a/src/rootserver/freeze/ob_daily_major_freeze_launcher.cpp b/src/rootserver/freeze/ob_daily_major_freeze_launcher.cpp index ae769d9bf..2811d9afa 100644 --- a/src/rootserver/freeze/ob_daily_major_freeze_launcher.cpp +++ b/src/rootserver/freeze/ob_daily_major_freeze_launcher.cpp @@ -21,7 +21,7 @@ #include "share/ob_tablet_checksum_operator.h" #include "observer/ob_srv_network_frame.h" #include "share/rc/ob_tenant_base.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -206,7 +206,7 @@ int ObDailyMajorFreezeLauncher::try_gc_tablet_checksum() // nothing } else { ObMySQLTransaction trans; - SMART_VAR(ObArray, all_compaction_scn) { + SMART_VAR(ObArray, all_compaction_scn) { if (OB_FAIL(trans.start(sql_proxy_, tenant_id_))) { LOG_WARN("fail to start transaction", KR(ret), K_(tenant_id)); } else if (OB_FAIL(ObTabletChecksumOperator::load_all_compaction_scn(trans, @@ -214,7 +214,7 @@ int ObDailyMajorFreezeLauncher::try_gc_tablet_checksum() LOG_WARN("fail to load all compaction scn", KR(ret), K_(tenant_id)); } else if (all_compaction_scn.count() > MIN_RESERVED_COUNT) { const int64_t snapshot_ver_cnt = all_compaction_scn.count(); - const palf::SCN &gc_snapshot_scn = all_compaction_scn.at(snapshot_ver_cnt - MIN_RESERVED_COUNT - 1); + const SCN &gc_snapshot_scn = all_compaction_scn.at(snapshot_ver_cnt - MIN_RESERVED_COUNT - 1); if (OB_FAIL(ObTabletChecksumOperator::delete_tablet_checksum_items(trans, tenant_id_, gc_snapshot_scn))) { LOG_WARN("fail to delete tablet checksum items", KR(ret), K_(tenant_id), K(gc_snapshot_scn)); diff --git a/src/rootserver/freeze/ob_freeze_info_detector.cpp b/src/rootserver/freeze/ob_freeze_info_detector.cpp index 5ce221165..14e8b3b66 100644 --- a/src/rootserver/freeze/ob_freeze_info_detector.cpp +++ b/src/rootserver/freeze/ob_freeze_info_detector.cpp @@ -249,7 +249,7 @@ int ObFreezeInfoDetector::can_start_work(bool &can_work) if (is_gc_scn_inited_) { // ... } else { - palf::SCN snapshot_gc_scn; + SCN snapshot_gc_scn; ObGlobalStatProxy global_stat_proxy(*sql_proxy_, tenant_id_); if (OB_FAIL(global_stat_proxy.get_snapshot_gc_scn(snapshot_gc_scn))) { LOG_WARN("can not get snapshot gc ts", KR(ret), K_(tenant_id)); diff --git a/src/rootserver/freeze/ob_freeze_info_manager.cpp b/src/rootserver/freeze/ob_freeze_info_manager.cpp index 73fdbd261..0765208cf 100644 --- a/src/rootserver/freeze/ob_freeze_info_manager.cpp +++ b/src/rootserver/freeze/ob_freeze_info_manager.cpp @@ -441,8 +441,8 @@ int ObFreezeInfoManager::get_freeze_info( ObRecursiveMutexGuard guard(lock_); if (OB_FAIL(check_inner_stat())) { LOG_WARN("fail to check inner stat", KR(ret)); - } else if (palf::SCN::base_scn() == frozen_scn) { - frozen_status.frozen_scn_ = palf::SCN::base_scn(); + } else if (SCN::base_scn() == frozen_scn) { + frozen_status.frozen_scn_ = SCN::base_scn(); } else if (OB_FAIL(freeze_info_.get_frozen_status(frozen_scn, frozen_status))) { LOG_WARN("fail to get frozen status", KR(ret), K(frozen_scn), K_(freeze_info)); } @@ -475,6 +475,7 @@ int ObFreezeInfoManager::renew_snapshot_gc_scn() int64_t affected_rows = 0; ObMySQLTransaction trans; ObRecursiveMutexGuard guard(lock_); + int64_t max_stale_time_ns = transaction::ObWeakReadUtil::default_max_stale_time_for_weak_consistency() * 1000; if (OB_FAIL(check_inner_stat())) { LOG_WARN("inner error", KR(ret)); @@ -485,11 +486,12 @@ int ObFreezeInfoManager::renew_snapshot_gc_scn() LOG_WARN("fail to select snapshot_gc_scn for update", KR(ret), K_(tenant_id)); } else if (OB_FAIL(get_gts(cur_gts_scn))) { LOG_WARN("fail to get_gts", KR(ret)); - } else if (FALSE_IT(new_snapshot_gc_scn = SCN::minus(cur_gts_scn, WEAK_TS_NS))) { + } else if (FALSE_IT(new_snapshot_gc_scn = SCN::minus(cur_gts_scn, max_stale_time_ns))) { + LOG_WARN("fail to calc new snapshot_gc_scn", KR(ret), K(cur_gts_scn)); } else if ((new_snapshot_gc_scn <= freeze_info_.latest_snapshot_gc_scn_) || (cur_snapshot_gc_scn >= new_snapshot_gc_scn)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid snaptshot gc time", KR(ret), K(cur_snapshot_gc_scn), K(new_snapshot_gc_scn), + LOG_WARN("invalid snaptshot gc time", KR(ret), K(cur_snapshot_gc_scn), K(new_snapshot_gc_scn), K(freeze_info_.latest_snapshot_gc_scn_)); } else if (OB_FAIL(ObGlobalStatProxy::update_snapshot_gc_scn(trans, tenant_id_, new_snapshot_gc_scn, affected_rows))) { @@ -518,7 +520,7 @@ int ObFreezeInfoManager::renew_snapshot_gc_scn() return ret; } -int ObFreezeInfoManager::get_gts(palf::SCN >s_scn) const +int ObFreezeInfoManager::get_gts(SCN >s_scn) const { int ret = OB_SUCCESS; bool is_external_consistent = true; @@ -551,17 +553,20 @@ int ObFreezeInfoManager::try_gc_freeze_info() ObRecursiveMutexGuard guard(lock_); int ret = OB_SUCCESS; - const uint64_t MAX_KEEP_INTERVAL_NS = 30 * 24 * 60 * 60 * 1000L * 1000L * 1000L; // 30 day + const int64_t MAX_KEEP_INTERVAL_NS = 30 * 24 * 60 * 60 * 1000L * 1000L * 1000L; // 30 day const int64_t MIN_REMAINED_VERSION_COUNT = 32; - SCN min_frozen_scn; SCN cur_gts_scn; - SCN cur_snapshot_gc_scn; + if (OB_FAIL(get_gts(cur_gts_scn))) { + LOG_WARN("fail to get_gts", KR(ret)); + } + SCN min_frozen_scn = SCN::minus(cur_gts_scn, MAX_KEEP_INTERVAL_NS); ObFreezeInfoProxy freeze_info_proxy(tenant_id_); ObMySQLTransaction trans; ObArray all_frozen_status; + SCN cur_snapshot_gc_scn; - if (OB_FAIL(check_inner_stat())) { + if (FAILEDx(check_inner_stat())) { LOG_WARN("inner stat error", K(ret)); } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id_))) { LOG_WARN("fail to start transaction", KR(ret), K_(tenant_id)); @@ -569,9 +574,6 @@ int ObFreezeInfoManager::try_gc_freeze_info() LOG_WARN("fail to select snapshot_gc_scn for update", KR(ret), K_(tenant_id)); } else if (OB_FAIL(freeze_info_proxy.get_all_freeze_info(trans, all_frozen_status))) { LOG_WARN("fail to get all freeze info", KR(ret), K_(tenant_id)); - } else if (OB_FAIL(get_gts(cur_gts_scn))) { - LOG_WARN("fail to get gts", KR(ret)); - } else if (FALSE_IT(min_frozen_scn = SCN::minus(cur_gts_scn, MAX_KEEP_INTERVAL_NS))) { } else { const int64_t frozen_status_cnt = all_frozen_status.count(); if (frozen_status_cnt > MIN_REMAINED_VERSION_COUNT) { @@ -615,20 +617,33 @@ int ObFreezeInfoManager::try_update_zone_info(const int64_t expected_epoch) int ObFreezeInfoManager::check_snapshot_gc_scn() { int ret = OB_SUCCESS; + SCN cur_gts_scn; + SCN snapshot_gc_scn; + int64_t delay = 0; ObRecursiveMutexGuard guard(lock_); if (OB_FAIL(check_inner_stat())) { LOG_WARN("fail to check inner stat", KR(ret)); + } else if (OB_FAIL(get_gts(cur_gts_scn))) { + LOG_WARN("fail to get_gts", KR(ret)); } else { - const uint64_t snapshot_gc_ts = freeze_info_.latest_snapshot_gc_scn_.convert_to_ts(); - const uint64_t delay = ((snapshot_gc_ts == 0) ? 0 : (ObTimeUtility::current_time() - snapshot_gc_ts)); - if (REACH_TIME_INTERVAL(60 * 1000 * 1000)) { - if (delay > SNAPSHOT_GC_TS_ERROR) { - LOG_ERROR("rs_monitor_check : snapshot_gc_ts delay for a long time", - K(snapshot_gc_ts), K(delay), K_(tenant_id)); - } else if (delay > SNAPSHOT_GC_TS_WARN) { - LOG_WARN("rs_monitor_check : snapshot_gc_ts delay for a long time", - K(snapshot_gc_ts), K(delay), K_(tenant_id)); + snapshot_gc_scn = freeze_info_.latest_snapshot_gc_scn_; + if (snapshot_gc_scn > cur_gts_scn) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to check snapshot_gc_scn, snapshot_gc_scn is larger than cur_gts_scn", + KR(ret), K(snapshot_gc_scn), K(cur_gts_scn), K_(tenant_id)); + } else { + const int64_t snapshot_gc_time_us = freeze_info_.latest_snapshot_gc_scn_.convert_to_ts(); + const int64_t delay = ((snapshot_gc_time_us == 0) ? 0 : (ObTimeUtility::current_time() - snapshot_gc_time_us)); + + if (TC_REACH_TIME_INTERVAL(60 * 1000 * 1000)) { + if (delay > SNAPSHOT_GC_TS_ERROR) { + LOG_ERROR("rs_monitor_check : snapshot_gc_ts delay for a long time", + K(snapshot_gc_time_us), K(delay), K_(tenant_id)); + } else if (delay > SNAPSHOT_GC_TS_WARN) { + LOG_WARN("rs_monitor_check : snapshot_gc_ts delay for a long time", + K(snapshot_gc_time_us), K(delay), K_(tenant_id)); + } } } } diff --git a/src/rootserver/freeze/ob_freeze_info_manager.h b/src/rootserver/freeze/ob_freeze_info_manager.h index e4591896c..93e71ea7e 100644 --- a/src/rootserver/freeze/ob_freeze_info_manager.h +++ b/src/rootserver/freeze/ob_freeze_info_manager.h @@ -17,7 +17,7 @@ #include "share/ob_freeze_info_proxy.h" #include "common/storage/ob_freeze_define.h" #include "share/ob_rpc_struct.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -41,7 +41,7 @@ public: // and stored order by freeze_scn asc common::ObSEArray frozen_statuses_; // local cached latest snapshot gc scn - palf::SCN latest_snapshot_gc_scn_; + share::SCN latest_snapshot_gc_scn_; ObFreezeInfo() : frozen_statuses_(), latest_snapshot_gc_scn_() @@ -58,12 +58,12 @@ public: } int assign(const ObFreezeInfo &other); - int get_latest_frozen_scn(palf::SCN &frozen_scn) const; + int get_latest_frozen_scn(share::SCN &frozen_scn) const; int get_min_freeze_info_greater_than( - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, share::ObSimpleFrozenStatus &frozen_status) const; int get_frozen_status( - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, share::ObSimpleFrozenStatus &frozen_status) const; TO_STRING_KV(K_(frozen_statuses), K_(latest_snapshot_gc_scn)); @@ -91,7 +91,7 @@ public: int set_freeze_info(); - int get_freeze_info(const palf::SCN &frozen_scn, + int get_freeze_info(const share::SCN &frozen_scn, share::ObSimpleFrozenStatus &frozen_status); int renew_snapshot_gc_scn(); @@ -102,9 +102,9 @@ public: int check_need_broadcast(bool &need_broadcast); int broadcast_freeze_info(const int64_t expected_epoch); - int get_global_last_merged_scn(palf::SCN &global_last_merged_scn) const; - int get_global_broadcast_scn(palf::SCN &global_broadcast_scn) const; - int get_local_latest_frozen_scn(palf::SCN &frozen_scn); + int get_global_last_merged_scn(share::SCN &global_last_merged_scn) const; + int get_global_broadcast_scn(share::SCN &global_broadcast_scn) const; + int get_local_latest_frozen_scn(share::SCN &frozen_scn); void reset_freeze_info(); @@ -113,13 +113,13 @@ private: int generate_frozen_scn( const ObFreezeInfo &freeze_info, - const palf::SCN &snapshot_gc_scn, - palf::SCN &new_frozen_scn); + const share::SCN &snapshot_gc_scn, + share::SCN &new_frozen_scn); - int set_local_snapshot_gc_scn(const palf::SCN &new_scn); + int set_local_snapshot_gc_scn(const share::SCN &new_scn); - int get_gts(palf::SCN >s_scn) const; - int get_schema_version(const palf::SCN &frozen_scn, int64_t &schema_version) const; + int get_gts(share::SCN >s_scn) const; + int get_schema_version(const share::SCN &frozen_scn, int64_t &schema_version) const; int get_min_freeze_info(share::ObSimpleFrozenStatus &frozen_status); int get_min_freeze_info_to_broadcast(share::ObSimpleFrozenStatus &frozen_status) const; @@ -129,6 +129,7 @@ public: static const int64_t SNAPSHOT_GC_TS_WARN = 30LL * 60LL * 1000LL * 1000LL; static const int64_t SNAPSHOT_GC_TS_ERROR = 2LL * 60LL * 60LL * 1000LL * 1000LL; + private: bool is_inited_; int64_t tenant_id_; diff --git a/src/rootserver/freeze/ob_major_freeze_helper.cpp b/src/rootserver/freeze/ob_major_freeze_helper.cpp index 53283cbf5..df5186051 100644 --- a/src/rootserver/freeze/ob_major_freeze_helper.cpp +++ b/src/rootserver/freeze/ob_major_freeze_helper.cpp @@ -19,6 +19,7 @@ namespace oceanbase { +using namespace share; namespace rootserver { @@ -356,7 +357,7 @@ int ObMajorFreezeHelper::do_one_tenant_admin_merge( int ObMajorFreezeHelper::get_frozen_status( const int64_t tenant_id, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, share::ObSimpleFrozenStatus &frozen_status) { int ret = OB_SUCCESS; @@ -374,13 +375,13 @@ int ObMajorFreezeHelper::get_frozen_status( int ObMajorFreezeHelper::get_frozen_scn( const int64_t tenant_id, - palf::SCN &frozen_scn) + SCN &frozen_scn) { int ret = OB_SUCCESS; share::ObSimpleFrozenStatus frozen_status; // use min_scn to get frozen_status, means get one with biggest frozen_scn - if (OB_FAIL(get_frozen_status(tenant_id, palf::SCN::min_scn(), frozen_status))) { + if (OB_FAIL(get_frozen_status(tenant_id, SCN::min_scn(), frozen_status))) { LOG_WARN("fail to get frozen info", KR(ret)); } else { frozen_scn = frozen_status.frozen_scn_; diff --git a/src/rootserver/freeze/ob_major_freeze_helper.h b/src/rootserver/freeze/ob_major_freeze_helper.h index ef1fda510..fd199139b 100644 --- a/src/rootserver/freeze/ob_major_freeze_helper.h +++ b/src/rootserver/freeze/ob_major_freeze_helper.h @@ -14,16 +14,13 @@ #define OCEANBASE_ROOTSERVER_FREEZE_OB_MAJOR_FREEZE_HELPER_H_ #include "rpc/frame/ob_req_transport.h" #include "rootserver/freeze/ob_major_freeze_rpc_define.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { -namespace palf -{ -class SCN; -} namespace share { +class SCN; class ObSimpleFrozenStatus; } namespace rootserver @@ -102,9 +99,9 @@ public: static int clear_merge_error(const ObTenantAdminMergeParam ¶m); static int get_frozen_status(const int64_t tenant_id, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, share::ObSimpleFrozenStatus &frozen_status); - static int get_frozen_scn(const int64_t tenant_id, palf::SCN &frozen_scn); + static int get_frozen_scn(const int64_t tenant_id, share::SCN &frozen_scn); private: static int get_freeze_info( diff --git a/src/rootserver/freeze/ob_major_freeze_service.h b/src/rootserver/freeze/ob_major_freeze_service.h index 83dfe82e1..a6ae3bd4b 100644 --- a/src/rootserver/freeze/ob_major_freeze_service.h +++ b/src/rootserver/freeze/ob_major_freeze_service.h @@ -15,7 +15,7 @@ #include "share/ob_ls_id.h" #include "logservice/ob_log_base_type.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "lib/lock/ob_recursive_mutex.h" namespace oceanbase @@ -38,18 +38,18 @@ public: int init(uint64_t tenant_id); - int flush(palf::SCN &rec_scn) + int flush(share::SCN &rec_scn) { UNUSED(rec_scn); return OB_SUCCESS; } - palf::SCN get_rec_scn() override { return palf::SCN::max_scn(); } + share::SCN get_rec_scn() override { return share::SCN::max_scn(); } // for replay, do nothing int replay(const void *buffer, const int64_t buf_size, const palf::LSN &lsn, - const palf::SCN &scn) + const share::SCN &scn) { UNUSED(buffer); UNUSED(buf_size); diff --git a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp index e72ca678b..09f190ea4 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_checker.cpp +++ b/src/rootserver/freeze/ob_major_merge_progress_checker.cpp @@ -22,7 +22,7 @@ #include "share/ls/ob_ls_table_operator.h" #include "share/tablet/ob_tablet_table_iterator.h" #include "share/ob_freeze_info_proxy.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -64,7 +64,7 @@ int ObMajorMergeProgressChecker::init( int ObMajorMergeProgressChecker::check_merge_progress( const volatile bool &stop, - const palf::SCN &global_broadcast_scn, + const SCN &global_broadcast_scn, ObAllZoneMergeProgress &all_progress) { int ret = OB_SUCCESS; @@ -155,7 +155,7 @@ int ObMajorMergeProgressChecker::check_tablet( const ObTabletInfo &tablet, const common::hash::ObHashMap &tablet_map, ObAllZoneMergeProgress &all_progress, - const palf::SCN &global_broadcast_scn, + const SCN &global_broadcast_scn, ObSchemaGetterGuard &schema_guard) { int ret = OB_SUCCESS; @@ -209,7 +209,7 @@ int ObMajorMergeProgressChecker::check_tablet( int ObMajorMergeProgressChecker::check_tablet_data_version( ObAllZoneMergeProgress &all_progress, - const palf::SCN &global_broadcast_scn, + const SCN &global_broadcast_scn, const ObTabletInfo &tablet, const share::ObLSInfo &ls_info) { @@ -230,11 +230,11 @@ int ObMajorMergeProgressChecker::check_tablet_data_version( || (REPLICA_TYPE_ENCRYPTION_LOGONLY == ls_r->get_replica_type())) { // logonly replica no need check } else { - palf::SCN rep_snapshot_scn; + SCN rep_snapshot_scn; if (OB_FAIL(rep_snapshot_scn.convert_for_tx(r->get_snapshot_version()))) { LOG_WARN("fail to convert val to SCN", KR(ret), "snapshot_version", r->get_snapshot_version()); } else { - if ((p->smallest_snapshot_scn_ <= palf::SCN::min_scn()) + if ((p->smallest_snapshot_scn_ <= SCN::min_scn()) || (p->smallest_snapshot_scn_ > rep_snapshot_scn)) { p->smallest_snapshot_scn_ = rep_snapshot_scn; } diff --git a/src/rootserver/freeze/ob_major_merge_progress_checker.h b/src/rootserver/freeze/ob_major_merge_progress_checker.h index f32aae14e..93f5a5b4e 100644 --- a/src/rootserver/freeze/ob_major_merge_progress_checker.h +++ b/src/rootserver/freeze/ob_major_merge_progress_checker.h @@ -53,17 +53,17 @@ public: share::ObIServerTrace &server_trace); int check_merge_progress(const volatile bool &stop, - const palf::SCN &global_broadcast_scn, + const share::SCN &global_broadcast_scn, share::ObAllZoneMergeProgress &all_progress); private: int check_tablet(const share::ObTabletInfo &tablet, const common::hash::ObHashMap &tablet_map, share::ObAllZoneMergeProgress &all_progress, - const palf::SCN &global_broadcast_scn, + const share::SCN &global_broadcast_scn, share::schema::ObSchemaGetterGuard &schema_guard); int check_tablet_data_version(share::ObAllZoneMergeProgress &all_progress, - const palf::SCN &global_broadcast_scn, + const share::SCN &global_broadcast_scn, const share::ObTabletInfo &tablet, const share::ObLSInfo &ls_info); int check_majority_integrated(share::schema::ObSchemaGetterGuard &schema_guard, diff --git a/src/rootserver/freeze/ob_major_merge_scheduler.cpp b/src/rootserver/freeze/ob_major_merge_scheduler.cpp index 8e9eb2c5f..f49595a4a 100644 --- a/src/rootserver/freeze/ob_major_merge_scheduler.cpp +++ b/src/rootserver/freeze/ob_major_merge_scheduler.cpp @@ -418,7 +418,7 @@ int ObMajorMergeScheduler::generate_next_global_broadcast_scn(const int64_t expe { int ret = OB_SUCCESS; - palf::SCN new_global_broadcast_scn; + SCN new_global_broadcast_scn; // MERGE_STATUS: IDLE -> MERGING if (OB_FAIL(zone_merge_mgr_->generate_next_global_broadcast_scn(expected_epoch, new_global_broadcast_scn))) { LOG_WARN("fail to generate next broadcast scn", KR(ret), K(expected_epoch)); @@ -493,7 +493,7 @@ int ObMajorMergeScheduler::schedule_zones_to_merge( int ObMajorMergeScheduler::start_zones_merge(const ObZoneArray &to_merge, const int64_t expected_epoch) { int ret = OB_SUCCESS; - palf::SCN global_broadcast_scn; + SCN global_broadcast_scn; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -527,7 +527,7 @@ int ObMajorMergeScheduler::update_merge_status(const int64_t expected_epoch) { int ret = OB_SUCCESS; ObAllZoneMergeProgress all_progress; - palf::SCN global_broadcast_scn; + SCN global_broadcast_scn; bool all_merged = true; ObSimpleFrozenStatus frozen_status; @@ -568,11 +568,11 @@ int ObMajorMergeScheduler::update_merge_status(const int64_t expected_epoch) LOG_INFO("zone merge not finish", "zone", progress->zone_, "unmerged_cnt", progress->unmerged_tablet_cnt_); } - palf::SCN cur_all_merged_scn; - const palf::SCN &ori_all_merged_scn = info.all_merged_scn(); - palf::SCN last_merged_scn = (merged ? info.broadcast_scn() : info.last_merged_scn()); + SCN cur_all_merged_scn; + const SCN &ori_all_merged_scn = info.all_merged_scn(); + SCN last_merged_scn = (merged ? info.broadcast_scn() : info.last_merged_scn()); - if (progress->smallest_snapshot_scn_ <= palf::SCN::min_scn()) { + if (progress->smallest_snapshot_scn_ <= SCN::min_scn()) { cur_all_merged_scn = info.broadcast_scn(); } else { cur_all_merged_scn = progress->smallest_snapshot_scn_; @@ -656,6 +656,8 @@ int ObMajorMergeScheduler::try_update_global_merged_scn(const int64_t expected_e LOG_WARN("not inited", KR(ret)); } else if (OB_FAIL(zone_merge_mgr_->get_snapshot(global_info, infos))) { LOG_WARN("fail to get zone info", KR(ret)); + } else if (global_info.is_merge_error()) { + LOG_WARN("should not update global merged scn, cuz is_merge_error is true", K(global_info)); } else { if (global_info.last_merged_scn() != global_info.global_broadcast_scn()) { bool merged = true; diff --git a/src/rootserver/freeze/ob_tenant_major_freeze.cpp b/src/rootserver/freeze/ob_tenant_major_freeze.cpp index c7a7fb1a6..3a6f18abc 100644 --- a/src/rootserver/freeze/ob_tenant_major_freeze.cpp +++ b/src/rootserver/freeze/ob_tenant_major_freeze.cpp @@ -125,7 +125,7 @@ void ObTenantMajorFreeze::resume() merge_scheduler_.resume(); } -int ObTenantMajorFreeze::get_frozen_scn(palf::SCN &frozen_scn) +int ObTenantMajorFreeze::get_frozen_scn(SCN &frozen_scn) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -137,7 +137,7 @@ int ObTenantMajorFreeze::get_frozen_scn(palf::SCN &frozen_scn) return ret; } -int ObTenantMajorFreeze::get_global_broadcast_scn(palf::SCN &global_broadcast_scn) const +int ObTenantMajorFreeze::get_global_broadcast_scn(SCN &global_broadcast_scn) const { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -274,8 +274,8 @@ int ObTenantMajorFreeze::check_tenant_status() const int ObTenantMajorFreeze::check_freeze_info() { int ret = OB_SUCCESS; - palf::SCN latest_frozen_scn; - palf::SCN global_last_merged_scn; + SCN latest_frozen_scn; + SCN global_last_merged_scn; ObZoneMergeInfo::MergeStatus global_merge_status = ObZoneMergeInfo::MergeStatus::MERGE_STATUS_MAX; if (IS_NOT_INIT) { diff --git a/src/rootserver/freeze/ob_tenant_major_freeze.h b/src/rootserver/freeze/ob_tenant_major_freeze.h index f09a230dd..473f19b26 100644 --- a/src/rootserver/freeze/ob_tenant_major_freeze.h +++ b/src/rootserver/freeze/ob_tenant_major_freeze.h @@ -60,8 +60,8 @@ public: uint64_t get_tenant_id() const { return tenant_id_; } - int get_frozen_scn(palf::SCN &frozen_scn); - int get_global_broadcast_scn(palf::SCN &global_broadcast_scn) const; + int get_frozen_scn(share::SCN &frozen_scn); + int get_global_broadcast_scn(share::SCN &global_broadcast_scn) const; int launch_major_freeze(); diff --git a/src/rootserver/freeze/ob_tenant_major_merge_strategy.cpp b/src/rootserver/freeze/ob_tenant_major_merge_strategy.cpp index 49b9b7c5e..c4dcf683b 100644 --- a/src/rootserver/freeze/ob_tenant_major_merge_strategy.cpp +++ b/src/rootserver/freeze/ob_tenant_major_merge_strategy.cpp @@ -51,7 +51,7 @@ int ObTenantMajorMergeStrategy::filter_merging_zones(common::ObIArrayget_global_broadcast_scn(global_broadcast_scn))) { LOG_WARN("fail to get get_global_broadcast_scn", KR(ret), K_(tenant_id)); } diff --git a/src/rootserver/freeze/ob_zone_merge_manager.cpp b/src/rootserver/freeze/ob_zone_merge_manager.cpp index b123aaab9..3631365cf 100644 --- a/src/rootserver/freeze/ob_zone_merge_manager.cpp +++ b/src/rootserver/freeze/ob_zone_merge_manager.cpp @@ -804,9 +804,12 @@ int ObZoneMergeManagerBase::update_global_merge_info_after_merge(const int64_t e const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); if (OB_FAIL(check_inner_stat())) { LOG_WARN("fail to check inner stat", KR(ret), K_(tenant_id)); + } else if (global_merge_info_.is_in_verifying_status()) { + LOG_INFO("already in verifying status, no need to update global merge status again", K_(tenant_id), + "global merge status", global_merge_info_.merge_status_); } else if (global_merge_info_.is_merge_error()) { ret = OB_INNER_STAT_ERROR; - LOG_WARN("should not continue checking checksum, cuz is_merge_error is true", KR(ret), K_(global_merge_info)); + LOG_WARN("should not update global merge status, cuz is_merge_error is true", KR(ret), K_(global_merge_info)); } else { if (OB_FAIL(trans.start(proxy_, meta_tenant_id))) { LOG_WARN("fail to start transaction", KR(ret), K_(tenant_id), K(meta_tenant_id)); diff --git a/src/rootserver/freeze/ob_zone_merge_manager.h b/src/rootserver/freeze/ob_zone_merge_manager.h index feb9a7d40..8ae352657 100644 --- a/src/rootserver/freeze/ob_zone_merge_manager.h +++ b/src/rootserver/freeze/ob_zone_merge_manager.h @@ -15,7 +15,7 @@ #include "share/ob_zone_merge_info.h" #include "lib/mysqlclient/ob_mysql_proxy.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -49,23 +49,23 @@ public: virtual int start_zone_merge(const common::ObZone &zone, const int64_t expected_epoch); virtual int finish_zone_merge(const common::ObZone &zone, const int64_t expected_epoch, - const palf::SCN &last_merged_scn, - const palf::SCN &all_merged_scn); + const share::SCN &last_merged_scn, + const share::SCN &all_merged_scn); int suspend_merge(const int64_t expected_epoch); int resume_merge(const int64_t expected_epoch); int set_merge_error(const int64_t merge_error, const int64_t expected_epoch); int set_zone_merging(const common::ObZone &zone, const int64_t expected_epoch); - int check_need_broadcast(const palf::SCN &frozen_scn, bool &need_broadcast); - int set_global_freeze_info(const palf::SCN &frozen_scn, const int64_t expected_epoch); + int check_need_broadcast(const share::SCN &frozen_scn, bool &need_broadcast); + int set_global_freeze_info(const share::SCN &frozen_scn, const int64_t expected_epoch); - int get_global_broadcast_scn(palf::SCN &global_broadcast_scn) const; - int get_global_last_merged_scn(palf::SCN &global_last_merged_scn) const; + int get_global_broadcast_scn(share::SCN &global_broadcast_scn) const; + int get_global_last_merged_scn(share::SCN &global_last_merged_scn) const; int get_global_merge_status(share::ObZoneMergeInfo::MergeStatus &global_merge_status) const; int get_global_last_merged_time(int64_t &global_last_merged_time) const; int get_global_merge_start_time(int64_t &global_merge_start_time) const; - virtual int generate_next_global_broadcast_scn(const int64_t expected_epoch, palf::SCN &next_scn); + virtual int generate_next_global_broadcast_scn(const int64_t expected_epoch, share::SCN &next_scn); virtual int try_update_global_last_merged_scn(const int64_t expected_epoch); virtual int update_global_merge_info_after_merge(const int64_t expected_epoch); virtual int try_update_zone_merge_info(const int64_t expected_epoch); @@ -89,6 +89,7 @@ private: const common::ObIArray &ori_merge_infos, const common::ObIArray &zone_list, common::ObIArray &to_insert_infos); + protected: common::SpinRWLock lock_; static int copy_infos(ObZoneMergeManagerBase &dest, const ObZoneMergeManagerBase &src); @@ -119,18 +120,18 @@ public: virtual int start_zone_merge(const common::ObZone &zone, const int64_t expected_epoch); virtual int finish_zone_merge(const common::ObZone &zone, const int64_t expected_epoch, - const palf::SCN &last_merged_scn, - const palf::SCN &all_merged_scn); + const share::SCN &last_merged_scn, + const share::SCN &all_merged_scn); virtual int suspend_merge(const int64_t expected_epoch); virtual int resume_merge(const int64_t expected_epoch); virtual int set_merge_error(const int64_t merge_error, const int64_t expected_epoch); virtual int set_zone_merging(const common::ObZone &zone, const int64_t expected_epoch); - virtual int check_need_broadcast(const palf::SCN &frozen_scn, + virtual int check_need_broadcast(const share::SCN &frozen_scn, bool &need_broadcast); - virtual int set_global_freeze_info(const palf::SCN &frozen_scn, const int64_t expected_epoch); + virtual int set_global_freeze_info(const share::SCN &frozen_scn, const int64_t expected_epoch); - virtual int generate_next_global_broadcast_scn(const int64_t expected_epoch, palf::SCN &next_scn); + virtual int generate_next_global_broadcast_scn(const int64_t expected_epoch, share::SCN &next_scn); virtual int try_update_global_last_merged_scn(const int64_t expected_epoch); virtual int update_global_merge_info_after_merge(const int64_t expected_epoch); virtual int try_update_zone_merge_info(const int64_t expected_epoch); diff --git a/src/rootserver/ob_bootstrap.cpp b/src/rootserver/ob_bootstrap.cpp index 632ec60a2..d892ae771 100644 --- a/src/rootserver/ob_bootstrap.cpp +++ b/src/rootserver/ob_bootstrap.cpp @@ -52,7 +52,7 @@ #include "observer/ob_server_struct.h" #include "rootserver/freeze/ob_freeze_info_manager.h" #include "rootserver/ob_table_creator.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -659,7 +659,7 @@ int ObBootstrap::create_all_core_table_partition() ObMySQLTransaction trans; ObMySQLProxy &sql_proxy = ddl_service_.get_sql_proxy(); ObTableCreator table_creator(OB_SYS_TENANT_ID, - palf::SCN::base_scn(), + SCN::base_scn(), lst_operator_, trans); if (OB_FAIL(trans.start(&sql_proxy, OB_SYS_TENANT_ID))) { @@ -708,7 +708,7 @@ int ObBootstrap::create_all_partitions() ObMySQLTransaction trans; ObMySQLProxy &sql_proxy = ddl_service_.get_sql_proxy(); ObTableCreator table_creator(OB_SYS_TENANT_ID, - palf::SCN::base_scn(), + SCN::base_scn(), lst_operator_, trans); if (OB_FAIL(trans.start(&sql_proxy, OB_SYS_TENANT_ID))) { @@ -1157,7 +1157,7 @@ int ObBootstrap::init_global_stat() } else { const int64_t baseline_schema_version = -1; const int64_t rootservice_epoch = 0; - const palf::SCN snapshot_gc_scn = palf::SCN::min_scn(); + const SCN snapshot_gc_scn = SCN::min_scn(); const int64_t snapshot_gc_timestamp = 0; ObGlobalStatProxy global_stat_proxy(trans, OB_SYS_TENANT_ID); ObSchemaStatusProxy *schema_status_proxy = GCTX.schema_status_proxy_; @@ -1366,7 +1366,7 @@ int ObBootstrap::insert_sys_ls_(const share::schema::ObTenantSchema &tenant_sche if (OB_FAIL(status_info.init(OB_SYS_TENANT_ID, SYS_LS, ls_group_id, share::OB_LS_NORMAL, unit_group_id, primary_zone))) { LOG_WARN("failed to init ls info", KR(ret), K(primary_zone)); - } else if (OB_FAIL(life_agent.create_new_ls(status_info, palf::SCN::base_scn(), primary_zone_str.string()))) { + } else if (OB_FAIL(life_agent.create_new_ls(status_info, SCN::base_scn(), primary_zone_str.string()))) { LOG_WARN("failed to get init member list", KR(ret), K(status_info), K(primary_zone_str)); } } diff --git a/src/rootserver/ob_ddl_operator.cpp b/src/rootserver/ob_ddl_operator.cpp index 70ee870e8..b12a676a1 100644 --- a/src/rootserver/ob_ddl_operator.cpp +++ b/src/rootserver/ob_ddl_operator.cpp @@ -74,7 +74,7 @@ #include "share/ob_zone_merge_info.h" #include "storage/tx/ob_i_ts_source.h" #include "share/stat/ob_dbms_stats_maintenance_window.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -2687,7 +2687,8 @@ int ObDDLOperator::insert_single_column(ObMySQLTransaction &trans, LOG_WARN("fail to gen new schema_version", K(ret), K(tenant_id)); } else if (FALSE_IT(new_column.set_schema_version(new_schema_version))) { //do nothing - } else if (OB_FAIL(schema_service->get_table_sql_service().insert_single_column(trans, new_table_schema, new_column))) { + } else if (OB_FAIL(schema_service->get_table_sql_service().insert_single_column( + trans, new_table_schema, new_column, true))) { LOG_WARN("insert single column failed", K(ret)); } return ret; @@ -2721,7 +2722,7 @@ int ObDDLOperator::delete_single_column(ObMySQLTransaction &trans, } int ObDDLOperator::alter_table_create_index(const ObTableSchema &new_table_schema, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObIArray &gen_columns, ObTableSchema &index_schema, common::ObMySQLTransaction &trans) @@ -3855,6 +3856,59 @@ int ObDDLOperator::update_single_column(common::ObMySQLTransaction &trans, return ret; } +int ObDDLOperator::batch_update_system_table_columns( + common::ObMySQLTransaction &trans, + const share::schema::ObTableSchema &orig_table_schema, + share::schema::ObTableSchema &new_table_schema, + const common::ObIArray &add_column_ids, + const common::ObIArray &alter_column_ids, + const common::ObString *ddl_stmt_str/*=NULL*/) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = new_table_schema.get_tenant_id(); + const uint64_t table_id = new_table_schema.get_table_id(); + int64_t new_schema_version = OB_INVALID_VERSION; + ObSchemaService *schema_service_impl = schema_service_.get_schema_service(); + if (OB_ISNULL(schema_service_impl)) { + ret = OB_ERR_SYS; + LOG_WARN("schema_service_impl must not null", KR(ret)); + } else if (OB_FAIL(schema_service_.gen_new_schema_version(tenant_id, new_schema_version))) { + LOG_WARN("fail to gen new schema_version", KR(ret), K(tenant_id)); + } else { + (void) new_table_schema.set_schema_version(new_schema_version); + ObColumnSchemaV2 *new_column = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < add_column_ids.count(); i++) { + const uint64_t column_id = add_column_ids.at(i); + if (OB_ISNULL(new_column = new_table_schema.get_column_schema(column_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get column", KR(ret), K(tenant_id), K(table_id), K(column_id)); + } else if (FALSE_IT(new_column->set_schema_version(new_schema_version))) { + } else if (OB_FAIL(schema_service_impl->get_table_sql_service().insert_single_column( + trans, new_table_schema, *new_column, false))) { + LOG_WARN("fail to insert column", KR(ret), K(tenant_id), K(table_id), K(column_id)); + } + } // end for + + for (int64_t i = 0; OB_SUCC(ret) && i < alter_column_ids.count(); i++) { + const uint64_t column_id = alter_column_ids.at(i); + if (OB_ISNULL(new_column = new_table_schema.get_column_schema(column_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get column", KR(ret), K(tenant_id), K(table_id), K(column_id)); + } else if (FALSE_IT(new_column->set_schema_version(new_schema_version))) { + } else if (OB_FAIL(schema_service_impl->get_table_sql_service().update_single_column( + trans, orig_table_schema, new_table_schema, *new_column, false))) { + LOG_WARN("fail to insert column", KR(ret), K(tenant_id), K(table_id), K(column_id)); + } + } // end for + + if (FAILEDx(schema_service_impl->get_table_sql_service().update_table_attribute( + trans, new_table_schema, OB_DDL_ALTER_TABLE, ddl_stmt_str))) { + LOG_WARN("failed to update table attribute", KR(ret), K(tenant_id), K(table_id)); + } + } + return ret; +} + int ObDDLOperator::update_partition_option(common::ObMySQLTransaction &trans, ObTableSchema &table_schema) { @@ -4034,7 +4088,7 @@ int ObDDLOperator::drop_table( } else if (table_schema.is_index_table() && !is_inner_table(table_schema.get_table_id())) { ObSnapshotInfoManager snapshot_mgr; ObArray tablet_ids; - palf::SCN invalid_scn; + SCN invalid_scn; if (OB_FAIL(snapshot_mgr.init(GCTX.self_addr()))) { LOG_WARN("fail to init snapshot mgr", K(ret)); } else if (OB_FAIL(table_schema.get_tablet_ids(tablet_ids))) { @@ -8612,10 +8666,6 @@ int ObDDLOperator::create_udt(ObUDTTypeInfo &udt_info, CK (OB_NOT_NULL(obj_info)); // set object body id OX (obj_info->set_coll_type(new_udt_id)); - // If it is a create body operation, the function declaration of spec needs to be deleted first - if (FAILEDx(del_routines_in_udt(udt_info, trans, schema_guard))) { - LOG_WARN("failed to delete object routines", K(ret)); - } // udt_info.set_type_id(new_udt_id); } udt_info.set_schema_version(new_schema_version); @@ -8642,11 +8692,7 @@ int ObDDLOperator::create_udt(ObUDTTypeInfo &udt_info, int64_t new_schema_version = OB_INVALID_VERSION; OZ (schema_service_.gen_new_schema_version(tenant_id, new_schema_version)); OX (routine_info.set_schema_version(new_schema_version)); - // OZ (schema_service->get_routine_sql_service().update_routine(routine_info, &trans)); - if (FAILEDx(schema_service->get_routine_sql_service().create_routine(routine_info, - &trans, NULL))) { - LOG_WARN("insert routine info failed", K(routine_info), K(ret)); - } + OZ (schema_service->get_routine_sql_service().update_routine(routine_info, &trans)); } } } diff --git a/src/rootserver/ob_ddl_operator.h b/src/rootserver/ob_ddl_operator.h index d76feb222..c509004f8 100644 --- a/src/rootserver/ob_ddl_operator.h +++ b/src/rootserver/ob_ddl_operator.h @@ -44,7 +44,7 @@ class ObMySQLTransaction; class ObMySQLProxy; class ObISQLClient; } -namespace palf +namespace share { class SCN; } @@ -295,6 +295,13 @@ public: int delete_single_column(common::ObMySQLTransaction &trans, share::schema::ObTableSchema &new_table_schema, const common::ObString &column_name); + int batch_update_system_table_columns( + common::ObMySQLTransaction &trans, + const share::schema::ObTableSchema &orig_table_schema, + share::schema::ObTableSchema &new_table_schema, + const common::ObIArray &add_column_ids, + const common::ObIArray &alter_column_ids, + const common::ObString *ddl_stmt_str = NULL); int create_sequence_in_create_table(share::schema::ObTableSchema &table_schema, common::ObMySQLTransaction &trans, share::schema::ObSchemaGetterGuard &schema_guard, @@ -312,7 +319,7 @@ public: common::ObMySQLTransaction &trans, share::schema::ObSchemaGetterGuard &schema_guard); virtual int alter_table_create_index(const share::schema::ObTableSchema &new_table_schema, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, common::ObIArray &gen_columns, share::schema::ObTableSchema &index_schema, common::ObMySQLTransaction &trans); diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 910243f1e..b71499cfb 100644 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -99,7 +99,7 @@ #include "storage/tablelock/ob_table_lock_rpc_client.h" #include "rootserver/restore/ob_restore_util.h"//insert_user_tenant_restore_job #include "logservice/palf/palf_base_info.h"//PalfBaseInfo -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -327,7 +327,7 @@ int ObDDLService::create_user_tables( const ObString &ddl_stmt_str, const ObErrorInfo &error_info, ObIArray &table_schemas, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObSchemaGetterGuard &schema_guard, const obrpc::ObSequenceDDLArg &sequence_ddl_arg, const uint64_t last_replay_log_id, @@ -405,7 +405,7 @@ int ObDDLService::create_inner_expr_index(ObMySQLTransaction &trans, ObTableSchema &new_table_schema, ObIArray &new_columns, ObTableSchema &index_schema, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, const ObString *ddl_stmt_str) { int ret = OB_SUCCESS; @@ -483,7 +483,7 @@ int ObDDLService::create_global_index( const obrpc::ObCreateIndexArg &arg, const share::schema::ObTableSchema &table_schema, share::schema::ObTableSchema &index_schema, - const palf::SCN &frozen_scn) + const SCN &frozen_scn) { int ret = OB_SUCCESS; if (OB_FAIL(check_inner_stat())) { @@ -503,7 +503,7 @@ int ObDDLService::create_global_inner_expr_index( share::schema::ObTableSchema &new_table_schema, common::ObIArray &new_columns, share::schema::ObTableSchema &index_schema, - const palf::SCN &frozen_scn) + const SCN &frozen_scn) { int ret = OB_SUCCESS; if (OB_FAIL(check_inner_stat())) { @@ -520,7 +520,7 @@ int ObDDLService::create_global_inner_expr_index( int ObDDLService::create_index_table( const obrpc::ObCreateIndexArg &arg, ObTableSchema &table_schema, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObMySQLTransaction &sql_trans) { int ret = OB_SUCCESS; @@ -898,7 +898,7 @@ int ObDDLService::try_format_partition_schema(ObPartitionSchema &partition_schem int ObDDLService::generate_schema( const ObCreateTableArg &arg, ObTableSchema &schema, - const palf::SCN &frozen_scn) + const SCN &frozen_scn) { int ret = OB_SUCCESS; const ObIArray &constraints = arg.constraint_list_; @@ -1659,7 +1659,7 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist, const ObString &ddl_stmt_str, const ObErrorInfo &error_info, ObIArray &table_schemas, - const palf::SCN &ref_frozen_scn, + const SCN &ref_frozen_scn, const obrpc::ObSequenceDDLArg &sequence_ddl_arg, const uint64_t last_replay_log_id, const ObIArray *dep_infos, @@ -1831,7 +1831,7 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist, } } - palf::SCN frozen_scn; + SCN frozen_scn; if (OB_FAIL(ret)) { } else if (OB_ISNULL(GCTX.root_service_)) { ret = OB_ERR_UNEXPECTED; @@ -1941,7 +1941,7 @@ int ObDDLService::create_tables_in_trans(const bool if_not_exist, // If sql_trans is NULL, it need to create a transaction inside the function. int ObDDLService::create_table_in_trans( ObTableSchema &table_schema, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, const ObString *ddl_stmt_str, ObMySQLTransaction *sql_trans, share::schema::ObSchemaGetterGuard &schema_guard) @@ -2726,7 +2726,7 @@ int ObDDLService::create_hidden_table_with_pk_changed( const ObSArray &index_columns, const ObTableSchema &origin_table_schema, ObTableSchema &new_table_schema, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObSchemaGetterGuard &schema_guard, ObDDLOperator &ddl_operator, ObMySQLTransaction &trans, @@ -3509,7 +3509,7 @@ int ObDDLService::alter_table_partition_by( const ObTableSchema &orig_table_schema, ObTableSchema &new_table_schema, ObSchemaGetterGuard &schema_guard, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObDDLOperator &ddl_operator, ObMySQLTransaction &trans) { @@ -3535,46 +3535,59 @@ int ObDDLService::convert_to_character( const ObTableSchema &orig_table_schema, ObTableSchema &new_table_schema, ObSchemaGetterGuard &schema_guard, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObDDLOperator &ddl_operator, ObMySQLTransaction &trans) { int ret = OB_SUCCESS; const bool bind_tablets = false; bool can_convert = false; + const ObSQLMode sql_mode = alter_table_arg.alter_table_schema_.get_sql_mode(); + bool is_oracle_mode = false; AlterTableSchema &alter_table_schema = alter_table_arg.alter_table_schema_; ObCollationType collation_type = alter_table_schema.get_collation_type(); new_table_schema.set_collation_type(collation_type); new_table_schema.set_charset_type(ObCharset::charset_type_by_coll(collation_type)); ObTableSchema::const_column_iterator tmp_begin = orig_table_schema.column_begin(); ObTableSchema::const_column_iterator tmp_end = orig_table_schema.column_end(); - for (; OB_SUCC(ret) && tmp_begin != tmp_end; tmp_begin++) { - ObColumnSchemaV2 *orig_col = (*tmp_begin); - if (OB_ISNULL(orig_col)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("col is NULL", K(ret)); - } else if (OB_FAIL(check_can_convert_to_character(*orig_col, can_convert))) { - LOG_WARN("check can convert to character", K(ret)); - } else if (can_convert) { - ObColumnSchemaV2 *col = new_table_schema.get_column_schema(orig_col->get_column_name()); - if (OB_ISNULL(col)) { + if (OB_FAIL(orig_table_schema.check_if_oracle_compat_mode(is_oracle_mode))) { + LOG_WARN("failed to get oracle mode", K(ret)); + } else { + for (; OB_SUCC(ret) && tmp_begin != tmp_end; tmp_begin++) { + ObColumnSchemaV2 *orig_col = (*tmp_begin); + if (OB_ISNULL(orig_col)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("col is NULL", K(ret)); - } else { - col->set_collation_type(collation_type); - col->set_charset_type(ObCharset::charset_type_by_coll(collation_type)); + } else if (OB_FAIL(check_can_convert_to_character(*orig_col, can_convert))) { + LOG_WARN("check can convert to character", K(ret)); + } else if (can_convert) { + ObColumnSchemaV2 *col = new_table_schema.get_column_schema(orig_col->get_column_name()); + if (OB_ISNULL(col)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("col is NULL", K(ret)); + } else { + col->set_collation_type(collation_type); + col->set_charset_type(ObCharset::charset_type_by_coll(collation_type)); + if (OB_FAIL(fill_column_collation(sql_mode, + is_oracle_mode, + new_table_schema, + alter_table_arg.allocator_, + *col))) { + LOG_WARN("failed to fill column collation", K(ret)); + } + } } } + OZ (create_user_hidden_table(orig_table_schema, + new_table_schema, + &alter_table_arg.sequence_ddl_arg_, + bind_tablets, + schema_guard, + frozen_scn, + ddl_operator, + trans, + alter_table_arg.allocator_)); } - OZ (create_user_hidden_table(orig_table_schema, - new_table_schema, - &alter_table_arg.sequence_ddl_arg_, - bind_tablets, - schema_guard, - frozen_scn, - ddl_operator, - trans, - alter_table_arg.allocator_)); return ret; } @@ -3637,7 +3650,7 @@ int ObDDLService::alter_table_primary_key(obrpc::ObAlterTableArg &alter_table_ar const ObTableSchema &origin_table_schema, ObTableSchema &new_table_schema, ObSchemaGetterGuard &schema_guard, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObDDLOperator &ddl_operator, ObMySQLTransaction &trans, common::ObArenaAllocator &allocator) @@ -4153,7 +4166,7 @@ int ObDDLService::remap_index_tablets_to_new_indexs( ObSchemaGetterGuard &schema_guard, ObSArray &table_schemas, common::ObMySQLTransaction &trans, - const palf::SCN &frozen_scn) + const SCN &frozen_scn) { int ret = OB_SUCCESS; int64_t new_schema_version = OB_INVALID_VERSION; @@ -4323,7 +4336,7 @@ int ObDDLService::swap_orig_and_hidden_table_state( int ObDDLService::remap_index_tablets_and_take_effect( obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn) + const SCN &frozen_scn) { int ret = OB_SUCCESS; ObSArray index_ids; @@ -4553,7 +4566,7 @@ int ObDDLService::create_index_tablet(const ObTableSchema &index_schema, { int ret = OB_SUCCESS; int64_t tenant_id = index_schema.get_tenant_id(); - palf::SCN frozen_scn; + SCN frozen_scn; if (OB_FAIL(check_inner_stat())) { LOG_WARN("check_inner_stat error", K(is_inited()), KR(ret)); } else if (!index_schema.is_index_table()) { @@ -4637,7 +4650,7 @@ int ObDDLService::alter_table_index(const obrpc::ObAlterTableArg &alter_table_ar const ObTableSchema &origin_table_schema, ObTableSchema &new_table_schema, ObSchemaGetterGuard &schema_guard, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObDDLOperator &ddl_operator, ObMySQLTransaction &trans, ObArenaAllocator &allocator, @@ -5764,47 +5777,47 @@ int ObDDLService::fill_column_collation( const bool is_oracle_mode, const ObTableSchema &table_schema, common::ObIAllocator &allocator, - AlterColumnSchema &alter_column_schema) + ObColumnSchemaV2 &column_schema) { int ret = OB_SUCCESS; - ObObjTypeClass col_tc = alter_column_schema.get_data_type_class(); + ObObjTypeClass col_tc = column_schema.get_data_type_class(); ObCollationType collation_type = table_schema.get_collation_type(); ObCharsetType charset_type = table_schema.get_charset_type(); const ObCollationType cur_extended_type_info_collation = ObCharset::get_system_collation(); if (ObStringTC == col_tc) { if (OB_FAIL(ObDDLResolver::check_and_fill_column_charset_info( - alter_column_schema, charset_type, collation_type))) { + column_schema, charset_type, collation_type))) { RS_LOG(WARN, "failed to fill column charset info", K(ret)); } else if (OB_FAIL(ObDDLResolver::check_string_column_length( - alter_column_schema, is_oracle_mode))) { + column_schema, is_oracle_mode))) { RS_LOG(WARN, "failed to check string column length", K(ret)); } } else if (ObRawTC == col_tc) { - if (OB_FAIL(ObDDLResolver::check_raw_column_length(alter_column_schema))) { - RS_LOG(WARN, "failed to check raw column length", K(ret), K(alter_column_schema)); + if (OB_FAIL(ObDDLResolver::check_raw_column_length(column_schema))) { + RS_LOG(WARN, "failed to check raw column length", K(ret), K(column_schema)); } - } else if (ob_is_text_tc(alter_column_schema.get_data_type())) { + } else if (ob_is_text_tc(column_schema.get_data_type())) { if (OB_FAIL(ObDDLResolver::check_and_fill_column_charset_info( - alter_column_schema, table_schema.get_charset_type(), table_schema.get_collation_type()))) { + column_schema, table_schema.get_charset_type(), table_schema.get_collation_type()))) { RS_LOG(WARN, "failed to fill column charset info", K(ret)); - } else if (OB_FAIL(ObDDLResolver::check_text_column_length_and_promote(alter_column_schema, + } else if (OB_FAIL(ObDDLResolver::check_text_column_length_and_promote(column_schema, table_schema.get_table_id()))) { RS_LOG(WARN, "failed to check text or blob column length", K(ret)); } } else if (ObEnumSetTC == col_tc) { - if (OB_FAIL(ObDDLResolver::check_and_fill_column_charset_info(alter_column_schema, charset_type, collation_type))) { - LOG_WARN("fail to check and fill column charset info", K(ret), K(alter_column_schema)); + if (OB_FAIL(ObDDLResolver::check_and_fill_column_charset_info(column_schema, charset_type, collation_type))) { + LOG_WARN("fail to check and fill column charset info", K(ret), K(column_schema)); } else if (OB_FAIL(ObResolverUtils::check_extended_type_info( allocator, - alter_column_schema.get_extended_type_info(), + column_schema.get_extended_type_info(), cur_extended_type_info_collation, - alter_column_schema.get_column_name_str(), - alter_column_schema.get_data_type(), - alter_column_schema.get_collation_type(), + column_schema.get_column_name_str(), + column_schema.get_data_type(), + column_schema.get_collation_type(), sql_mode))) { - LOG_WARN("fail to fill extended type info", K(ret), K(alter_column_schema)); - } else if (OB_FAIL(ObDDLResolver::calc_enum_or_set_data_length(alter_column_schema))) { - LOG_WARN("fail to calc data length", K(ret), K(alter_column_schema)); + LOG_WARN("fail to fill extended type info", K(ret), K(column_schema)); + } else if (OB_FAIL(ObDDLResolver::calc_enum_or_set_data_length(column_schema))) { + LOG_WARN("fail to calc data length", K(ret), K(column_schema)); } } return ret; @@ -7716,7 +7729,7 @@ int ObDDLService::update_prev_id_for_add_column(const ObTableSchema &origin_tabl int ObDDLService::alter_table_column(const ObTableSchema &origin_table_schema, const AlterTableSchema &alter_table_schema, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObTableSchema &new_table_schema, obrpc::ObAlterTableArg &alter_table_arg, ObSchemaGetterGuard &schema_guard, @@ -8229,7 +8242,7 @@ int ObDDLService::create_aux_lob_table_if_need(ObTableSchema &data_table_schema, ObArray aux_table_schemas; const uint64_t tenant_id = data_table_schema.get_tenant_id(); bool need_sync_schema_version = false; - palf::SCN frozen_scn; + SCN frozen_scn; if (OB_FAIL(ObMajorFreezeHelper::get_frozen_scn(tenant_id, frozen_scn))) { LOG_WARN("failed to get frozen status for create tablet", KR(ret), K(tenant_id)); @@ -8967,7 +8980,7 @@ int ObDDLService::update_global_index(ObAlterTableArg &arg, const uint64_t tenant_id, const ObTableSchema &orig_table_schema, ObDDLOperator &ddl_operator, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObMySQLTransaction &trans) { int ret = OB_SUCCESS; @@ -9503,7 +9516,7 @@ int ObDDLService::check_enable_sys_table_ddl(const ObTableSchema &table_schema, // FIXME: this function should move to observer int ObDDLService::alter_table_sess_active_time_in_trans(obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, obrpc::ObAlterTableRes &res) { int ret = OB_SUCCESS; @@ -9602,7 +9615,7 @@ int ObDDLService::update_tables_attribute(ObIArray &new_table_sc //fix me :Check whether the newly added index column covers the partition column --by rongxuan.lc // It can be repaired after the featrue that add index in alter_table statement int ObDDLService::alter_table_in_trans(obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, obrpc::ObAlterTableRes &res) { int ret = OB_SUCCESS; @@ -10036,7 +10049,7 @@ int ObDDLService::alter_table_in_trans(obrpc::ObAlterTableArg &alter_table_arg, || obrpc::ObAlterTableArg::ADD_SUB_PARTITION == alter_table_arg.alter_part_type_ || obrpc::ObAlterTableArg::TRUNCATE_PARTITION == alter_table_arg.alter_part_type_ || obrpc::ObAlterTableArg::TRUNCATE_SUB_PARTITION == alter_table_arg.alter_part_type_) { - palf::SCN frozen_scn; + SCN frozen_scn; if (OB_ISNULL(GCTX.root_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("root service is null", KR(ret)); @@ -10498,7 +10511,7 @@ int ObDDLService::check_ddl_with_primary_key_operation( } int ObDDLService::do_offline_ddl_in_trans(obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, obrpc::ObAlterTableRes &res) { int ret = OB_SUCCESS; @@ -10875,7 +10888,7 @@ int ObDDLService::check_restore_point_allow(const int64_t tenant_id, const ObTab return ret; } -int ObDDLService::check_all_server_frozen_scn(const palf::SCN &frozen_scn) +int ObDDLService::check_all_server_frozen_scn(const SCN &frozen_scn) { int ret = OB_SUCCESS; if (OB_ISNULL(rpc_proxy_) || OB_ISNULL(server_mgr_)) { @@ -11717,7 +11730,7 @@ int ObDDLService::check_add_list_subpartition(const ObPartition &orig_part, cons return ret; } int ObDDLService::alter_table(obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, obrpc::ObAlterTableRes &res) { int ret = OB_SUCCESS; @@ -12595,7 +12608,7 @@ int ObDDLService::truncate_table_in_trans(const obrpc::ObTruncateTableArg &arg, } } - palf::SCN frozen_scn; + SCN frozen_scn; share::ObSimpleFrozenStatus frozen_status; if (OB_FAIL(ret)) { } else if (OB_ISNULL(GCTX.root_service_)) { @@ -13012,7 +13025,7 @@ int ObDDLService::is_foreign_key_name_prefix_match(const ObForeignKeyInfo &origi } int ObDDLService::prepare_hidden_table_schema(const ObTableSchema &orig_table_schema, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObIAllocator &allocator, ObTableSchema &hidden_table_schema) { @@ -13133,7 +13146,7 @@ int ObDDLService::create_user_hidden_table(const ObTableSchema &orig_table_schem const obrpc::ObSequenceDDLArg *sequence_ddl_arg, const bool bind_tablets, ObSchemaGetterGuard &schema_guard, - const palf::SCN &ref_frozen_scn, + const SCN &ref_frozen_scn, ObDDLOperator &ddl_operator, ObMySQLTransaction &trans, ObIAllocator &allocator) @@ -13143,7 +13156,7 @@ int ObDDLService::create_user_hidden_table(const ObTableSchema &orig_table_schem ObArray aux_table_schemas; ObSEArray schemas; // 1(hidden_table) + 2(aux_lob_table) bool need_sync_schema_version = false; - palf::SCN frozen_scn = palf::SCN::min_scn(); + SCN frozen_scn = SCN::min_scn(); bool is_add_identity_column = false; hidden_table_schema.set_in_offline_ddl_white_list(orig_table_schema.check_can_do_ddl()); // allow offline ddl execute if there's no offline ddl doing if (OB_ISNULL(GCTX.root_service_)) { @@ -13475,7 +13488,7 @@ int ObDDLService::add_new_index_schema(obrpc::ObAlterTableArg &alter_table_arg, const share::schema::ObTableSchema &orig_table_schema, const ObTableSchema &hidden_table_schema, ObSchemaGetterGuard &schema_guard, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObSArray &new_table_schemas, ObSArray &index_ids) { @@ -13728,7 +13741,7 @@ int ObDDLService::reconstruct_index_schema(const ObTableSchema &orig_table_schem const common::ObIArray &drop_cols_id_arr, const ObColumnNameMap &col_name_map, const common::ObTimeZoneInfo &tz_info, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObIAllocator &allocator, ObSArray &new_table_schemas, ObSArray &index_ids) @@ -13891,7 +13904,7 @@ int ObDDLService::rebuild_hidden_table_index_in_trans( } int ObDDLService::rebuild_hidden_table_index(obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObSArray &index_ids) { int ret = OB_SUCCESS; @@ -15450,7 +15463,7 @@ int ObDDLService::cleanup_garbage(ObAlterTableArg &alter_table_arg) int ObDDLService::truncate_table(const ObTruncateTableArg &arg, const obrpc::ObDDLRes &ddl_res, - const palf::SCN &frozen_scn) + const SCN &frozen_scn) { int ret = OB_SUCCESS; ObSchemaGetterGuard schema_guard; @@ -15862,7 +15875,7 @@ int ObDDLService::rebuild_table_schema_with_new_id(const ObTableSchema &orig_tab const int64_t session_id, const share::schema::ObTableType table_type_, ObSchemaService &schema_service, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObIArray &new_schemas, ObArenaAllocator &allocator, const uint64_t define_user_id) @@ -16048,7 +16061,7 @@ int ObDDLService::rebuild_table_schema_with_new_id(const ObTableSchema &orig_tab } int ObDDLService::create_table_like(const ObCreateTableLikeArg &arg, - const palf::SCN &frozen_scn) + const SCN &frozen_scn) { int ret = OB_SUCCESS; ObSchemaGetterGuard schema_guard; @@ -18259,7 +18272,7 @@ int ObDDLService::drop_table(const ObDropTableArg &drop_table_arg, const obrpc:: return ret; } -int ObDDLService::rebuild_index(const ObRebuildIndexArg &arg, const palf::SCN &frozen_scn, obrpc::ObAlterTableRes &res) +int ObDDLService::rebuild_index(const ObRebuildIndexArg &arg, const SCN &frozen_scn, obrpc::ObAlterTableRes &res) { int ret = OB_SUCCESS; const uint64_t tenant_id = arg.tenant_id_; @@ -18386,7 +18399,7 @@ int ObDDLService::rebuild_index(const ObRebuildIndexArg &arg, const palf::SCN &f int ObDDLService::rebuild_index_in_trans( ObSchemaGetterGuard &schema_guard, ObTableSchema &index_schema, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, const ObString *ddl_stmt_str, ObMySQLTransaction *sql_trans) { @@ -18501,6 +18514,204 @@ int ObDDLService::update_index_status(const obrpc::ObUpdateIndexStatusArg &arg) return ret; } +int ObDDLService::upgrade_table_schema(const obrpc::ObUpgradeTableSchemaArg &arg) +{ + int ret = OB_SUCCESS; + FLOG_INFO("[UPGRADE] begin upgrade system table", K(arg)); + const uint64_t tenant_id = arg.get_tenant_id(); + const uint64_t table_id = arg.get_table_id(); + int64_t start_time = ObTimeUtility::current_time(); + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("fail to check inner stat", KR(ret)); + } else if (!arg.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(arg)); + } else if (!GCONF.enable_sys_table_ddl) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("upgrade table schema when enable_sys_table_ddl is off is not allowed", + KR(ret), K(arg)); + } else { + HEAP_VAR(ObTableSchema, hard_code_schema) { + ObSchemaGetterGuard schema_guard; + bool exist = false; + if (OB_FAIL(get_hard_code_system_table_schema_( + tenant_id, table_id, hard_code_schema))) { + LOG_WARN("fail to get hard code table schema", KR(ret), K(tenant_id), K(table_id)); + } else if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) { + LOG_WARN("get_schema_guard with version in inner table failed", KR(ret), K(tenant_id)); + } else if (OB_FAIL(schema_guard.check_table_exist(tenant_id, table_id, exist))) { + LOG_WARN("fail to check table exist", KR(ret), K(tenant_id), K(table_id)); + } else if (!exist) { + if (OB_FAIL(create_system_table_(schema_guard, hard_code_schema))) { + LOG_WARN("fail to create system table", KR(ret), K(tenant_id), K(table_id)); + } + } else if (OB_FAIL(alter_system_table_column_(schema_guard, hard_code_schema))) { + LOG_WARN("fail to alter system table's column", KR(ret), K(tenant_id), K(table_id)); + } + } + } + FLOG_INFO("[UPGRADE] end upgrade system table", + KR(ret), K(tenant_id), K(table_id), + "cost", ObTimeUtility::current_time() - start_time); + return ret; + +} + +int ObDDLService::get_hard_code_system_table_schema_( + const uint64_t tenant_id, + const uint64_t table_id, + ObTableSchema &hard_code_schema) +{ + int ret = OB_SUCCESS; + if (OB_INVALID_TENANT_ID == tenant_id + && !is_system_table(table_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant_id or table_id", KR(ret), K(tenant_id), K(table_id)); + } else { + bool finded = false; + const schema_create_func *creator_ptr_array[] = { + share::core_table_schema_creators, + share::sys_table_schema_creators, NULL }; + for (const schema_create_func **creator_ptr_ptr = creator_ptr_array; + OB_SUCC(ret) && !finded && OB_NOT_NULL(*creator_ptr_ptr); ++creator_ptr_ptr) { + for (const schema_create_func *creator_ptr = *creator_ptr_ptr; + OB_SUCC(ret) && !finded && OB_NOT_NULL(*creator_ptr); ++creator_ptr) { + hard_code_schema.reset(); + bool exist = false; + if (OB_FAIL((*creator_ptr)(hard_code_schema))) { + LOG_WARN("create table schema failed", KR(ret)); + } else if (!is_sys_tenant(tenant_id) + && OB_FAIL(ObSchemaUtils::construct_tenant_space_full_table( + tenant_id, hard_code_schema))) { + LOG_WARN("fail to construct tenant space table", KR(ret), K(tenant_id)); + } else if (OB_FAIL(ObSysTableChecker::is_inner_table_exist( + tenant_id, hard_code_schema, exist))) { + LOG_WARN("fail to check inner table exist", + KR(ret), K(tenant_id), K(hard_code_schema)); + } else if (!exist) { + // skip + } else if (hard_code_schema.get_table_id() == table_id) { + finded = true; + } + } // end for + } // end for + + if (OB_SUCC(ret) && !finded) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("hard code table schema not exist", KR(ret), K(tenant_id), K(table_id)); + } + } + return ret; +} + +int ObDDLService::create_system_table_( + ObSchemaGetterGuard &schema_guard, + const ObTableSchema &hard_code_schema) +{ + int ret = OB_SUCCESS; + bool if_not_exist = true; + SCN frozen_scn = SCN::min_scn(); + ObArray table_schemas; + // the following variable is not used + ObString ddl_stmt_str; + ObErrorInfo error_info; + obrpc::ObSequenceDDLArg sequence_ddl_arg; + uint64_t last_replay_log_id = 0; + ObArray dep_infos; + ObArray mock_fk_parent_table_schema_array; + // sys index、sys lob table will be added in create_user_tables() + if (OB_FAIL(table_schemas.push_back(hard_code_schema))) { + LOG_WARN("fail to push back new table schema", KR(ret)); + } else if (OB_FAIL(ObMajorFreezeHelper::get_frozen_scn( + hard_code_schema.get_tenant_id(), frozen_scn))) { + LOG_WARN("get_frozen_scn failed", KR(ret), "tenant_id", hard_code_schema.get_tenant_id()); + } else if (OB_FAIL(create_user_tables(if_not_exist, ddl_stmt_str, + error_info, table_schemas, frozen_scn, schema_guard, sequence_ddl_arg, + last_replay_log_id, &dep_infos, mock_fk_parent_table_schema_array))) { + LOG_WARN("fail to create system table", KR(ret), K(hard_code_schema)); + } + return ret; +} + +int ObDDLService::alter_system_table_column_( + ObSchemaGetterGuard &schema_guard, + const ObTableSchema &hard_code_schema) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = hard_code_schema.get_tenant_id(); + const uint64_t table_id = hard_code_schema.get_table_id(); + const ObTableSchema *orig_table_schema = NULL; + ObArray add_column_ids; + ObArray alter_column_ids; + if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, orig_table_schema))) { + LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); + } else if (OB_ISNULL(orig_table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table not exist", KR(ret), K(tenant_id), K(table_id)); + } else if (OB_FAIL(ObRootInspection::check_and_get_system_table_column_diff( + *orig_table_schema, hard_code_schema, add_column_ids, alter_column_ids))) { + LOG_WARN("fail to check system table's column schemas", KR(ret), K(tenant_id), K(table_id)); + } else if (0 == add_column_ids.count() && 0 == alter_column_ids.count()) { + LOG_INFO("system table's column schemas not changed, just skip", KR(ret), K(tenant_id), K(table_id)); + } else { + ObDDLSQLTransaction trans(schema_service_); + ObDDLOperator ddl_operator(*schema_service_, *sql_proxy_); + int64_t refreshed_schema_version = 0; + HEAP_VAR(ObTableSchema, new_table_schema) { + + if (OB_FAIL(schema_guard.get_schema_version(tenant_id, refreshed_schema_version))) { + LOG_WARN("failed to get tenant schema version", KR(ret), K(tenant_id)); + } else if (OB_FAIL(trans.start(sql_proxy_, tenant_id, refreshed_schema_version))) { + LOG_WARN("failed to start trans", KR(ret), K(tenant_id), K(refreshed_schema_version)); + } else if (OB_FAIL(new_table_schema.assign(*orig_table_schema))) { + LOG_WARN("fail to assign table schema", KR(ret), K(tenant_id), K(table_id)); + } else { + const ObColumnSchemaV2 *hard_code_column = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < add_column_ids.count(); i++) { + const uint64_t column_id = add_column_ids.at(i); + if (OB_ISNULL(hard_code_column = hard_code_schema.get_column_schema(column_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get column schema", KR(ret), K(tenant_id), K(table_id), K(column_id)); + } else if (OB_FAIL(new_table_schema.add_column(*hard_code_column))) { + LOG_WARN("fail to add column", KR(ret), KPC(hard_code_column)); + } + } // end for + + ObColumnSchemaV2 new_column; + for (int64_t i = 0; OB_SUCC(ret) && i < alter_column_ids.count(); i++) { + const uint64_t column_id = alter_column_ids.at(i); + if (OB_ISNULL(hard_code_column = hard_code_schema.get_column_schema(column_id))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get column schema", KR(ret), K(tenant_id), K(table_id), K(column_id)); + } else if (OB_FAIL(new_column.assign(*hard_code_column))) { + LOG_WARN("fail to assign column", KR(ret), KPC(hard_code_column)); + } else if (OB_FAIL(new_table_schema.alter_column(new_column, share::schema::ObTableSchema::CHECK_MODE_ONLINE))) { + LOG_WARN("fail to alter column", KR(ret), K(new_column)); + } + } // end for + + if (FAILEDx(ddl_operator.batch_update_system_table_columns(trans, + *orig_table_schema, new_table_schema, add_column_ids, alter_column_ids, NULL))) { + LOG_WARN("fail to batch update columns", KR(ret), K(new_table_schema)); + } + } + + if (trans.is_started()) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(trans.end(OB_SUCC(ret)))) { + LOG_WARN("trans end failed", "is_commit", OB_SUCC(ret), K(tmp_ret)); + ret = (OB_SUCC(ret)) ? tmp_ret : ret; + } + } + if (FAILEDx(publish_schema(tenant_id))) { + LOG_WARN("fail to publish schema", KR(ret), K(tenant_id)); + } + + } // end HEAP_VAR + } + return ret; +} + int ObDDLService::add_table_schema( ObTableSchema &table_schema, @@ -18521,18 +18732,30 @@ int ObDDLService::drop_inner_table(const share::schema::ObTableSchema &table_sch ObString *stmt = NULL; ObSchemaGetterGuard schema_guard; const uint64_t tenant_id = table_schema.get_tenant_id(); + const uint64_t table_id = table_schema.get_table_id(); + const ObSimpleTableSchemaV2 * table = NULL; if (OB_FAIL(check_inner_stat())) { - LOG_WARN("variable is not init"); - } else if (!is_inner_table(table_schema.get_table_id())) { + LOG_WARN("variable is not init", KR(ret), K(tenant_id), K(table_id)); + } else if (!is_inner_table(table_id)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("table not inner table", "table_id", table_schema.get_table_id(), K(ret)); + LOG_WARN("table not inner table", KR(ret), K(tenant_id), K(table_id)); } else if (OB_FAIL(get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) { - LOG_WARN("fail to get schema guard with version in inner table", K(ret), K(tenant_id)); - } else if (OB_FAIL(drop_table_in_trans(schema_guard, table_schema, false, table_schema.is_index_table(), + LOG_WARN("fail to get schema guard with version in inner table", KR(ret), K(tenant_id)); + } else if (OB_FAIL(schema_guard.get_simple_table_schema(tenant_id, table_id, table))) { + LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); + } else if (OB_ISNULL(table)) { + // bugfix: https://work.aone.alibaba-inc.com/issue/45050614 + // virtual table index may be dropped with virtual table, so here we ignore OB_TABLE_NOT_EXIST failure. + LOG_INFO("table has already been dropped, just ignore", + K(tenant_id), K(table_id), "table_name", table_schema.get_table_name()); + } else if (OB_FAIL(drop_table_in_trans(schema_guard, + table_schema, + false, + table_schema.is_index_table(), false, /* to recyclebin*/ stmt, NULL, NULL, NULL))) { - LOG_WARN("drop table in transaction failed", K(ret), K(table_schema)); + LOG_WARN("drop table in transaction failed", KR(ret), K(tenant_id), K(table_id)); } return ret; } @@ -19610,7 +19833,7 @@ int ObDDLService::create_tenant_end(const uint64_t tenant_id) MEMSET(ddl_str_buf, 0, DDL_STR_BUF_SIZE); ObString ddl_stmt_str; if (tenant_schema->is_restore()) { - palf::SCN gts; + SCN gts; int64_t pos = 0; if (OB_FAIL(get_tenant_external_consistent_ts(tenant_id, gts))) { SERVER_LOG(WARN, "failed to get_tenant_gts", KR(ret), K(tenant_id)); @@ -21674,7 +21897,8 @@ int ObDDLService::lock_tenant(const ObString &tenant_name, const bool is_lock) int ObDDLService::add_system_variable(const ObAddSysVarArg &arg) { - LOG_INFO("receive add system variable request", K(arg)); + FLOG_INFO("[UPGRADE] begin upgrade system variable", K(arg)); + int64_t start_time = ObTimeUtility::current_time(); DEBUG_SYNC(BEFORE_UPRADE_SYSTEM_VARIABLE); int ret = OB_SUCCESS; ObDDLSQLTransaction trans(schema_service_); @@ -21714,14 +21938,15 @@ int ObDDLService::add_system_variable(const ObAddSysVarArg &arg) LOG_WARN("sys var schema is null", KR(ret), K(arg)); } else if (!arg.update_sys_var_) { // case 1. add sys var, and sys var exist - if (arg.if_not_exist_) { - execute = false; + if (new_sys_var.is_equal_for_add(*old_schema)) { + // new sys var will be mocked by schema when upgrade, + // only persist new sys var when sys var is equal with old sys var. } else { - ret = OB_ERR_PARAM_DUPLICATE; + ret = OB_SCHEMA_ERROR; LOG_WARN("system variable duplicated", KR(ret), K(var_name)); } } else { - // upate sys var + // update sys var if (new_sys_var.is_equal_except_value(*old_schema)) { // case 2. new sys var is same with existed schema(except value), do nothing execute = false; @@ -21773,6 +21998,8 @@ int ObDDLService::add_system_variable(const ObAddSysVarArg &arg) LOG_WARN("publish schema failed", KR(ret)); } } + FLOG_INFO("[UPGRADE] end upgrade system variable", + KR(ret), K(arg), "cost", ObTimeUtility::current_time() - start_time); return ret; } @@ -29570,7 +29797,7 @@ int ObDDLService::add_sys_table_lob_aux( return ret; } -int ObDDLService::get_tenant_external_consistent_ts(const int64_t tenant_id, palf::SCN &scn) +int ObDDLService::get_tenant_external_consistent_ts(const int64_t tenant_id, SCN &scn) { int ret = OB_SUCCESS; const int64_t timeout_us = THIS_WORKER.is_timeout_ts_valid() ? diff --git a/src/rootserver/ob_ddl_service.h b/src/rootserver/ob_ddl_service.h index f21e753ca..0a0675e88 100644 --- a/src/rootserver/ob_ddl_service.h +++ b/src/rootserver/ob_ddl_service.h @@ -49,12 +49,9 @@ namespace obrpc class ObAccountArg; } -namespace palf -{ -class SCN; -} namespace share { +class SCN; class ObLSTableOperator; class ObAutoincrementService; class ObSplitInfo; @@ -129,16 +126,16 @@ public: // create_index_table will fill table_id and frozen_version to table_schema virtual int create_index_table(const obrpc::ObCreateIndexArg &arg, share::schema::ObTableSchema &table_schema, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, ObMySQLTransaction &sql_trans); int rebuild_index(const obrpc::ObRebuildIndexArg &arg, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, obrpc::ObAlterTableRes &res); int rebuild_index_in_trans(share::schema::ObSchemaGetterGuard &schema_guard, share::schema::ObTableSchema &table_schema, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, const ObString *ddl_stmt_str, ObMySQLTransaction *sql_trans); @@ -147,7 +144,7 @@ public: share::schema::ObTableSchema &new_table_schema, common::ObIArray &new_columns, share::schema::ObTableSchema &index_schema, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, const common::ObString *ddl_stmt_str); // check whether the foreign key related table is executing offline ddl, creating index, and constrtaint task. // And ddl should be refused if the foreign key related table is executing above ddl. @@ -158,7 +155,7 @@ public: const obrpc::ObCreateIndexArg &arg, const share::schema::ObTableSchema &table_schema, share::schema::ObTableSchema &index_schema, - const palf::SCN &frozen_scn); + const share::SCN &frozen_scn); int create_global_inner_expr_index( ObMySQLTransaction &trans, const obrpc::ObCreateIndexArg &arg, @@ -166,7 +163,7 @@ public: share::schema::ObTableSchema &new_table_schema, common::ObIArray &new_columns, share::schema::ObTableSchema &index_schema, - const palf::SCN &frozen_scn); + const share::SCN &frozen_scn); template int check_primary_zone_locality_condition( const SCHEMA &schema, @@ -206,7 +203,7 @@ public: const common::ObString &ddl_stmt_str, const share::schema::ObErrorInfo &error_info, common::ObIArray &table_schemas, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, share::schema::ObSchemaGetterGuard &schema_guard, const obrpc::ObSequenceDDLArg &sequence_ddl_arg, const uint64_t last_replay_log_id, @@ -214,10 +211,10 @@ public: ObIArray &mock_fk_parent_table_schema_array); virtual int create_table_like(const obrpc::ObCreateTableLikeArg &arg, - const palf::SCN &frozen_scn); + const share::SCN &frozen_scn); virtual int alter_table(obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, obrpc::ObAlterTableRes &res); virtual int drop_table(const obrpc::ObDropTableArg &drop_table_arg, const obrpc::ObDDLRes &res); int check_table_exists(const uint64_t tenant_id, @@ -227,6 +224,7 @@ public: const share::schema::ObTableSchema **table_schema); virtual int update_index_status(const obrpc::ObUpdateIndexStatusArg &arg); + int upgrade_table_schema(const obrpc::ObUpgradeTableSchemaArg &arg); virtual int add_table_schema(share::schema::ObTableSchema &table_schema, share::schema::ObSchemaGetterGuard &schema_guard); virtual int drop_inner_table(const share::schema::ObTableSchema &table_schema); @@ -260,7 +258,7 @@ public: virtual int try_format_partition_schema(share::schema::ObPartitionSchema &table_schema); virtual int generate_schema(const obrpc::ObCreateTableArg &arg, share::schema::ObTableSchema &schema, - const palf::SCN &frozen_scn); + const share::SCN &frozen_scn); int create_index_tablet(const ObTableSchema &index_schema, ObMySQLTransaction &trans, share::schema::ObSchemaGetterGuard &schema_guard); @@ -268,7 +266,7 @@ public: const share::schema::ObTableSchema &orgin_table_schema, share::schema::ObTableSchema &new_table_schema, share::schema::ObSchemaGetterGuard &schema_guard, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, ObDDLOperator &ddl_operator, ObMySQLTransaction &trans, common::ObArenaAllocator &allocator, @@ -285,7 +283,7 @@ public: int alter_table_column( const share::schema::ObTableSchema &origin_table_schema, const share::schema::AlterTableSchema & alter_table_schema, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, share::schema::ObTableSchema &new_table_schema, obrpc::ObAlterTableArg &alter_table_arg, share::schema::ObSchemaGetterGuard &schema_guard, @@ -415,7 +413,7 @@ public: ObMySQLTransaction &trans); virtual int truncate_table(const obrpc::ObTruncateTableArg &arg, const obrpc::ObDDLRes &ddl_res, - const palf::SCN &frozen_scn); + const share::SCN &frozen_scn); int drop_not_null_cst_in_column_flag(const ObTableSchema &orig_table_schema, const AlterTableSchema &alter_table_schema, ObTableSchema &new_table_schema, @@ -496,7 +494,7 @@ public: * @param [out] index_ids: new index table id */ int rebuild_hidden_table_index(obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, common::ObSArray &index_ids); /** * This function is called by the storage layer in the fourth stage of offline ddl @@ -558,7 +556,7 @@ public: const common::ObSArray &index_columns, const share::schema::ObTableSchema &orgin_table_schema, share::schema::ObTableSchema &new_table_schema, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, share::schema::ObSchemaGetterGuard &schema_guard, ObDDLOperator &ddl_operator, common::ObMySQLTransaction &trans, @@ -922,7 +920,7 @@ public: ObSchemaGetterGuard &schema_guard, ObSArray &table_schemas, common::ObMySQLTransaction &trans, - const palf::SCN &frozen_scn); + const share::SCN &frozen_scn); int swap_orig_and_hidden_table_state( obrpc::ObAlterTableArg &alter_table_arg, const ObTableSchema &orig_table_schema, @@ -933,7 +931,7 @@ public: ObSArray &new_table_schemas); int remap_index_tablets_and_take_effect( obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn); + const share::SCN &frozen_scn); int update_autoinc_schema(obrpc::ObAlterTableArg &alter_table_arg); int build_aux_lob_table_schema_if_need(ObTableSchema &data_table_schema, ObIArray &table_schemas); @@ -1024,7 +1022,7 @@ private: const uint64_t tenant_id, const share::schema::ObTableSchema &orig_table_schema, ObDDLOperator &ddl_operator, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, ObMySQLTransaction &trans); int fill_interval_info_for_set_interval(const ObTableSchema &orig_table_schema, ObTableSchema &new_table_schema, @@ -1063,7 +1061,7 @@ private: template int set_default_tablegroup_id(SCHEMA &schema); int create_table_in_trans(share::schema::ObTableSchema &table_schema, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, const common::ObString *ddl_stmt_str, ObMySQLTransaction *sql_trans, share::schema::ObSchemaGetterGuard &schema_guard); @@ -1162,7 +1160,7 @@ private: const common::ObString &ddl_stmt_str, const share::schema::ObErrorInfo &error_info, common::ObIArray &table_schemas, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, const obrpc::ObSequenceDDLArg &sequence_ddl_arg, const uint64_t last_replay_log_id, const common::ObIArray *dep_infos, @@ -1179,7 +1177,7 @@ private: const share::schema::ObSchemaOperationType operation_type, const common::ObString &ddl_stmt_str); int alter_table_in_trans(obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, obrpc::ObAlterTableRes &res); int need_modify_not_null_constraint_validate(const obrpc::ObAlterTableArg &alter_table_arg, bool &is_add_not_null_col, @@ -1194,7 +1192,7 @@ private: int check_ddl_with_primary_key_operation(const obrpc::ObAlterTableArg &alter_table_arg, bool &with_primary_key_operation); int do_offline_ddl_in_trans(obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, obrpc::ObAlterTableRes &res); int gen_new_index_table_name( const common::ObString &orig_index_table_name, @@ -1219,7 +1217,7 @@ private: const obrpc::ObSequenceDDLArg *sequence_ddl_arg, const bool bind_tablets, share::schema::ObSchemaGetterGuard &schema_guard, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, ObDDLOperator &ddl_operator, common::ObMySQLTransaction &trans, common::ObIAllocator &allocator); @@ -1254,7 +1252,7 @@ private: common::ObIAllocator &allocator); int prepare_hidden_table_schema( const share::schema::ObTableSchema &orig_table_schema, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, common::ObIAllocator &allocator, share::schema::ObTableSchema &hidden_table_schema); int rebuild_hidden_table_priv( @@ -1314,7 +1312,7 @@ private: const share::schema::ObTableSchema &orgin_table_schema, share::schema::ObTableSchema &new_table_schema, share::schema::ObSchemaGetterGuard &schema_guard, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, ObDDLOperator &ddl_operator, common::ObMySQLTransaction &trans, common::ObArenaAllocator &allocator); @@ -1322,14 +1320,14 @@ private: const share::schema::ObTableSchema &orgin_table_schema, share::schema::ObTableSchema &new_table_schema, share::schema::ObSchemaGetterGuard &schema_guard, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, ObDDLOperator &ddl_operator, common::ObMySQLTransaction &trans); int convert_to_character(obrpc::ObAlterTableArg &alter_table_arg, const share::schema::ObTableSchema &orgin_table_schema, share::schema::ObTableSchema &new_table_schema, share::schema::ObSchemaGetterGuard &schema_guard, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, ObDDLOperator &ddl_operator, common::ObMySQLTransaction &trans); int check_alter_table_constraint( @@ -1402,7 +1400,7 @@ private: const share::schema::ObTableSchema &orig_table_schema, const ObTableSchema &hidden_table_schema, ObSchemaGetterGuard &schema_guard, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, ObSArray &new_table_schemas, ObSArray &index_ids); int check_index_table_need_rebuild( @@ -1417,7 +1415,7 @@ private: const common::ObIArray &drop_cols_id_arr, const share::ObColumnNameMap &col_name_map, const common::ObTimeZoneInfo &tz_info, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, common::ObIAllocator &allocator, common::ObSArray &new_table_schemas, common::ObSArray &index_ids); @@ -1428,7 +1426,7 @@ private: share::schema::ObTableSchema &new_table_schema, share::schema::ObTableSchema &index_schema); int alter_table_sess_active_time_in_trans(obrpc::ObAlterTableArg &alter_table_arg, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, obrpc::ObAlterTableRes &res); int truncate_table_in_trans(const obrpc::ObTruncateTableArg &arg, const share::schema::ObTableSchema &orig_table_schema, @@ -1466,7 +1464,7 @@ private: const bool is_oracle_mode, const share::schema::ObTableSchema &table_schema, common::ObIAllocator &allocator, - share::schema::AlterColumnSchema &alter_column_schema); + share::schema::ObColumnSchemaV2 &alter_column_schema); int resolve_orig_default_value(share::schema::ObColumnSchemaV2 &column_schema, const common::ObTimeZoneInfoWrap &tz_info_wrap, const common::ObString *nls_formats, @@ -1691,7 +1689,7 @@ public: common::ObIArray &zone_region_list, const common::ObIArray &zone_list); - int check_all_server_frozen_scn(const palf::SCN &frozen_scn); + int check_all_server_frozen_scn(const share::SCN &frozen_scn); int handle_security_audit(const obrpc::ObSecurityAuditArg &arg); static int check_and_get_object_name(share::schema::ObSchemaGetterGuard &schema_guard, @@ -1970,7 +1968,7 @@ private: const int64_t session_id, const share::schema::ObTableType table_type_, share::schema::ObSchemaService &schema_service, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, common::ObIArray &new_scheams, common::ObArenaAllocator &allocator, const uint64_t define_user_id); @@ -2151,7 +2149,7 @@ private: share::schema::AlterTableSchema &inc_table_schema, share::schema::AlterTableSchema &del_table_schema); //get gts value, return OB_STATE_NOT_MATCH when is not external consistent - int get_tenant_external_consistent_ts(const int64_t tenant_id, palf::SCN &scn); + int get_tenant_external_consistent_ts(const int64_t tenant_id, share::SCN &scn); int get_part_by_part_id( const share::schema::ObPartitionSchema &partition_schema, const int64_t part_id, @@ -2159,6 +2157,17 @@ private: int check_table_pk(const share::schema::ObTableSchema &orig_table_schema); int check_can_convert_to_character(const share::schema::ObColumnSchemaV2 &col_schema, bool &can_convert); int clean_global_context(const ObContextSchema &context_schema); + + int get_hard_code_system_table_schema_( + const uint64_t tenant_id, + const uint64_t table_id, + share::schema::ObTableSchema &hard_code_schema); + int create_system_table_( + share::schema::ObSchemaGetterGuard &schema_guard, + const share::schema::ObTableSchema &hard_code_schema); + int alter_system_table_column_( + share::schema::ObSchemaGetterGuard &schema_guard, + const share::schema::ObTableSchema &hard_code_schema); private: bool inited_; volatile bool stopped_; diff --git a/src/rootserver/ob_index_builder.cpp b/src/rootserver/ob_index_builder.cpp index 2ca858305..381655fee 100644 --- a/src/rootserver/ob_index_builder.cpp +++ b/src/rootserver/ob_index_builder.cpp @@ -44,7 +44,7 @@ #include #include "rootserver/ddl_task/ob_ddl_scheduler.h" #include "rootserver/ddl_task/ob_ddl_task.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -69,7 +69,7 @@ ObIndexBuilder::~ObIndexBuilder() int ObIndexBuilder::create_index( const ObCreateIndexArg &arg, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, obrpc::ObAlterTableRes &res) { int ret = OB_SUCCESS; @@ -242,7 +242,7 @@ int ObIndexBuilder::do_create_global_index( share::schema::ObSchemaGetterGuard &schema_guard, const obrpc::ObCreateIndexArg &arg, const share::schema::ObTableSchema &table_schema, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, obrpc::ObAlterTableRes &res) { int ret = OB_SUCCESS; @@ -388,7 +388,7 @@ int ObIndexBuilder::do_create_local_index( share::schema::ObSchemaGetterGuard &schema_guard, const obrpc::ObCreateIndexArg &create_index_arg, const share::schema::ObTableSchema &table_schema, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, obrpc::ObAlterTableRes &res) { int ret = OB_SUCCESS; @@ -486,7 +486,7 @@ int ObIndexBuilder::do_create_local_index( // and table_name, which will used for getting data table schema in generate_schema int ObIndexBuilder::do_create_index( const ObCreateIndexArg &arg, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, obrpc::ObAlterTableRes &res) { int ret = OB_SUCCESS; @@ -578,7 +578,7 @@ int ObIndexBuilder::do_create_index( */ int ObIndexBuilder::generate_schema( const ObCreateIndexArg &arg, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObTableSchema &data_schema, const bool global_index_without_column_info, ObTableSchema &schema) @@ -754,7 +754,7 @@ int ObIndexBuilder::generate_schema( } int ObIndexBuilder::set_basic_infos(const ObCreateIndexArg &arg, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, const ObTableSchema &data_schema, ObTableSchema &schema) { diff --git a/src/rootserver/ob_index_builder.h b/src/rootserver/ob_index_builder.h index 7f61bfa33..119bb3d9d 100644 --- a/src/rootserver/ob_index_builder.h +++ b/src/rootserver/ob_index_builder.h @@ -26,13 +26,10 @@ namespace common class ObMySQLProxy; } -namespace palf -{ -class SCN; -} namespace share { class ObReplicaFilterHolder; +class SCN; namespace schema { class ObMultiVersionSchemaService; @@ -59,7 +56,7 @@ public: virtual ~ObIndexBuilder(); int create_index(const obrpc::ObCreateIndexArg &arg, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, obrpc::ObAlterTableRes &res); int drop_index(const obrpc::ObDropIndexArg &arg, obrpc::ObDropIndexRes &res); @@ -67,22 +64,22 @@ public: // if not all index table updated return OB_EAGAIN. int do_create_index( const obrpc::ObCreateIndexArg &arg, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, obrpc::ObAlterTableRes &res); int do_create_global_index( share::schema::ObSchemaGetterGuard &schema_guard, const obrpc::ObCreateIndexArg &arg, const share::schema::ObTableSchema &table_schema, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, obrpc::ObAlterTableRes &res); int do_create_local_index( share::schema::ObSchemaGetterGuard &schema_guard, const obrpc::ObCreateIndexArg &arg, const share::schema::ObTableSchema &table_schema, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, obrpc::ObAlterTableRes &res); int generate_schema(const obrpc::ObCreateIndexArg &arg, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, share::schema::ObTableSchema &data_schema, const bool global_index_without_column_info, share::schema::ObTableSchema &index_schema); @@ -113,7 +110,7 @@ private: }; int set_basic_infos(const obrpc::ObCreateIndexArg &arg, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, const share::schema::ObTableSchema &data_schema, share::schema::ObTableSchema &schema); int set_index_table_columns(const obrpc::ObCreateIndexArg &arg, diff --git a/src/rootserver/ob_primary_ls_service.cpp b/src/rootserver/ob_primary_ls_service.cpp index dd4ffe4b9..509f885fc 100755 --- a/src/rootserver/ob_primary_ls_service.cpp +++ b/src/rootserver/ob_primary_ls_service.cpp @@ -2049,7 +2049,7 @@ void ObPrimaryLSService::do_work() int64_t idle_time_us = 100 * 1000L; int tmp_ret = OB_SUCCESS; while (!has_set_stop()) { - idle_time_us = 10 * 1000 * 1000L; + idle_time_us = is_meta_tenant(tenant_id_) ? 100 * 1000L : 10 * 1000 * 1000L; { ObCurTraceId::init(GCONF.self_addr_); share::schema::ObSchemaGetterGuard schema_guard; diff --git a/src/rootserver/ob_primary_ls_service.h b/src/rootserver/ob_primary_ls_service.h index 0ce6c6305..3f201194d 100644 --- a/src/rootserver/ob_primary_ls_service.h +++ b/src/rootserver/ob_primary_ls_service.h @@ -14,7 +14,7 @@ #define OCEANBASE_ROOTSERVER_OB_PRIMARY_LS_SERVICE_H #include "lib/thread/ob_reentrant_thread.h"//ObRsReentrantThread #include "logservice/ob_log_base_type.h" -#include "logservice/palf/scn.h"//SCN +#include "share/scn.h"//SCN #include "share/ls/ob_ls_status_operator.h"//ObLSStatusOperator #include "share/ls/ob_ls_operator.h" //ObLSAttr #include "share/ob_thread_mgr.h" //OBTGDefIDEnum @@ -165,7 +165,7 @@ public: //for recovery tenant, create new ls according to ls_id and ls_group_id int create_new_ls_for_recovery(const share::ObLSID &ls_id, const uint64_t ls_group_id, - const palf::SCN &create_scn, + const share::SCN &create_scn, common::ObMySQLTransaction &trans); //for recovery tenant, if ls is in creating in __all_ls_status, create the ls int process_ls_stats_for_recovery(); @@ -179,7 +179,7 @@ public: @param[in] create_ls_with_palf: restore create init ls @param[in] palf_base_info : palf base info */ - int create_ls_with_palf(const share::ObLSStatusInfo &info, const palf::SCN &create_scn, + int create_ls_with_palf(const share::ObLSStatusInfo &info, const share::SCN &create_scn, const bool create_ls_with_palf, const palf::PalfBaseInfo &palf_base_info); @@ -224,7 +224,7 @@ private: int drop_ls_(const share::ObLSStatusInfo &ls_info); int do_drop_ls_(const share::ObLSStatusInfo &ls_info); int do_tenant_drop_ls_(const share::ObLSStatusInfo &ls_info); - int do_create_ls_(const share::ObLSStatusInfo &info, const palf::SCN &create_scn); + int do_create_ls_(const share::ObLSStatusInfo &info, const share::SCN &create_scn); int sys_ls_tenant_drop_(const share::ObLSStatusInfo &info); int check_sys_ls_can_offline_(bool &can_offline); int check_ls_empty_(const share::ObLSStatusInfo &info, bool &empty); @@ -320,9 +320,9 @@ public: virtual void do_work() override; public: - virtual palf::SCN get_rec_scn() override { return palf::SCN::max_scn();} - virtual int flush(palf::SCN &scn) override { return OB_SUCCESS; } - int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const palf::SCN &scn) override + virtual share::SCN get_rec_scn() override { return share::SCN::max_scn();} + virtual int flush(share::SCN &scn) override { return OB_SUCCESS; } + int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn) override { UNUSED(buffer); UNUSED(nbytes); diff --git a/src/rootserver/ob_recovery_ls_service.cpp b/src/rootserver/ob_recovery_ls_service.cpp index e2f0b8973..45167c1bd 100644 --- a/src/rootserver/ob_recovery_ls_service.cpp +++ b/src/rootserver/ob_recovery_ls_service.cpp @@ -20,7 +20,7 @@ #include "logservice/ob_log_handler.h" //ObLogHandler #include "logservice/palf/log_entry.h" //LogEntry #include "logservice/palf/log_define.h" -#include "logservice/palf/scn.h"//SCN +#include "share/scn.h"//SCN #include "logservice/ob_garbage_collector.h"//ObGCLSLog #include "logservice/restoreservice/ob_log_restore_handler.h"//ObLogRestoreHandler #include "observer/ob_server_struct.h" //GCTX diff --git a/src/rootserver/ob_recovery_ls_service.h b/src/rootserver/ob_recovery_ls_service.h index 66ae90d52..3f6687599 100644 --- a/src/rootserver/ob_recovery_ls_service.h +++ b/src/rootserver/ob_recovery_ls_service.h @@ -36,6 +36,7 @@ namespace share class ObLSTableOperator; struct ObLSAttr; struct ObLSRecoveryStat; +class SCN; namespace schema { class ObMultiVersionSchemaService; @@ -47,10 +48,6 @@ namespace logservice class ObLogHandler; class ObGCLSLog; } -namespace palf -{ -class SCN; -} namespace transaction { class ObTxLogBlock; @@ -75,28 +72,28 @@ public: virtual void do_work() override; private: //get log iterator by start_scn - int seek_log_iterator_(const palf::SCN &syn_scn, + int seek_log_iterator_(const share::SCN &syn_scn, palf::PalfBufferIterator &iterator); - int process_ls_log_(const palf::SCN &start_scn,palf::PalfBufferIterator &iterator); + int process_ls_log_(const share::SCN &start_scn,palf::PalfBufferIterator &iterator); int process_gc_log_(logservice::ObGCLSLog &gc_log, - const palf::SCN &syn_scn); + const share::SCN &syn_scn); int process_ls_tx_log_(transaction::ObTxLogBlock &tx_log, - const palf::SCN &syn_scn); + const share::SCN &syn_scn); int process_ls_operator_(const share::ObLSAttr &ls_attr, - const palf::SCN &syn_scn); + const share::SCN &syn_scn); int create_new_ls_(const share::ObLSAttr &ls_attr, - const palf::SCN &syn_scn, + const share::SCN &syn_scn, common::ObMySQLTransaction &trans); int process_recovery_ls_manager(); - int construct_ls_recovery_stat(const palf::SCN &syn_scn, + int construct_ls_recovery_stat(const share::SCN &syn_scn, share::ObLSRecoveryStat &ls_stat); //wait other ls is larger than sycn ts - int check_valid_to_operator_ls_(const palf::SCN &syn_scn); + int check_valid_to_operator_ls_(const share::SCN &syn_scn); int check_can_do_recovery_(); //check restore finish and update sync scn to recovery_unitl_scn int update_sys_ls_restore_finish_(); //readable scn need report - int report_sys_ls_recovery_stat_(const palf::SCN &sync_scn); + int report_sys_ls_recovery_stat_(const share::SCN &sync_scn); private: bool inited_; uint64_t tenant_id_; diff --git a/src/rootserver/ob_root_inspection.cpp b/src/rootserver/ob_root_inspection.cpp index c378b8669..a8c550727 100644 --- a/src/rootserver/ob_root_inspection.cpp +++ b/src/rootserver/ob_root_inspection.cpp @@ -726,7 +726,7 @@ int ObRootInspection::check_all() // check sys schema tmp = OB_SUCCESS; - if (OB_SUCCESS != (tmp = check_sys_table_schemas())) { + if (OB_SUCCESS != (tmp = check_sys_table_schemas_())) { LOG_WARN("check_sys_table_schemas failed", K(tmp)); ret = (OB_SUCCESS == ret) ? tmp : ret; } @@ -976,51 +976,6 @@ int ObRootInspection::check_and_insert_sys_params(uint64_t tenant_id, LOG_WARN("some sys var exist in hard code, but does not exist in inner table, " "they will be inserted into table", K(table_name), K(miss_names)); } - ObSqlString sql; - ObSysParam sys_param; - obrpc::ObAddSysVarArg arg; - arg.exec_tenant_id_ = tenant_id; - const ObZone global_zone = ""; - for (int64_t i = 0; OB_SUCC(ret) && i < miss_names.count(); ++i) { - sql.reset(); - sys_param.reset(); - arg.sysvar_.reset(); - arg.if_not_exist_ = true; - arg.sysvar_.set_tenant_id(tenant_id); - int64_t var_store_idx = OB_INVALID_INDEX; - if (OB_ISNULL(rpc_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("common rpc proxy is null"); - } else if (OB_FAIL(check_cancel())) { - LOG_WARN("check_cancel failed", K(ret)); - } else if (OB_FAIL(ObSysVarFactory::calc_sys_var_store_idx_by_name( - ObString(miss_names.at(i).ptr()), var_store_idx))) { - LOG_WARN("fail to calc sys var store idx by name", K(ret), K(miss_names.at(i).ptr())); - } else if (false == ObSysVarFactory::is_valid_sys_var_store_idx(var_store_idx)) { - ret = OB_SCHEMA_ERROR; - LOG_WARN("calc sys var store idx success but store_idx is invalid", K(ret), K(var_store_idx)); - } else { - const ObString &name = ObSysVariables::get_name(var_store_idx); - const ObObjType &type = ObSysVariables::get_type(var_store_idx); - const ObString &value = ObSysVariables::get_value(var_store_idx); - const ObString &min = ObSysVariables::get_min(var_store_idx); - const ObString &max = ObSysVariables::get_max(var_store_idx); - const ObString &info = ObSysVariables::get_info(var_store_idx); - const int64_t flag = ObSysVariables::get_flags(var_store_idx); - if (OB_FAIL(sys_param.init(tenant_id, global_zone, name.ptr(), type, - value.ptr(), min.ptr(), max.ptr(), info.ptr(), flag))) { - LOG_WARN("sys_param init failed", K(tenant_id), K(name), K(type), K(value), - K(min), K(max), K(info), K(flag), K(ret)); - } else if (!sys_param.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("sys param is invalid", K(sys_param), K(ret)); - } else if (OB_FAIL(ObSchemaUtils::convert_sys_param_to_sysvar_schema(sys_param, arg.sysvar_))) { - LOG_WARN("convert sys param to sysvar schema failed", K(ret)); - } else if (OB_FAIL(rpc_proxy_->add_system_variable(arg))) { - LOG_WARN("add system variable failed", K(ret)); - } - } - } } return ret; } @@ -1190,7 +1145,7 @@ int ObRootInspection::calc_diff_names(const uint64_t tenant_id, return ret; } -int ObRootInspection::check_sys_table_schemas() +int ObRootInspection::check_sys_table_schemas_() { int ret = OB_SUCCESS; ObArray tenant_ids; @@ -1337,7 +1292,7 @@ int ObRootInspection::check_table_schema(const ObTableSchema &hard_code_table, || !inner_table.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid table_schema", K(hard_code_table), K(inner_table), K(ret)); - } else if (OB_FAIL(check_table_options(inner_table, hard_code_table))) { + } else if (OB_FAIL(check_table_options_(inner_table, hard_code_table))) { LOG_WARN("check_table_options failed", "table_id", hard_code_table.get_table_id(), K(ret)); } else { if (hard_code_table.get_column_count() != inner_table.get_column_count()) { @@ -1361,7 +1316,7 @@ int ObRootInspection::check_table_schema(const ObTableSchema &hard_code_table, hard_code_column->get_column_name(), K(ret)); } else { const bool ignore_column_id = is_virtual_table(hard_code_table.get_table_id()); - if (OB_FAIL(check_column_schema(hard_code_table.get_table_name(), + if (OB_FAIL(check_column_schema_(hard_code_table.get_table_name(), *column, *hard_code_column, ignore_column_id))) { LOG_WARN("column schema mismatch with hard code column schema", "table_name",inner_table.get_table_name(), "column", *column, @@ -1377,6 +1332,110 @@ int ObRootInspection::check_table_schema(const ObTableSchema &hard_code_table, return ret; } +int ObRootInspection::check_and_get_system_table_column_diff( + const share::schema::ObTableSchema &table_schema, + const share::schema::ObTableSchema &hard_code_schema, + common::ObIArray &add_column_ids, + common::ObIArray &alter_column_ids) +{ + int ret = OB_SUCCESS; + add_column_ids.reset(); + alter_column_ids.reset(); + if (!table_schema.is_valid() || !hard_code_schema.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid table_schema", KR(ret), K(table_schema), K(hard_code_schema)); + } else if (table_schema.get_tenant_id() != hard_code_schema.get_tenant_id() + || table_schema.get_table_id() != hard_code_schema.get_table_id() + || 0 != table_schema.get_table_name_str().compare(hard_code_schema.get_table_name_str()) + || !is_system_table(table_schema.get_table_id())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid table_schema", KR(ret), + "tenant_id", table_schema.get_tenant_id(), + "table_id", table_schema.get_table_id(), + "table_name", table_schema.get_table_name(), + "hard_code_tenant_id", hard_code_schema.get_tenant_id(), + "hard_code_table_id", hard_code_schema.get_table_id(), + "hard_code_table_name", hard_code_schema.get_table_name()); + } else { + const uint64_t tenant_id = table_schema.get_tenant_id(); + const uint64_t table_id = table_schema.get_table_id(); + const ObColumnSchemaV2 *column = NULL; + const ObColumnSchemaV2 *hard_code_column = NULL; + ObColumnSchemaV2 tmp_column; // check_column_can_be_altered_online() may change dst_column, is ugly. + bool ignore_column_id = false; + + // case 1. check if columns should be dropped. + // case 2. check if column can be altered online. + for (int64_t i = 0; OB_SUCC(ret) && i < table_schema.get_column_count(); i++) { + column = table_schema.get_column_schema_by_idx(i); + if (OB_ISNULL(column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column schema is null", KR(ret), K(tenant_id), K(table_id), K(i)); + } else if (OB_ISNULL(hard_code_column = hard_code_schema.get_column_schema(column->get_column_id()))) { + ret = OB_NOT_SUPPORTED; // case 1 + LOG_WARN("can't drop system table's column", KR(ret), + K(tenant_id), K(table_id), + "table_name", table_schema.get_table_name(), + "column_id", column->get_column_id(), + "column_name", column->get_column_name()); + } else { + // case 2 + int tmp_ret = check_column_schema_(table_schema.get_table_name_str(), + *column, + *hard_code_column, + ignore_column_id); + if (OB_SUCCESS == tmp_ret) { + // not changed + } else if (OB_SCHEMA_ERROR != tmp_ret) { + ret = tmp_ret; + LOG_WARN("fail to check column schema", KR(ret), + K(tenant_id), K(table_id), KPC(column), KPC(hard_code_column)); + } else if (OB_FAIL(tmp_column.assign(*hard_code_column))) { + LOG_WARN("fail to assign hard code column schema", KR(ret), + K(tenant_id), K(table_id), "column_id", hard_code_column->get_column_id()); + } else if (OB_FAIL(table_schema.check_column_can_be_altered_online(column, &tmp_column))) { + LOG_WARN("fail to check alter column online", KR(ret), + K(tenant_id), K(table_id), + "table_name", table_schema.get_table_name(), + "column_id", column->get_column_id(), + "column_name", column->get_column_name()); + } else if (OB_FAIL(alter_column_ids.push_back(column->get_column_id()))) { + LOG_WARN("fail to push back column_id", KR(ret), K(tenant_id), K(table_id), + "column_id", column->get_column_id()); + } + } + } // end for + + // case 3: check if columns should be added. + for (int64_t i = 0; OB_SUCC(ret) && i < hard_code_schema.get_column_count(); i++) { + hard_code_column = hard_code_schema.get_column_schema_by_idx(i); + if (OB_ISNULL(hard_code_column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column schema is null", KR(ret), K(tenant_id), K(table_id), K(i)); + } else if (OB_NOT_NULL(column = table_schema.get_column_schema(hard_code_column->get_column_id()))) { + // column exist, just skip + } else { + const uint64_t hard_code_column_id = hard_code_column->get_column_id(); + const ObColumnSchemaV2 *last_column = NULL; + if (table_schema.get_column_count() <= 0 + || OB_ISNULL(last_column = table_schema.get_column_schema_by_idx( + table_schema.get_column_count() - 1))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid column count or column", KR(ret), K(table_schema)); + } else if (table_schema.get_max_used_column_id() >= hard_code_column_id + || last_column->get_column_id() >= hard_code_column_id) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("column should be added at last", KR(ret), KPC(hard_code_column), K(table_schema)); + } else if (OB_FAIL(add_column_ids.push_back(hard_code_column_id))) { + LOG_WARN("fail to push back column_id", KR(ret), K(tenant_id), K(table_id), + "column_id", hard_code_column_id); + } + } + } // end for + } + return ret; +} + bool ObRootInspection::check_str_with_lower_case_(const ObString &str) { bool bret = false; @@ -1462,8 +1521,8 @@ int ObRootInspection::check_sys_view_( return ret; } -int ObRootInspection::check_table_options(const ObTableSchema &table, - const ObTableSchema &hard_code_table) +int ObRootInspection::check_table_options_(const ObTableSchema &table, + const ObTableSchema &hard_code_table) { int ret = OB_SUCCESS; if (!table.is_valid() || !hard_code_table.is_valid()) { @@ -1615,23 +1674,23 @@ int ObRootInspection::check_table_options(const ObTableSchema &table, return ret; } -int ObRootInspection::check_column_schema(const ObString &table_name, - const ObColumnSchemaV2 &column, - const ObColumnSchemaV2 &hard_code_column, - const bool ignore_column_id) +int ObRootInspection::check_column_schema_(const ObString &table_name, + const ObColumnSchemaV2 &column, + const ObColumnSchemaV2 &hard_code_column, + const bool ignore_column_id) { int ret = OB_SUCCESS; if (table_name.empty() || !column.is_valid() || !hard_code_column.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("table_name is empty or invalid column or invalid hard_code_column", - K(table_name), K(column), K(hard_code_column), K(ret)); + KR(ret), K(table_name), K(column), K(hard_code_column)); } else { #define CMP_COLUMN_ATTR(attr) \ if (OB_SUCC(ret)) { \ if (column.get_##attr() != hard_code_column.get_##attr()) { \ ret = OB_SCHEMA_ERROR; \ - LOG_WARN(#attr " mismatch", K(table_name), "column_name", column.get_column_name(), \ - "in_memory", column.get_##attr(), "hard_code", hard_code_column.get_##attr(), K(ret)); \ + LOG_WARN(#attr " mismatch", KR(ret), K(table_name), "column_name", column.get_column_name(), \ + "in_memory", column.get_##attr(), "hard_code", hard_code_column.get_##attr()); \ } \ } @@ -1639,10 +1698,18 @@ int ObRootInspection::check_column_schema(const ObString &table_name, if (OB_SUCC(ret)) { \ if (column.is_##attr() != hard_code_column.is_##attr()) { \ ret = OB_SCHEMA_ERROR; \ - LOG_WARN(#attr " mismatch", K(table_name), "column_name", column.get_column_name(), \ - "in_memory", column.is_##attr(), "hard_code", hard_code_column.is_##attr(), K(ret)); \ + LOG_WARN(#attr " mismatch", KR(ret), K(table_name), "column_name", column.get_column_name(), \ + "in_memory", column.is_##attr(), "hard_code", hard_code_column.is_##attr()); \ } \ } + if (OB_SUCC(ret)) { + if (column.get_column_name_str() != hard_code_column.get_column_name_str()) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("column_name mismatch", KR(ret), K(table_name), + "in_memory", column.get_column_name(), + "hard_code", hard_code_column.get_column_name()); + } + } if (!ignore_column_id) { CMP_COLUMN_ATTR(column_id); diff --git a/src/rootserver/ob_root_inspection.h b/src/rootserver/ob_root_inspection.h index 5510295c9..06d601c60 100644 --- a/src/rootserver/ob_root_inspection.h +++ b/src/rootserver/ob_root_inspection.h @@ -168,18 +168,26 @@ public: // return OB_SCHEMA_ERROR for table schema mismatch virtual int check_table_schema(const uint64_t tenant_id, const share::schema::ObTableSchema &hard_code_table); - // TODO:(yanmu.ztl) standby cluster won't sync sys table schema any more. To be removed. + static int check_table_schema(const share::schema::ObTableSchema &hard_code_table, const share::schema::ObTableSchema &inner_table); - // TODO:(yanmu.ztl) standby cluster wont't sync sys table schema any more. Should be private. - int check_sys_table_schemas(); + + // For system tables, check and get column schemas' difference + // between table schema in memory and hard code table schema. + // 1. Drop column: Not supported. + // 2. Add column: Can only add columns at last. + // 3. Alter column: Can only alter columns online. + static int check_and_get_system_table_column_diff( + const share::schema::ObTableSchema &table_schema, + const share::schema::ObTableSchema &hard_code_schema, + common::ObIArray &add_column_ids, + common::ObIArray &alter_column_ids); private: static const int64_t NAME_BUF_LEN = 64; typedef common::ObFixedLengthString Name; int check_zone(); int check_sys_stat(); int check_sys_param(); - int check_sys_table_schemas_(const uint64_t tenant_id); template int get_names(const common::ObDList &list, common::ObIArray &names); @@ -199,12 +207,15 @@ private: common::ObIArray &fetch_names, /* data from inner table*/ common::ObIArray &extra_names, /* inner table more than hard code*/ common::ObIArray &miss_names /* inner table less than hard code*/); - static int check_table_options(const share::schema::ObTableSchema &table, - const share::schema::ObTableSchema &hard_code_table); - static int check_column_schema(const common::ObString &table_name, - const share::schema::ObColumnSchemaV2 &column, - const share::schema::ObColumnSchemaV2 &hard_code_column, - const bool ignore_column_id); + + int check_sys_table_schemas_(); + int check_sys_table_schemas_(const uint64_t tenant_id); + static int check_table_options_(const share::schema::ObTableSchema &table, + const share::schema::ObTableSchema &hard_code_table); + static int check_column_schema_(const common::ObString &table_name, + const share::schema::ObColumnSchemaV2 &column, + const share::schema::ObColumnSchemaV2 &hard_code_column, + const bool ignore_column_id); bool check_str_with_lower_case_(const ObString &str); int check_sys_view_(const uint64_t tenant_id, diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index dc98a9aa0..deb2398b8 100644 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -92,7 +92,7 @@ #include "storage/ob_file_system_router.h" #include "rootserver/freeze/ob_major_freeze_helper.h" #include "share/restore/ob_physical_restore_table_operator.h"//ObPhysicalRestoreTableOperator -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -1429,20 +1429,22 @@ int ObRootService::submit_offline_server_task(const common::ObAddr &server) return ret; } -int ObRootService::submit_upgrade_task(const int64_t version) +int ObRootService::submit_upgrade_task( + const obrpc::ObUpgradeJobArg::Action action, + const int64_t version) { int ret = OB_SUCCESS; - ObUpgradeTask task(upgrade_executor_, version); + ObUpgradeTask task(upgrade_executor_, action, version); task.set_retry_times(0); //not repeat if (!inited_) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); } else if (OB_FAIL(upgrade_executor_.can_execute())) { - LOG_WARN("can't run task now", KR(ret), K(version)); + LOG_WARN("can't run task now", KR(ret), K(action), K(version)); } else if (OB_FAIL(task_queue_.add_async_task(task))) { - LOG_WARN("submit upgrade task fail", KR(ret), K(version)); + LOG_WARN("submit upgrade task fail", KR(ret), K(action), K(version)); } else { - LOG_INFO("submit upgrade task success", KR(ret), K(version)); + LOG_INFO("submit upgrade task success", KR(ret), K(action), K(version)); } return ret; } @@ -2918,7 +2920,7 @@ int ObRootService::create_table(const ObCreateTableArg &arg, ObCreateTableRes &r LOG_WARN("invalid arg", K(arg), K(ret)); } else { ObArray table_schemas; - palf::SCN frozen_scn; + SCN frozen_scn; ObSchemaGetterGuard schema_guard; const ObDatabaseSchema *db_schema = NULL; schema_guard.set_session_id(arg.schema_.get_session_id()); @@ -3670,7 +3672,7 @@ int ObRootService::execute_ddl_task(const obrpc::ObAlterTableArg &arg, { LOG_DEBUG("receive execute ddl task arg", K(arg)); int ret = OB_SUCCESS; - palf::SCN frozen_scn; + SCN frozen_scn; if (!inited_) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); @@ -3856,7 +3858,7 @@ int ObRootService::alter_table(const obrpc::ObAlterTableArg &arg, obrpc::ObAlter LOG_WARN("fail to precheck_interval_part", K(arg), KR(ret)); } } else { - palf::SCN frozen_scn; + SCN frozen_scn; if (OB_FAIL(ret)) { } else if (OB_FAIL(ObMajorFreezeHelper::get_frozen_scn(tenant_id, frozen_scn))) { LOG_WARN("get_frozen_scn failed", K(ret), K(arg)); @@ -3949,7 +3951,7 @@ int ObRootService::create_index(const ObCreateIndexArg &arg, obrpc::ObAlterTable ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), K(ret)); } else { - palf::SCN frozen_scn; + SCN frozen_scn; ObIndexBuilder index_builder(ddl_service_); if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(arg.tenant_id_, schema_guard))) { LOG_WARN("get schema guard in inner table failed", K(ret)); @@ -4186,7 +4188,7 @@ int ObRootService::rebuild_index(const obrpc::ObRebuildIndexArg &arg, obrpc::ObA ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), K(ret)); } else { - palf::SCN frozen_scn; + SCN frozen_scn; if (OB_FAIL(ObMajorFreezeHelper::get_frozen_scn(arg.tenant_id_, frozen_scn))) { LOG_WARN("get_frozen_scn failed", K(ret)); } else if (OB_FAIL(ddl_service_.rebuild_index(arg, frozen_scn, res))) { @@ -4253,7 +4255,7 @@ int ObRootService::truncate_table(const obrpc::ObTruncateTableArg &arg, obrpc::O ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), K(ret)); } else { - palf::SCN frozen_scn; + SCN frozen_scn; if (OB_FAIL(ObMajorFreezeHelper::get_frozen_scn(arg.tenant_id_, frozen_scn))) { LOG_WARN("get_frozen_scn failed", K(ret)); } else if (arg.is_add_to_scheduler_) { @@ -4309,7 +4311,7 @@ int ObRootService::create_table_like(const ObCreateTableLikeArg &arg) ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), K(ret)); } else { - palf::SCN frozen_scn; + SCN frozen_scn; if (OB_FAIL(ObMajorFreezeHelper::get_frozen_scn(arg.tenant_id_, frozen_scn))) { LOG_WARN("get_frozen_scn failed", K(ret)); } else if (OB_FAIL(ddl_service_.create_table_like(arg, frozen_scn))) { @@ -7930,13 +7932,15 @@ int ObRootService::run_upgrade_job(const obrpc::ObUpgradeJobArg &arg) } else if (!arg.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), KR(ret)); - } else if (version < CLUSTER_VERSION_2270 - || !ObUpgradeChecker::check_cluster_version_exist(version)) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("unsupported version to run upgrade job", KR(ret), K(version)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "run upgrade job with such version is"); - } else if (ObUpgradeJobArg::RUN_UPGRADE_JOB == arg.action_) { - if (OB_FAIL(submit_upgrade_task(arg.version_))) { + } else if (ObUpgradeJobArg::UPGRADE_POST_ACTION == arg.action_ + || ObUpgradeJobArg::UPGRADE_SYSTEM_VARIABLE == arg.action_ + || ObUpgradeJobArg::UPGRADE_SYSTEM_TABLE == arg.action_) { + if (ObUpgradeJobArg::UPGRADE_POST_ACTION == arg.action_ + && !ObUpgradeChecker::check_cluster_version_exist(version)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("unsupported version to run upgrade job", KR(ret), K(version)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "run upgrade job with such version is"); + } else if (OB_FAIL(submit_upgrade_task(arg.action_, version))) { LOG_WARN("fail to submit upgrade task", KR(ret), K(arg)); } } else if (ObUpgradeJobArg::STOP_UPGRADE_JOB == arg.action_) { @@ -7953,6 +7957,18 @@ int ObRootService::run_upgrade_job(const obrpc::ObUpgradeJobArg &arg) return ret; } +int ObRootService::upgrade_table_schema(const obrpc::ObUpgradeTableSchemaArg &arg) +{ + int ret = OB_SUCCESS; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_FAIL(ddl_service_.upgrade_table_schema(arg))) { + LOG_WARN("fail to upgrade table schema", KR(ret), K(arg)); + } + return ret; +} + int ObRootService::merge_finish(const obrpc::ObMergeFinishArg &arg) { int ret = OB_SUCCESS; @@ -8966,7 +8982,7 @@ int ObRootService::check_merge_finish(const obrpc::ObCheckMergeFinishArg &arg) { int ret = OB_SUCCESS; LOG_INFO("receive check_merge_finish request", K(arg)); - palf::SCN last_merged_scn = palf::SCN::min_scn(); + SCN last_merged_scn = SCN::min_scn(); share::ObSimpleFrozenStatus frozen_status; if (!inited_) { ret = OB_NOT_INIT; diff --git a/src/rootserver/ob_root_service.h b/src/rootserver/ob_root_service.h index 900c4e46b..b972fe629 100644 --- a/src/rootserver/ob_root_service.h +++ b/src/rootserver/ob_root_service.h @@ -670,6 +670,7 @@ public: int admin_upgrade_virtual_schema(); int run_job(const obrpc::ObRunJobArg &arg); int run_upgrade_job(const obrpc::ObUpgradeJobArg &arg); + int upgrade_table_schema(const obrpc::ObUpgradeTableSchemaArg &arg); int admin_flush_cache(const obrpc::ObAdminFlushCacheArg &arg); int admin_upgrade_cmd(const obrpc::Bool &arg); int admin_rolling_upgrade_cmd(const obrpc::ObAdminRollingUpgradeArg &arg); @@ -695,7 +696,7 @@ public: int report_single_replica(const int64_t tenant_id, const share::ObLSID &ls_id); // @see RsListChangeCb int submit_update_rslist_task(const bool force_update = false); - int submit_upgrade_task(const int64_t version); + int submit_upgrade_task(const obrpc::ObUpgradeJobArg::Action action, const int64_t version); int submit_upgrade_storage_format_version_task(); int submit_create_inner_schema_task(); int submit_async_minor_freeze_task(const obrpc::ObRootMinorFreezeArg &arg); diff --git a/src/rootserver/ob_rs_job_table_operator.cpp b/src/rootserver/ob_rs_job_table_operator.cpp index 3eeb79dec..58b44c23f 100644 --- a/src/rootserver/ob_rs_job_table_operator.cpp +++ b/src/rootserver/ob_rs_job_table_operator.cpp @@ -63,7 +63,9 @@ static const char* job_type_str_array[JOB_TYPE_MAX] = { "UPGRADE_STORAGE_FORMAT_VERSION", "STOP_UPGRADE_STORAGE_FORMAT_VERSION", "CREATE_INNER_SCHEMA", - "RUN_UPGRADE_POST_JOB" + "UPGRADE_POST_ACTION", + "UPGRADE_SYSTEM_VARIABLE", + "UPGRADE_SYSTEM_TABLE" }; const char* ObRsJobTableOperator::get_job_type_str(ObRsJobType job_type) diff --git a/src/rootserver/ob_rs_job_table_operator.h b/src/rootserver/ob_rs_job_table_operator.h index f16995f4a..a3535dc4f 100644 --- a/src/rootserver/ob_rs_job_table_operator.h +++ b/src/rootserver/ob_rs_job_table_operator.h @@ -82,7 +82,9 @@ enum ObRsJobType JOB_TYPE_UPGRADE_STORAGE_FORMAT_VERSION, JOB_TYPE_STOP_UPGRADE_STORAGE_FORMAT_VERSION, JOB_TYPE_CREATE_INNER_SCHEMA, - JOB_TYPE_RUN_UPGRADE_POST_JOB, + JOB_TYPE_UPGRADE_POST_ACTION, + JOB_TYPE_UPGRADE_SYSTEM_VARIABLE, + JOB_TYPE_UPGRADE_SYSTEM_TABLE, JOB_TYPE_MAX }; diff --git a/src/rootserver/ob_rs_rpc_processor.h b/src/rootserver/ob_rs_rpc_processor.h index 99aca11f8..483b3c5ee 100644 --- a/src/rootserver/ob_rs_rpc_processor.h +++ b/src/rootserver/ob_rs_rpc_processor.h @@ -35,6 +35,7 @@ bool is_allow_when_disable_ddl(const obrpc::ObRpcPacketCode pcode, const obrpc:: if (OB_ISNULL(ddl_arg)) { } else if (obrpc::OB_COMMIT_ALTER_TENANT_LOCALITY == pcode || obrpc::OB_SCHEMA_REVISE == pcode // for upgrade + || obrpc::OB_UPGRADE_TABLE_SCHEMA == pcode || ((obrpc::OB_MODIFY_TENANT == pcode || obrpc::OB_MODIFY_SYSVAR == pcode || obrpc::OB_DO_KEYSTORE_DDL == pcode @@ -46,21 +47,6 @@ bool is_allow_when_disable_ddl(const obrpc::ObRpcPacketCode pcode, const obrpc:: return bret; } -// precondition: enable_ddl = false -bool is_allow_when_upgrade(const obrpc::ObRpcPacketCode pcode, const obrpc::ObDDLArg *ddl_arg) -{ - bool bret = false; - UNUSED(pcode); - if (obrpc::OB_UPGRADE_STAGE_DBUPGRADE != GCTX.get_upgrade_stage()) { - bret = true; - } else if (OB_ISNULL(ddl_arg)) { - bret = false; - } else { - bret = ddl_arg->is_allow_when_upgrade(); - } - return bret; -} - bool is_allow_when_create_tenant(const obrpc::ObRpcPacketCode pcode) { bool bret = false; @@ -126,8 +112,7 @@ protected: RS_LOG(WARN, "RS major freeze not finished, can not process ddl request", K(ret), K(pcode)); } else if (is_ddl_like_ - && ((!GCONF.enable_ddl && !is_allow_when_disable_ddl(pcode, ddl_arg_)) - || (GCONF.enable_ddl && !is_allow_when_upgrade(pcode, ddl_arg_)))) { + && (!GCONF.enable_ddl && !is_allow_when_disable_ddl(pcode, ddl_arg_))) { ret = OB_OP_NOT_ALLOW; RS_LOG(WARN, "ddl operation not allow, can not process this request", K(ret), K(pcode)); } else { @@ -476,6 +461,7 @@ DEFINE_RS_RPC_PROCESSOR(obrpc::OB_BROADCAST_SCHEMA, ObBroadcastSchemaP, broadcas // only for upgrade DEFINE_RS_RPC_PROCESSOR(obrpc::OB_CHECK_MERGE_FINISH, ObCheckMergeFinishP, check_merge_finish(arg_)); DEFINE_RS_RPC_PROCESSOR(obrpc::OB_GET_RECYCLE_SCHEMA_VERSIONS, ObGetRecycleSchemaVersionsP, get_recycle_schema_versions(arg_, result_)); +DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_UPGRADE_TABLE_SCHEMA, ObRpcUpgradeTableSchemaP, upgrade_table_schema(arg_)); //label security ddl DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_HANDLE_LABEL_SE_POLICY_DDL, ObRpcHandleLabelSePolicyDDLP, handle_label_se_policy_ddl(arg_)); DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_HANDLE_LABEL_SE_COMPONENT_DDL, ObRpcHandleLabelSeComponentDDLP, handle_label_se_component_ddl(arg_)); diff --git a/src/rootserver/ob_schema_history_recycler.cpp b/src/rootserver/ob_schema_history_recycler.cpp index 8257c2eed..232b7b289 100644 --- a/src/rootserver/ob_schema_history_recycler.cpp +++ b/src/rootserver/ob_schema_history_recycler.cpp @@ -532,7 +532,7 @@ int ObSchemaHistoryRecycler::get_recycle_schema_version_by_global_stat( if (OB_FAIL(check_inner_stat())) { LOG_WARN("schema history recycler is stopped", KR(ret)); } else { - palf::SCN spec_frozen_scn; + SCN spec_frozen_scn; for (int64_t i = 0; OB_SUCC(ret) && i < tenant_ids.count(); i++) { const uint64_t tenant_id = tenant_ids.at(i); ObFreezeInfoProxy freeze_info_proxy(tenant_id); diff --git a/src/rootserver/ob_snapshot_info_manager.cpp b/src/rootserver/ob_snapshot_info_manager.cpp index 113c82d89..c2fffda03 100644 --- a/src/rootserver/ob_snapshot_info_manager.cpp +++ b/src/rootserver/ob_snapshot_info_manager.cpp @@ -163,7 +163,7 @@ int ObSnapshotInfoManager::get_snapshot(common::ObMySQLProxy &proxy, int ObSnapshotInfoManager::get_snapshot(common::ObMySQLProxy &proxy, const uint64_t tenant_id, share::ObSnapShotType snapshot_type, - const palf::SCN &snapshot_scn, + const SCN &snapshot_scn, share::ObSnapshotInfo &snapshot_info) { int ret = OB_SUCCESS; diff --git a/src/rootserver/ob_snapshot_info_manager.h b/src/rootserver/ob_snapshot_info_manager.h index 6ee84d1a7..43d94cbd5 100644 --- a/src/rootserver/ob_snapshot_info_manager.h +++ b/src/rootserver/ob_snapshot_info_manager.h @@ -15,7 +15,7 @@ #include "share/ob_snapshot_table_proxy.h" #include "lib/net/ob_addr.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { namespace common @@ -25,9 +25,6 @@ class ObMySQLProxy; namespace share { class ObSnapshotInfo; -} -namespace palf -{ class SCN; } namespace rootserver @@ -53,7 +50,7 @@ public: int get_snapshot(common::ObMySQLProxy &proxy, const uint64_t tenant_id, share::ObSnapShotType snapshot_type, - const palf::SCN &snapshot_scn, + const share::SCN &snapshot_scn, share::ObSnapshotInfo &snapshot_info); int check_restore_point(common::ObMySQLProxy &proxy, @@ -69,7 +66,7 @@ public: share::ObSnapShotType snapshot_type, const uint64_t tenant_id, const int64_t schema_version, - const palf::SCN &snapshot_scn, + const share::SCN &snapshot_scn, const char *comment, const common::ObIArray &tablet_ids); int batch_release_snapshot_in_trans( @@ -77,7 +74,7 @@ public: share::ObSnapShotType snapshot_type, const uint64_t tenant_id, const int64_t schema_version, - const palf::SCN &snapshot_scn, + const share::SCN &snapshot_scn, const common::ObIArray &tablet_ids); private: diff --git a/src/rootserver/ob_system_admin_util.cpp b/src/rootserver/ob_system_admin_util.cpp index 894f69aae..c0f0cd486 100644 --- a/src/rootserver/ob_system_admin_util.cpp +++ b/src/rootserver/ob_system_admin_util.cpp @@ -1307,34 +1307,49 @@ int ObAdminUpgradeVirtualSchema::upgrade_( } else if (OB_ISNULL(ctx_.ddl_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ddl service is null", KR(ret)); - } else if (OB_FAIL(ctx_.ddl_service_->get_tenant_schema_guard_with_version_in_inner_table( - tenant_id, schema_guard))) { + } + // 1. check table name duplicated + if (FAILEDx(ctx_.ddl_service_->get_tenant_schema_guard_with_version_in_inner_table( + tenant_id, schema_guard))) { LOG_WARN("get schema guard in inner table failed", KR(ret), K(tenant_id)); - } else if (OB_FAIL(schema_guard.get_table_schema(table.get_tenant_id(), + } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table.get_database_id(), table.get_table_name(), table.is_index_table(), exist_schema))) { - // check if the table_name is occupied by others LOG_WARN("get table schema failed", KR(ret), K(tenant_id), "table", table.get_table_name()); if (OB_TABLE_NOT_EXIST == ret) { ret = OB_SUCCESS; } - } else if (OB_NOT_NULL(exist_schema)) { - // name modification except virtual table should first delete the old table, - // then create the new one to make virtual table upgrade valid - if (OB_FAIL(ctx_.ddl_service_->drop_inner_table(*exist_schema))) { - LOG_WARN("get table schema failed", KR(ret), K(tenant_id), - "table", table.get_table_name(), "table_id", table.get_table_id()); - } else if (OB_FAIL(ctx_.ddl_service_->get_tenant_schema_guard_with_version_in_inner_table( - tenant_id, schema_guard))) { - LOG_WARN("get schema guard in inner table failed", KR(ret), K(tenant_id)); - } + } else if (OB_ISNULL(exist_schema)) { + // no duplicate table name + } else if (OB_FAIL(ctx_.ddl_service_->drop_inner_table(*exist_schema))) { + LOG_WARN("get table schema failed", KR(ret), K(tenant_id), + "table", table.get_table_name(), "table_id", table.get_table_id()); + } else if (OB_FAIL(ctx_.ddl_service_->get_tenant_schema_guard_with_version_in_inner_table( + tenant_id, schema_guard))) { + LOG_WARN("get schema guard in inner table failed", KR(ret), K(tenant_id)); } - - // rebuild the inner table - if (OB_FAIL(ret)) { - } else if (OB_FAIL(ctx_.ddl_service_->add_table_schema(table, schema_guard))) { + // 2. try drop table first + exist_schema = NULL; + if (FAILEDx(schema_guard.get_table_schema(tenant_id, + table.get_table_id(), + exist_schema))) { + LOG_WARN("get table schema failed", KR(ret), "table", table.get_table_name(), + "table_id", table.get_table_id()); + if (OB_TABLE_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } + } else if (OB_ISNULL(exist_schema)) { + // missed table + } else if (OB_FAIL(ctx_.ddl_service_->drop_inner_table(*exist_schema))) { + LOG_WARN("drop table schema failed", KR(ret), "table_schema", *exist_schema); + } else if (OB_FAIL(ctx_.ddl_service_->get_tenant_schema_guard_with_version_in_inner_table( + tenant_id, schema_guard))) { + LOG_WARN("get schema guard in inner table failed", KR(ret), K(tenant_id)); + } + // 3. create table + if (FAILEDx(ctx_.ddl_service_->add_table_schema(table, schema_guard))) { LOG_WARN("add table schema failed", KR(ret), K(tenant_id), K(table)); } else if (OB_FAIL(ctx_.ddl_service_->refresh_schema(tenant_id))) { LOG_WARN("refresh schema failed", KR(ret), K(tenant_id)); diff --git a/src/rootserver/ob_table_creator.cpp b/src/rootserver/ob_table_creator.cpp index 41f25147e..1a5502a56 100644 --- a/src/rootserver/ob_table_creator.cpp +++ b/src/rootserver/ob_table_creator.cpp @@ -14,7 +14,7 @@ #include "rootserver/ob_table_creator.h" #include "rootserver/ob_root_service.h" #include "share/tablet/ob_tablet_to_table_history_operator.h" // ObTabletToTableHistoryOperator -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { diff --git a/src/rootserver/ob_table_creator.h b/src/rootserver/ob_table_creator.h index 84ef8810b..4b90b34b6 100644 --- a/src/rootserver/ob_table_creator.h +++ b/src/rootserver/ob_table_creator.h @@ -20,7 +20,7 @@ namespace oceanbase { -namespace palf +namespace share { class SCN; } @@ -36,7 +36,7 @@ class ObTableCreator public: ObTableCreator( const uint64_t tenant_id, - const palf::SCN &frozen_scn, + const share::SCN &frozen_scn, share::ObLSTableOperator &lst_operator, ObMySQLTransaction &trans) : tenant_id_(tenant_id), diff --git a/src/rootserver/ob_tablet_creator.cpp b/src/rootserver/ob_tablet_creator.cpp index 31c97163f..74f47f765 100644 --- a/src/rootserver/ob_tablet_creator.cpp +++ b/src/rootserver/ob_tablet_creator.cpp @@ -22,6 +22,7 @@ namespace oceanbase { +using namespace share; namespace rootserver { @@ -108,7 +109,7 @@ DEF_TO_STRING(ObTabletCreatorArg) ///////////////////////////////////////////////////////// int ObBatchCreateTabletHelper::init(const share::ObLSID &ls_key, const int64_t tenant_id, - const palf::SCN &major_frozen_scn) + const SCN &major_frozen_scn) { int ret = OB_SUCCESS; const int64_t bucket_count = hash::cal_next_prime(100); diff --git a/src/rootserver/ob_tablet_creator.h b/src/rootserver/ob_tablet_creator.h index 065ca26cb..d67762ba9 100644 --- a/src/rootserver/ob_tablet_creator.h +++ b/src/rootserver/ob_tablet_creator.h @@ -72,7 +72,7 @@ public: ObBatchCreateTabletHelper() :arg_(), table_schemas_map_(), result_(common::OB_NOT_MASTER), next_(NULL) {} int init(const share::ObLSID &ls_key, const int64_t tenant_id, - const palf::SCN &major_frozen_scn); + const share::SCN &major_frozen_scn); int try_add_table_schema(const share::schema::ObTableSchema *table_schema, int64_t &index); int add_arg_to_batch_arg(const ObTabletCreatorArg &arg); void reset() @@ -97,7 +97,7 @@ class ObTabletCreator public: ObTabletCreator( const uint64_t tenant_id, - const palf::SCN &major_frozen_scn, + const share::SCN &major_frozen_scn, share::ObLSTableOperator &lst_operator, ObMySQLTransaction &trans) : tenant_id_(tenant_id), @@ -119,7 +119,7 @@ private: const int64_t MAP_BUCKET_NUM = 1024; private: const uint64_t tenant_id_; - const palf::SCN major_frozen_scn_; + const share::SCN major_frozen_scn_; share::ObLSTableOperator *lst_operator_; ObArenaAllocator allocator_; common::hash::ObHashMap args_map_; diff --git a/src/rootserver/ob_tenant_recovery_reportor.cpp b/src/rootserver/ob_tenant_recovery_reportor.cpp index a072d8af7..1d9794e47 100644 --- a/src/rootserver/ob_tenant_recovery_reportor.cpp +++ b/src/rootserver/ob_tenant_recovery_reportor.cpp @@ -22,7 +22,7 @@ #include "share/ls/ob_ls_recovery_stat_operator.h" //ObLSRecoveryStatOperator #include "share/schema/ob_multi_version_schema_service.h"//is_tenant_full_schema #include "logservice/ob_log_service.h"//get_palf_role -#include "logservice/palf/scn.h"//SCN +#include "share/scn.h"//SCN #include "storage/tx_storage/ob_ls_handle.h" //ObLSHandle namespace oceanbase @@ -389,7 +389,7 @@ int ObTenantRecoveryReportor::update_replayable_point_from_meta_() } int ObTenantRecoveryReportor::get_sync_point_(const share::ObLSID &id, - palf::SCN &sync_scn, palf::SCN &read_scn) + SCN &sync_scn, SCN &read_scn) { int ret = OB_SUCCESS; palf::AccessMode access_mode; diff --git a/src/rootserver/ob_tenant_recovery_reportor.h b/src/rootserver/ob_tenant_recovery_reportor.h index 25a9bbb4b..54bece2b6 100644 --- a/src/rootserver/ob_tenant_recovery_reportor.h +++ b/src/rootserver/ob_tenant_recovery_reportor.h @@ -26,15 +26,12 @@ class ObMySQLProxy; namespace share { class ObLSID; +class SCN; } namespace storage { class ObLS; } -namespace palf -{ -class SCN; -} namespace rootserver { /*description: @@ -67,14 +64,14 @@ public: static int update_ls_recovery(storage::ObLS *ls, common::ObMySQLProxy *sql_proxy); int get_tenant_info(share::ObAllTenantInfo &tenant_info); - static int get_readable_scn(const share::ObLSID &id, palf::SCN &read_scn); + static int get_readable_scn(const share::ObLSID &id, share::SCN &read_scn); private: int load_tenant_info_(); - static int get_sync_point_(const share::ObLSID &id, palf::SCN &scn, palf::SCN &read_scn); + static int get_sync_point_(const share::ObLSID &id, share::SCN &scn, share::SCN &read_scn); int update_ls_recovery_stat_(); public: - static constexpr int64_t IDLE_TIME_US = 500 * 1000; + static constexpr int64_t IDLE_TIME_US = 100 * 1000; TO_STRING_KV(K_(is_inited), K_(tenant_id), K_(tenant_info)); private: bool is_inited_; diff --git a/src/rootserver/ob_tenant_role_transition_service.cpp b/src/rootserver/ob_tenant_role_transition_service.cpp index b22a4a3ef..6d64cabaa 100644 --- a/src/rootserver/ob_tenant_role_transition_service.cpp +++ b/src/rootserver/ob_tenant_role_transition_service.cpp @@ -13,7 +13,7 @@ #define USING_LOG_PREFIX RS #include "ob_tenant_role_transition_service.h" #include "logservice/palf/log_define.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "logservice/ob_log_service.h" #include "rootserver/ob_rs_async_rpc_proxy.h"//ObChangeLSAccessModeProxy #include "share/ob_rpc_struct.h"//ObLSAccessModeInfo @@ -313,7 +313,7 @@ int ObTenantRoleTransitionService::update_tenant_stat_info_() } int ObTenantRoleTransitionService::change_ls_access_mode_(palf::AccessMode target_access_mode, - const palf::SCN &ref_scn) + const SCN &ref_scn) { int ret = OB_SUCCESS; ObTimeoutCtx ctx; @@ -488,7 +488,7 @@ int ObTenantRoleTransitionService::get_ls_access_mode_(ObIArray &ls_access_info, - palf::AccessMode target_access_mode, const palf::SCN &ref_scn) + palf::AccessMode target_access_mode, const SCN &ref_scn) { int ret = OB_SUCCESS; ObTimeoutCtx ctx; diff --git a/src/rootserver/ob_tenant_role_transition_service.h b/src/rootserver/ob_tenant_role_transition_service.h index b5eccd592..fbc03d4ec 100644 --- a/src/rootserver/ob_tenant_role_transition_service.h +++ b/src/rootserver/ob_tenant_role_transition_service.h @@ -28,12 +28,9 @@ namespace common class ObMySQLProxy; class ObMySQLTransaction; } -namespace palf -{ -class SCN; -} namespace share { +class SCN; struct ObAllTenantInfo; } @@ -91,12 +88,12 @@ private: const share::ObTenantRole &target_tenant_role); int try_create_abort_ls_(const share::ObTenantSwitchoverStatus &status); int change_ls_access_mode_(palf::AccessMode target_access_mode, - const palf::SCN &ref_scn); + const share::SCN &ref_scn); int update_tenant_stat_info_(); int get_ls_access_mode_(ObIArray &ls_access_info); int do_change_ls_access_mode_(const ObIArray &ls_access_info, palf::AccessMode target_access_mode, - const palf::SCN &ref_scn); + const share::SCN &ref_scn); private: uint64_t tenant_id_; common::ObMySQLProxy *sql_proxy_; diff --git a/src/rootserver/ob_upgrade_executor.cpp b/src/rootserver/ob_upgrade_executor.cpp index 466e52401..d4aee5e48 100644 --- a/src/rootserver/ob_upgrade_executor.cpp +++ b/src/rootserver/ob_upgrade_executor.cpp @@ -15,6 +15,7 @@ #include "rootserver/ob_upgrade_executor.h" #include "rootserver/ob_rs_job_table_operator.h" #include "observer/ob_server_struct.h" +#include "rootserver/ob_root_inspection.h" namespace oceanbase { @@ -43,7 +44,7 @@ ObAsyncTask *ObUpgradeTask::deep_copy(char *buf, const int64_t buf_size) const ret = OB_INVALID_ARGUMENT; LOG_WARN("buf is not long enough", K(need_size), K(buf_size), KR(ret)); } else { - task = new(buf) ObUpgradeTask(*upgrade_executor_, version_); + task = new(buf) ObUpgradeTask(*upgrade_executor_, action_, version_); } return task; } @@ -51,22 +52,24 @@ ObAsyncTask *ObUpgradeTask::deep_copy(char *buf, const int64_t buf_size) const int ObUpgradeTask::process() { const int64_t start = ObTimeUtility::current_time(); - LOG_INFO("[UPGRADE] start to do execute upgrade task", K(start), K_(version)); + FLOG_INFO("[UPGRADE] start to do execute upgrade task", + K(start), K_(action), K_(version)); int ret = OB_SUCCESS; if (OB_ISNULL(upgrade_executor_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("upgrade_executor_ is null", KR(ret)); - } else if (OB_FAIL(upgrade_executor_->execute(version_))) { - LOG_WARN("fail to execute upgrade task", KR(ret)); + } else if (OB_FAIL(upgrade_executor_->execute(action_, version_))) { + LOG_WARN("fail to execute upgrade task", KR(ret), K_(action), K_(version)); } - LOG_INFO("[UPGRADE] finish execute upgrade task", - KR(ret), K_(version), "cost_us", ObTimeUtility::current_time() - start); + FLOG_INFO("[UPGRADE] finish execute upgrade task", + KR(ret), K_(action), K_(version), + "cost_us", ObTimeUtility::current_time() - start); return ret; } ObUpgradeExecutor::ObUpgradeExecutor() : inited_(false), stopped_(false), execute_(false), rwlock_(), - sql_proxy_(NULL), rpc_proxy_(NULL), schema_service_(NULL), + sql_proxy_(NULL), rpc_proxy_(NULL), common_rpc_proxy_(NULL), schema_service_(NULL), upgrade_processors_() {} @@ -88,6 +91,7 @@ int ObUpgradeExecutor::init( schema_service_ = &schema_service; sql_proxy_ = &sql_proxy; rpc_proxy_ = &rpc_proxy; + common_rpc_proxy_ = &common_proxy; stopped_ = false; execute_ = false; inited_ = true; @@ -128,9 +132,8 @@ int ObUpgradeExecutor::check_stop() const { int ret = OB_SUCCESS; SpinRLockGuard guard(rwlock_); - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret)); } else if (stopped_) { ret = OB_CANCELED; LOG_WARN("executor should stopped", KR(ret)); @@ -145,13 +148,12 @@ bool ObUpgradeExecutor::check_execute() const return bret; } -int ObUpgradeExecutor::set_execute_mark() +int ObUpgradeExecutor::set_execute_mark_() { int ret = OB_SUCCESS; SpinWLockGuard guard(rwlock_); - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret)); } else if (stopped_ || execute_) { ret = OB_OP_NOT_ALLOW; LOG_WARN("can't run job at the same time", KR(ret)); @@ -165,9 +167,8 @@ int ObUpgradeExecutor::can_execute() { int ret = OB_SUCCESS; SpinWLockGuard guard(rwlock_); - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret)); } else if (stopped_ || execute_) { ret = OB_OP_NOT_ALLOW; LOG_WARN("status not matched", KR(ret), @@ -177,17 +178,33 @@ int ObUpgradeExecutor::can_execute() return ret; } +int ObUpgradeExecutor::check_inner_stat_() const +{ + int ret = OB_SUCCESS; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", KR(ret)); + } else if (OB_ISNULL(schema_service_) + || OB_ISNULL(sql_proxy_) + || OB_ISNULL(rpc_proxy_) + || OB_ISNULL(common_rpc_proxy_)){ + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ptr is null", KR(ret), KP_(schema_service), + KP_(sql_proxy), KP_(rpc_proxy), KP_(common_rpc_proxy)); + } + return ret; +} + // wait schema sync in cluster -int ObUpgradeExecutor::check_schema_sync() +int ObUpgradeExecutor::check_schema_sync_() { const int64_t start = ObTimeUtility::current_time(); LOG_INFO("[UPGRADE] start to check schema sync", K(start)); int ret = OB_SUCCESS; bool enable_ddl = GCONF.enable_ddl; bool enable_sys_table_ddl = GCONF.enable_sys_table_ddl; - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", KR(ret)); + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret)); } else if (enable_ddl || enable_sys_table_ddl) { ret = OB_OP_NOT_ALLOW; LOG_WARN("ddl should disable now", KR(ret), K(enable_ddl), K(enable_sys_table_ddl)); @@ -213,7 +230,7 @@ int ObUpgradeExecutor::check_schema_sync() } // Ensure primary cluster's schema_version is not greator than standby clusters'. -int ObUpgradeExecutor::check_schema_sync( +int ObUpgradeExecutor::check_schema_sync_( obrpc::ObTenantSchemaVersions &primary_schema_versions, obrpc::ObTenantSchemaVersions &standby_schema_versions, bool &schema_sync) @@ -255,61 +272,31 @@ int ObUpgradeExecutor::check_schema_sync( return ret; } -int ObUpgradeExecutor::get_tenant_ids(common::ObIArray &tenant_ids) -{ - int ret = OB_SUCCESS; - ObSchemaGetterGuard guard; - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_FAIL(check_stop())) { - LOG_WARN("executor should stopped", KR(ret)); - } else if (OB_ISNULL(schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("schema_service is null", KR(ret)); - } else if (OB_FAIL(schema_service_->get_tenant_schema_guard( - OB_SYS_TENANT_ID, guard))) { - LOG_WARN("fail to get schema guard", KR(ret)); - } else if (OB_FAIL(guard.get_tenant_ids(tenant_ids))) { - LOG_WARN("fail to get tenant_ids", KR(ret)); - } - return ret; -} - -int ObUpgradeExecutor::execute(const int64_t version) +//TODO: +//1. Run upgrade job by tenant. +//2. Check tenant role/tenant status before run upgrade job. +int ObUpgradeExecutor::execute( + const obrpc::ObUpgradeJobArg::Action action, + const int64_t version) { ObCurTraceId::init(GCONF.self_addr_); int ret = OB_SUCCESS; - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); - } else if (OB_FAIL(set_execute_mark())) { + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret)); + } else if (OB_FAIL(set_execute_mark_())) { LOG_WARN("fail to set execute mark", KR(ret)); } else { - int64_t job_id = OB_INVALID_ID; - ObRsJobType job_type = ObRsJobType::JOB_TYPE_RUN_UPGRADE_POST_JOB; - char version_str[common::OB_CLUSTER_VERSION_LENGTH] = {0}; - int64_t len = ObClusterVersion::print_version_str( - version_str, common::OB_CLUSTER_VERSION_LENGTH, version); - if (OB_ISNULL(sql_proxy_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sql_proxy_ is null", KR(ret)); - } else if (OB_FAIL(check_stop())) { - LOG_WARN("executor should stopped", KR(ret)); - } else if (OB_FAIL(RS_JOB_CREATE_WITH_RET(job_id, job_type, *sql_proxy_, - "extra_info", ObString(len, version_str)))) { - LOG_WARN("fail to create rs job", KR(ret)); - } else if (job_id <= 0) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("job_id is invalid", KR(ret), K(job_id)); - } else if (OB_FAIL(run_upgrade_job(version))) { - LOG_WARN("fail to run upgrade job", KR(ret), K(job_id), K(version)); - } - int tmp_ret = OB_SUCCESS; - if (job_id > 0) { - if (OB_SUCCESS != (tmp_ret = RS_JOB_COMPLETE(job_id, ret, *sql_proxy_))) { - LOG_ERROR("fail to complete job", K(tmp_ret), KR(ret), K(job_id)); - ret = OB_FAIL(ret) ? ret : tmp_ret; + if (obrpc::ObUpgradeJobArg::UPGRADE_POST_ACTION == action) { + if (OB_FAIL(run_upgrade_post_job_(version))) { + LOG_WARN("fail to run upgrade post job", KR(ret), K(version)); + } + } else if (obrpc::ObUpgradeJobArg::UPGRADE_SYSTEM_VARIABLE == action) { + if (OB_FAIL(run_upgrade_system_variable_job_())) { + LOG_WARN("fail to run upgrade system variable job", KR(ret)); + } + } else if (obrpc::ObUpgradeJobArg::UPGRADE_SYSTEM_TABLE == action) { + if (OB_FAIL(run_upgrade_system_table_job_())) { + LOG_WARN("fail to run upgrade system table job", KR(ret)); } } execute_ = false; @@ -317,49 +304,267 @@ int ObUpgradeExecutor::execute(const int64_t version) return ret; } -// Python upgrade script will set enable_ddl = false before it run upgrade job. -int ObUpgradeExecutor::run_upgrade_job(const int64_t version) +// Python upgrade script may set enable_ddl = false before it run upgrade job. +// TODO: +// 1. support run upgrade post action from `COMPATIBLE` to current cluster version. +int ObUpgradeExecutor::run_upgrade_post_job_(const int64_t version) { int ret = OB_SUCCESS; - ObArray tenant_ids; - ObBaseUpgradeProcessor *processor = NULL; - if (!inited_) { - ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret)); } else if (OB_FAIL(check_stop())) { LOG_WARN("executor should stopped", KR(ret)); - } else if (version < CLUSTER_VERSION_2270 - || !ObUpgradeChecker::check_cluster_version_exist(version)) { + } else if (!ObUpgradeChecker::check_cluster_version_exist(version)) { ret = OB_NOT_SUPPORTED; LOG_WARN("unsupported version to run upgrade job", KR(ret), K(version)); - } else if (OB_FAIL(get_tenant_ids(tenant_ids))) { - LOG_WARN("fail to get tenant_ids", KR(ret), K(version)); - } else if (OB_FAIL(upgrade_processors_.get_processor_by_version( - version, processor))) { - LOG_WARN("fail to get processor by version", KR(ret), K(version)); } else { - // 1. Run upgrade jobs(by tenant_id desc) in primary cluster. - // 2. Only run sys tenant's upgrade job in standby clusters. - for (int64_t i = tenant_ids.count() - 1; OB_SUCC(ret) && i >= 0; i--) { - const uint64_t tenant_id = tenant_ids.at(i); - if (OB_SYS_TENANT_ID == tenant_id || !GCTX.is_standby_cluster()) { + ObArray tenant_ids; + ObBaseUpgradeProcessor *processor = NULL; + int64_t job_id = OB_INVALID_ID; + ObRsJobType job_type = ObRsJobType::JOB_TYPE_UPGRADE_POST_ACTION; + char version_str[common::OB_CLUSTER_VERSION_LENGTH] = {0}; + int64_t len = ObClusterVersion::print_version_str( + version_str, common::OB_CLUSTER_VERSION_LENGTH, version); + int tmp_ret = OB_SUCCESS; + int64_t backup_ret = OB_SUCCESS; + if (OB_FAIL(RS_JOB_CREATE_WITH_RET(job_id, job_type, *sql_proxy_, + "tenant_id", 0, + "extra_info", ObString(len, version_str)))) { + LOG_WARN("fail to create rs job", KR(ret)); + } else if (job_id <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("job_id is invalid", KR(ret), K(job_id)); + } else if (OB_FAIL(schema_service_->get_tenant_ids(tenant_ids))) { + LOG_WARN("fail to get tenant_ids", KR(ret), K(version)); + } else if (OB_FAIL(upgrade_processors_.get_processor_by_version( + version, processor))) { + LOG_WARN("fail to get processor by version", KR(ret), K(version)); + } else { + for (int64_t i = tenant_ids.count() - 1; OB_SUCC(ret) && i >= 0; i--) { + const uint64_t tenant_id = tenant_ids.at(i); int64_t start_ts = ObTimeUtility::current_time(); int64_t current_version = processor->get_version(); processor->set_tenant_id(tenant_id); - LOG_INFO("[UPGRADE] start to run post upgrade job by version", - K(tenant_id), K(current_version)); - if (OB_FAIL(processor->post_upgrade())) { + FLOG_INFO("[UPGRADE] start to run post upgrade job by version", + K(tenant_id), K(current_version)); + if (OB_FAIL(check_stop())) { + LOG_WARN("executor should stopped", KR(ret)); + } else if (OB_TMP_FAIL(processor->post_upgrade())) { LOG_WARN("run post upgrade by version failed", - KR(ret), K(tenant_id), K(current_version)); + KR(tmp_ret), K(tenant_id), K(current_version)); + backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret; } - LOG_INFO("[UPGRADE] finish post upgrade job by version", - KR(ret), K(tenant_id), K(current_version), - "cost", ObTimeUtility::current_time() - start_ts); + FLOG_INFO("[UPGRADE] finish post upgrade job by version", + KR(tmp_ret), K(tenant_id), K(current_version), + "cost", ObTimeUtility::current_time() - start_ts); + } // end for + } + ret = OB_SUCC(ret) ? backup_ret : ret; + if (job_id > 0) { + if (OB_SUCCESS != (tmp_ret = RS_JOB_COMPLETE(job_id, ret, *sql_proxy_))) { + LOG_ERROR("fail to complete job", K(tmp_ret), KR(ret), K(job_id)); + ret = OB_FAIL(ret) ? ret : tmp_ret; } } } return ret; } +int ObUpgradeExecutor::run_upgrade_system_variable_job_() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret)); + } else if (OB_FAIL(check_stop())) { + LOG_WARN("executor should stopped", KR(ret)); + } else { + ObArray tenant_ids; + int64_t job_id = OB_INVALID_ID; + ObRsJobType job_type = ObRsJobType::JOB_TYPE_UPGRADE_SYSTEM_VARIABLE; + int tmp_ret = OB_SUCCESS; + int backup_ret = OB_SUCCESS; + if (OB_FAIL(RS_JOB_CREATE_WITH_RET(job_id, job_type, *sql_proxy_, "tenant_id", 0))) { + LOG_WARN("fail to create rs job", KR(ret)); + } else if (job_id <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("job_id is invalid", KR(ret), K(job_id)); + } else if (OB_FAIL(schema_service_->get_tenant_ids(tenant_ids))) { + LOG_WARN("fail to get tenant_ids", KR(ret)); + } else { + for (int64_t i = tenant_ids.count() - 1; OB_SUCC(ret) && i >= 0; i--) { + const uint64_t tenant_id = tenant_ids.at(i); + int64_t start_ts = ObTimeUtility::current_time(); + FLOG_INFO("[UPGRADE] start to run upgrade system variable job", K(tenant_id)); + if (OB_FAIL(check_stop())) { + LOG_WARN("executor should stopped", KR(ret)); + } else if (OB_TMP_FAIL(ObUpgradeUtils::upgrade_sys_variable(*common_rpc_proxy_, *sql_proxy_, tenant_id))) { + LOG_WARN("fail to upgrade sys variable", KR(tmp_ret), K(tenant_id)); + backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret; + } + FLOG_INFO("[UPGRADE] finish run upgrade system variable job", + KR(tmp_ret), K(tenant_id), "cost", ObTimeUtility::current_time() - start_ts); + } // end for + ret = OB_SUCC(ret) ? backup_ret : ret; + } + if (job_id > 0) { + if (OB_SUCCESS != (tmp_ret = RS_JOB_COMPLETE(job_id, ret, *sql_proxy_))) { + LOG_ERROR("fail to complete job", K(tmp_ret), KR(ret), K(job_id)); + ret = OB_FAIL(ret) ? ret : tmp_ret; + } + } + } + return ret; +} + +// NOTICE: enable_sys_table_ddl should be true before run this job. +int ObUpgradeExecutor::run_upgrade_system_table_job_() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret)); + } else if (OB_FAIL(check_stop())) { + LOG_WARN("executor should stopped", KR(ret)); + } else { + ObArray tenant_ids; + int64_t job_id = OB_INVALID_ID; + ObRsJobType job_type = ObRsJobType::JOB_TYPE_UPGRADE_SYSTEM_TABLE; + int tmp_ret = OB_SUCCESS; + int backup_ret = OB_SUCCESS; + if (OB_FAIL(RS_JOB_CREATE_WITH_RET(job_id, job_type, *sql_proxy_, "tenant_id", 0))) { + LOG_WARN("fail to create rs job", KR(ret)); + } else if (job_id <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("job_id is invalid", KR(ret), K(job_id)); + } else if (OB_FAIL(schema_service_->get_tenant_ids(tenant_ids))) { + LOG_WARN("fail to get tenant_ids", KR(ret)); + } else { + for (int64_t i = tenant_ids.count() - 1; i >= 0; i--) { + const uint64_t tenant_id = tenant_ids.at(i); + int64_t start_ts = ObTimeUtility::current_time(); + FLOG_INFO("[UPGRADE] start to run upgrade system table job", K(tenant_id)); + if (OB_FAIL(check_stop())) { + LOG_WARN("executor should stopped", KR(ret)); + } else if (OB_TMP_FAIL(upgrade_system_table_(tenant_id))) { + LOG_WARN("fail to upgrade system table", KR(tmp_ret), K(tenant_id)); + backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret; + } + FLOG_INFO("[UPGRADE] finish run upgrade system table job", + KR(tmp_ret), K(tenant_id), "cost", ObTimeUtility::current_time() - start_ts); + } // end for + ret = OB_SUCC(ret) ? backup_ret : ret; + } + if (job_id > 0) { + if (OB_SUCCESS != (tmp_ret = RS_JOB_COMPLETE(job_id, ret, *sql_proxy_))) { + LOG_ERROR("fail to complete job", K(tmp_ret), KR(ret), K(job_id)); + ret = OB_FAIL(ret) ? ret : tmp_ret; + } + } + } + return ret; +} + +int ObUpgradeExecutor::upgrade_system_table_(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret)); + } else if (OB_FAIL(check_stop())) { + LOG_WARN("executor should stopped", KR(ret)); + } else { + ObArray upgrade_table_ids; // miss or mismatch + // Only core/system tables can be upgraded here. + // 1. __all_core_table can't be altered. + // 2. sys index table and sys lob table will be added with sys data table, and can't be altered. + const schema_create_func *creator_ptr_array[] = { + share::core_table_schema_creators, + share::sys_table_schema_creators, NULL }; + + // check system table + ObTableSchema table_schema; + bool exist = false; + for (const schema_create_func **creator_ptr_ptr = creator_ptr_array; + OB_SUCC(ret) && OB_NOT_NULL(*creator_ptr_ptr); ++creator_ptr_ptr) { + for (const schema_create_func *creator_ptr = *creator_ptr_ptr; + OB_SUCC(ret) && OB_NOT_NULL(*creator_ptr); ++creator_ptr) { + table_schema.reset(); + if (OB_FAIL(check_stop())) { + LOG_WARN("check_cancel failed", KR(ret)); + } else if (OB_FAIL((*creator_ptr)(table_schema))) { + LOG_WARN("create table schema failed", KR(ret)); + } else if (!is_sys_tenant(tenant_id) + && OB_FAIL(ObSchemaUtils::construct_tenant_space_full_table( + tenant_id, table_schema))) { + LOG_WARN("fail to construct tenant space table", KR(ret), K(tenant_id)); + } else if (OB_FAIL(ObSysTableChecker::is_inner_table_exist( + tenant_id, table_schema, exist))) { + LOG_WARN("fail to check inner table exist", + KR(ret), K(tenant_id), K(table_schema)); + } else if (!exist) { + // skip + } else if (OB_FAIL(check_table_schema_(tenant_id, table_schema))) { + const uint64_t table_id = table_schema.get_table_id(); + if (OB_SCHEMA_ERROR != ret) { + LOG_WARN("check_table_schema failed", KR(ret), K(tenant_id), K(table_id)); + } else { + FLOG_INFO("[UPGRADE] table need upgrade", K(tenant_id), K(table_id), + "table_name", table_schema.get_table_name()); + if (OB_FAIL(upgrade_table_ids.push_back(table_id))) { // overwrite ret + LOG_WARN("fail to push back upgrade table ids", KR(ret), K(tenant_id), K(table_id)); + } + } + } + } // end for + } // end for + + int tmp_ret = OB_SUCCESS; + int backup_ret = OB_SUCCESS; + // upgrade system table(create or alter) + obrpc::ObUpgradeTableSchemaArg arg; + const int64_t timeout = GCONF.internal_sql_execute_timeout; + for (int64_t i = 0; OB_SUCC(ret) && i < upgrade_table_ids.count(); i++) { + const uint64_t table_id = upgrade_table_ids.at(i); + int64_t start_ts = ObTimeUtility::current_time(); + FLOG_INFO("[UPGRADE] start upgrade system table", K(tenant_id), K(table_id)); + if (OB_FAIL(check_stop())) { + LOG_WARN("check_cancel failed", KR(ret)); + } else if (OB_FAIL(arg.init(tenant_id, table_id))) { + LOG_WARN("fail to init arg", KR(ret), K(tenant_id), K(table_id)); + } else if (OB_TMP_FAIL(common_rpc_proxy_->timeout(timeout).upgrade_table_schema(arg))) { + LOG_WARN("fail to uggrade table schema", KR(tmp_ret), K(timeout), K(arg)); + backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret; + } + FLOG_INFO("[UPGRADE] finish upgrade system table", + KR(tmp_ret), K(tenant_id), K(table_id), "cost", ObTimeUtility::current_time() - start_ts); + } // end for + ret = OB_SUCC(ret) ? backup_ret : ret; + } + return ret; +} + +int ObUpgradeExecutor::check_table_schema_(const uint64_t tenant_id, const ObTableSchema &hard_code_table) +{ + int ret = OB_SUCCESS; + const ObTableSchema *table = NULL; + ObSchemaGetterGuard schema_guard; + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret)); + } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { + LOG_WARN("failed to get schema guard", KR(ret), K(tenant_id)); + } else if (OB_FAIL(schema_guard.get_table_schema( + tenant_id, hard_code_table.get_table_id(), table))) { + LOG_WARN("get_table_schema failed", KR(ret), K(tenant_id), + "table_id", hard_code_table.get_table_id(), + "table_name", hard_code_table.get_table_name()); + } else if (OB_ISNULL(table)) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("table should not be null", KR(ret), K(tenant_id), + "table_id", hard_code_table.get_table_id(), + "table_name", hard_code_table.get_table_name()); + } else if (OB_FAIL(ObRootInspection::check_table_schema(hard_code_table, *table))) { + LOG_WARN("fail to check table schema", KR(ret), K(tenant_id), K(hard_code_table), KPC(table)); + } + return ret; +} + }//end rootserver }//end oceanbase diff --git a/src/rootserver/ob_upgrade_executor.h b/src/rootserver/ob_upgrade_executor.h index 40d30675b..4d4f4918e 100644 --- a/src/rootserver/ob_upgrade_executor.h +++ b/src/rootserver/ob_upgrade_executor.h @@ -18,6 +18,7 @@ #include "share/ob_check_stop_provider.h" #include "share/schema/ob_multi_version_schema_service.h" #include "share/schema/ob_schema_getter_guard.h" +#include "share/ob_rpc_struct.h" namespace oceanbase { @@ -29,8 +30,9 @@ class ObUpgradeTask: public share::ObAsyncTask { public: explicit ObUpgradeTask(ObUpgradeExecutor &upgrade_executor, + const obrpc::ObUpgradeJobArg::Action action, const int64_t version) - : upgrade_executor_(&upgrade_executor), version_(version) + : upgrade_executor_(&upgrade_executor), action_(action), version_(version) {} virtual ~ObUpgradeTask() {} virtual int64_t get_deep_copy_size() const; @@ -38,6 +40,7 @@ public: virtual int process(); private: ObUpgradeExecutor *upgrade_executor_; + obrpc::ObUpgradeJobArg::Action action_; int64_t version_; }; @@ -51,7 +54,8 @@ public: obrpc::ObSrvRpcProxy &rpc_proxy, obrpc::ObCommonRpcProxy &common_proxy); - int execute(const int64_t version); + int execute(const obrpc::ObUpgradeJobArg::Action action, + const int64_t version); int can_execute(); int check_stop() const; bool check_execute() const; @@ -59,16 +63,22 @@ public: void start(); int stop(); private: - int set_execute_mark(); + int check_inner_stat_() const; + int set_execute_mark_(); - int check_schema_sync(); - int check_schema_sync( + int run_upgrade_post_job_(const int64_t version); + int run_upgrade_system_variable_job_(); + int run_upgrade_system_table_job_(); + + int upgrade_system_table_(const uint64_t tenant_id); + int check_table_schema_(const uint64_t tenant_id, + const share::schema::ObTableSchema &hard_code_table); + + int check_schema_sync_(); + int check_schema_sync_( obrpc::ObTenantSchemaVersions &primary_schema_versions, obrpc::ObTenantSchemaVersions &standby_schema_versions, bool &schema_sync); - int get_tenant_ids(common::ObIArray &tenant_ids); - - int run_upgrade_job(const int64_t version); private: bool inited_; bool stopped_; @@ -76,6 +86,7 @@ private: common::SpinRWLock rwlock_; common::ObMySQLProxy *sql_proxy_; obrpc::ObSrvRpcProxy *rpc_proxy_; + obrpc::ObCommonRpcProxy *common_rpc_proxy_; share::schema::ObMultiVersionSchemaService *schema_service_; share::ObUpgradeProcesserSet upgrade_processors_; DISALLOW_COPY_AND_ASSIGN(ObUpgradeExecutor); diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index 73392ba8c..cf82072d9 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -32,7 +32,7 @@ #include "share/restore/ob_log_archive_source_mgr.h" #include "share/ls/ob_ls_recovery_stat_operator.h"//ObLSRecoveryStatOperator #include "logservice/palf/log_define.h"//scn -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase diff --git a/src/rootserver/restore/ob_restore_scheduler.h b/src/rootserver/restore/ob_restore_scheduler.h index 62263f579..a9916b304 100644 --- a/src/rootserver/restore/ob_restore_scheduler.h +++ b/src/rootserver/restore/ob_restore_scheduler.h @@ -24,12 +24,9 @@ namespace oceanbase { -namespace palf -{ -class SCN; -} namespace share { +class SCN; class ObLSTableOperator; struct ObLSAttr; struct ObHisRestoreJobPersistInfo; @@ -57,9 +54,9 @@ public: virtual void do_work() override; void destroy(); public: - virtual palf::SCN get_rec_scn() override { return palf::SCN::max_scn();} - virtual int flush(palf::SCN &rec_scn) override { return OB_SUCCESS; } - int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const palf::SCN &scn) override + virtual share::SCN get_rec_scn() override { return share::SCN::max_scn();} + virtual int flush(share::SCN &rec_scn) override { return OB_SUCCESS; } + int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn) override { UNUSED(buffer); UNUSED(nbytes); diff --git a/src/rootserver/restore/ob_restore_util.cpp b/src/rootserver/restore/ob_restore_util.cpp index b697a4805..284cc245e 100644 --- a/src/rootserver/restore/ob_restore_util.cpp +++ b/src/rootserver/restore/ob_restore_util.cpp @@ -329,14 +329,14 @@ int ObRestoreUtil::fill_restore_scn_(const obrpc::ObPhysicalRestoreTenantArg &ar } else if (!arg.with_restore_scn_) { int64_t round_id = 0; int64_t piece_id = 0; - palf::SCN max_checkpoint_scn = palf::SCN::min_scn(); + SCN max_checkpoint_scn = SCN::min_scn(); // restore to max checkpoint scn of log ARRAY_FOREACH_X(tenant_path_array, i, cnt, OB_SUCC(ret)) { const ObString &tenant_path = tenant_path_array.at(i); ObArchiveStore store; ObBackupDest dest; ObBackupFormatDesc format_desc; - palf::SCN cur_max_checkpoint_scn = palf::SCN::min_scn(); + SCN cur_max_checkpoint_scn = SCN::min_scn(); if (OB_FAIL(dest.set(tenant_path))) { LOG_WARN("fail to set dest", K(ret), K(tenant_path)); } else if (OB_FAIL(store.init(dest))) { @@ -352,7 +352,7 @@ int ObRestoreUtil::fill_restore_scn_(const obrpc::ObPhysicalRestoreTenantArg &ar } } if (OB_SUCC(ret)) { - if (palf::SCN::min_scn() == max_checkpoint_scn) { + if (SCN::min_scn() == max_checkpoint_scn) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid max checkpoint scn, no archvie tenant path", K(ret), K(tenant_path_array)); } else { @@ -366,13 +366,13 @@ int ObRestoreUtil::fill_restore_scn_(const obrpc::ObPhysicalRestoreTenantArg &ar int ObRestoreUtil::get_restore_source( const ObIArray& tenant_path_array, const common::ObString &passwd_array, - const palf::SCN &restore_scn, + const SCN &restore_scn, ObIArray &backup_set_list, ObIArray &backup_piece_list, ObIArray &log_path_list) { int ret = OB_SUCCESS; - palf::SCN restore_start_scn = palf::SCN::min_scn(); + SCN restore_start_scn = SCN::min_scn(); if (OB_FAIL(get_restore_backup_set_array_(tenant_path_array, passwd_array, restore_scn, restore_start_scn, backup_set_list))) { LOG_WARN("fail to get restore backup set array", K(ret), K(tenant_path_array), K(restore_scn)); @@ -392,8 +392,8 @@ int ObRestoreUtil::get_restore_source( int ObRestoreUtil::get_restore_backup_set_array_( const ObIArray &tenant_path_array, const common::ObString &passwd_array, - const palf::SCN &restore_scn, - palf::SCN &restore_start_scn, + const SCN &restore_scn, + SCN &restore_start_scn, ObIArray &backup_set_list) { int ret = OB_SUCCESS; @@ -472,8 +472,8 @@ int ObRestoreUtil::get_restore_log_path_list_( int ObRestoreUtil::get_restore_log_piece_array_( const ObIArray &tenant_path_array, - const palf::SCN &restore_start_scn, - const palf::SCN &restore_end_scn, + const SCN &restore_start_scn, + const SCN &restore_end_scn, ObIArray &backup_piece_list, ObIArray &log_path_list) { diff --git a/src/rootserver/restore/ob_restore_util.h b/src/rootserver/restore/ob_restore_util.h index 7d2b266bf..c0d12738c 100644 --- a/src/rootserver/restore/ob_restore_util.h +++ b/src/rootserver/restore/ob_restore_util.h @@ -53,7 +53,7 @@ public: static int get_restore_source( const ObIArray& tenant_path_array, const common::ObString &passwd_array, - const palf::SCN &restore_scn, + const share::SCN &restore_scn, ObIArray &backup_set_list, ObIArray &backup_piece_list, ObIArray &log_path_list); @@ -82,13 +82,13 @@ private: static int get_restore_backup_set_array_( const ObIArray &tenant_path_array, const common::ObString &passwd_array, - const palf::SCN &restore_scn, - palf::SCN &restore_start_scn, + const share::SCN &restore_scn, + share::SCN &restore_start_scn, ObIArray &backup_set_list); static int get_restore_log_piece_array_( const ObIArray &tenant_path_array, - const palf::SCN &restore_start_scn, - const palf::SCN &restore_end_scn, + const share::SCN &restore_start_scn, + const share::SCN &restore_end_scn, ObIArray &backup_piece_list, ObIArray &log_path_list); static int get_restore_backup_piece_list_( diff --git a/src/share/CMakeLists.txt b/src/share/CMakeLists.txt index a8a2446e0..017000b3c 100644 --- a/src/share/CMakeLists.txt +++ b/src/share/CMakeLists.txt @@ -180,6 +180,7 @@ ob_set_subtarget(ob_share common ob_inner_kv_table_operator.cpp ob_inner_table_operator.cpp ob_primary_standby_service.cpp + scn.cpp ) ob_set_subtarget(ob_share common_mixed diff --git a/src/share/ash/ob_active_sess_hist_task.cpp b/src/share/ash/ob_active_sess_hist_task.cpp index 06eb9a368..79ee93114 100644 --- a/src/share/ash/ob_active_sess_hist_task.cpp +++ b/src/share/ash/ob_active_sess_hist_task.cpp @@ -90,6 +90,7 @@ bool ObActiveSessHistTask::operator()(sql::ObSQLSessionMgr::Key key, ObSQLSessio stat.tenant_id_ = sess_info->get_effective_tenant_id(); stat.user_id_ = sess_info->get_user_id(); stat.session_id_ = sess_info->get_sessid(); + stat.plan_id_ = sess_info->get_current_plan_id(); stat.trace_id_ = sess_info->get_current_trace_id(); sess_info->get_cur_sql_id(stat.sql_id_, sizeof(stat.sql_id_)); ObActiveSessHistList::get_instance().add(stat); diff --git a/src/share/backup/ob_archive_checkpoint.cpp b/src/share/backup/ob_archive_checkpoint.cpp index 59a795e38..03af54d7d 100644 --- a/src/share/backup/ob_archive_checkpoint.cpp +++ b/src/share/backup/ob_archive_checkpoint.cpp @@ -39,14 +39,14 @@ void ObDestRoundCheckpointer::Counter::reset() interrupted_cnt_ = 0; doing_cnt_ = 0; stopped_cnt_ = 0; - max_scn_ = palf::SCN::min_scn(); - checkpoint_scn_ = palf::SCN::max_scn(); + max_scn_ = SCN::min_scn(); + checkpoint_scn_ = SCN::max_scn(); max_active_piece_id_ = INT64_MAX; } int ObDestRoundCheckpointer::init(ObArchiveRoundHandler *round_handler, const PieceGeneratedCb &piece_generated_cb, - const RoundCheckpointCb &round_checkpoint_cb, const palf::SCN &max_checkpoint_scn) + const RoundCheckpointCb &round_checkpoint_cb, const SCN &max_checkpoint_scn) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -55,7 +55,7 @@ int ObDestRoundCheckpointer::init(ObArchiveRoundHandler *round_handler, const Pi } else if (OB_ISNULL(round_handler)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("round_handler is null.", K(ret)); - } else if (palf::SCN::min_scn() >= max_checkpoint_scn) { + } else if (SCN::min_scn() >= max_checkpoint_scn) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid max_checkpoint_scn.", K(ret), K(max_checkpoint_scn)); } else { @@ -165,7 +165,7 @@ int ObDestRoundCheckpointer::gen_new_round_info_(const ObTenantArchiveRoundAttr int ret = OB_SUCCESS; // Current existing log stream count. int64_t actual_count = counter.ls_count_ - counter.deleted_ls_count_; - palf::SCN next_checkpoint_scn = palf::SCN::min_scn(); + SCN next_checkpoint_scn = SCN::min_scn(); need_checkpoint = true; if (OB_FAIL(new_round_info.deep_copy_from(old_round_info))) { LOG_WARN("failed to deep copy round info", K(ret), K(old_round_info), K(counter)); @@ -329,7 +329,7 @@ int ObDestRoundCheckpointer::generate_one_piece_(const ObTenantArchiveRoundAttr // stat data amount and checkpoint ts for current piece. const ObArray &ls_round_list = summary.ls_round_list_; - piece.piece_info_.max_scn_ = palf::SCN::min_scn(); + piece.piece_info_.max_scn_ = SCN::min_scn(); for (int64_t i = 0; OB_SUCC(ret) && i < ls_round_list.count(); i++) { const ObLSDestRoundSummary &ls_round = ls_round_list.at(i); // search the piece diff --git a/src/share/backup/ob_archive_checkpoint.h b/src/share/backup/ob_archive_checkpoint.h index ede4dd1b2..902397795 100644 --- a/src/share/backup/ob_archive_checkpoint.h +++ b/src/share/backup/ob_archive_checkpoint.h @@ -30,8 +30,8 @@ public: struct GeneratedLSPiece { ObLSID ls_id_; - palf::SCN start_scn_; - palf::SCN checkpoint_scn_; + SCN start_scn_; + SCN checkpoint_scn_; uint64_t min_lsn_; uint64_t max_lsn_; int64_t input_bytes_; @@ -63,7 +63,7 @@ public: ObDestRoundCheckpointer() : is_inited_(false), round_handler_(nullptr), max_checkpoint_scn_() {} int init(ObArchiveRoundHandler *round_handler, const PieceGeneratedCb &piece_generated_cb, - const RoundCheckpointCb &round_checkpoint_cb, const palf::SCN &max_checkpoint_scn); + const RoundCheckpointCb &round_checkpoint_cb, const SCN &max_checkpoint_scn); // This operation is allowed only if dest round is in BEGINNING/DOING/STOPPING state. int checkpoint(const ObTenantArchiveRoundAttr &round_info, const ObDestRoundSummary &summary); @@ -85,9 +85,9 @@ private: // First interrupt ls id. ObLSID interrupted_ls_id_; // The fastest log stream archived scn. - palf::SCN max_scn_; + SCN max_scn_; // The slowest log stream archived scn. - palf::SCN checkpoint_scn_; + SCN checkpoint_scn_; // Piece id which all log stream have generated. It is the max common piece id // for them. int64_t max_active_piece_id_; @@ -119,7 +119,7 @@ private: bool is_inited_; ObArchiveRoundHandler *round_handler_; - palf::SCN max_checkpoint_scn_; + SCN max_checkpoint_scn_; PieceGeneratedCb piece_generated_cb_; RoundCheckpointCb round_checkpoint_cb_; DISALLOW_COPY_AND_ASSIGN(ObDestRoundCheckpointer); diff --git a/src/share/backup/ob_archive_path.cpp b/src/share/backup/ob_archive_path.cpp index 1502020b1..3b1d24159 100644 --- a/src/share/backup/ob_archive_path.cpp +++ b/src/share/backup/ob_archive_path.cpp @@ -120,7 +120,7 @@ int ObArchivePathUtil::get_pieces_dir_path(const ObBackupDest &dest, ObBackupPat // // oss://archive/pieces/piece_d[dest_id]r[round_id]p[piece_id]_start_20220601T120000 int ObArchivePathUtil::get_piece_start_file_path(const ObBackupDest &dest, const int64_t dest_id, - const int64_t round_id, const int64_t piece_id, const palf::SCN &start_scn, + const int64_t round_id, const int64_t piece_id, const SCN &start_scn, ObBackupPath &path) { int ret = OB_SUCCESS; @@ -141,7 +141,7 @@ int ObArchivePathUtil::get_piece_start_file_path(const ObBackupDest &dest, const // oss://archive/pieces/piece_d[dest_id]r[round_id]p[piece_id]_end_20220601T120000 int ObArchivePathUtil::get_piece_end_file_path(const ObBackupDest &dest, const int64_t dest_id, - const int64_t round_id, const int64_t piece_id, const palf::SCN &end_scn, + const int64_t round_id, const int64_t piece_id, const SCN &end_scn, ObBackupPath &path) { int ret = OB_SUCCESS; @@ -223,7 +223,7 @@ int ObArchivePathUtil::get_piece_checkpoint_file_path(const ObBackupDest &dest, // oss://archive/piece_d[dest_id]r[round_id]p[piece_id]/piece_d[dest_id]r[round_id]p[piece_id]_20220601T120000_20220602T120000 int ObArchivePathUtil::get_piece_inner_placeholder_file_path(const ObBackupDest &dest, const int64_t dest_id, - const int64_t round_id, const int64_t piece_id, const palf::SCN &start_scn, const palf::SCN &end_scn, ObBackupPath &path) + const int64_t round_id, const int64_t piece_id, const SCN &start_scn, const SCN &end_scn, ObBackupPath &path) { int ret = OB_SUCCESS; int64_t pos = 0; diff --git a/src/share/backup/ob_archive_path.h b/src/share/backup/ob_archive_path.h index 43e08b484..e749cb3e1 100644 --- a/src/share/backup/ob_archive_path.h +++ b/src/share/backup/ob_archive_path.h @@ -51,12 +51,12 @@ public: // oss://archive/pieces/piece_d[dest_id]r[round_id]p[piece_id]_start_20220601T120000 static int get_piece_start_file_path(const ObBackupDest &dest, const int64_t dest_id, - const int64_t round_id, const int64_t piece_id, const palf::SCN &start_scn, + const int64_t round_id, const int64_t piece_id, const SCN &start_scn, ObBackupPath &path); // oss://archive/pieces/piece_d[dest_id]r[round_id]p[piece_id]_end_20220601T120000 static int get_piece_end_file_path(const ObBackupDest &dest, const int64_t dest_id, - const int64_t round_id, const int64_t piece_id, const palf::SCN &end_scn, + const int64_t round_id, const int64_t piece_id, const SCN &end_scn, ObBackupPath &path); // oss://archive/piece_d[dest_id]r[round_id]p[piece_id] @@ -77,8 +77,8 @@ public: // oss://archive/piece_d[dest_id]r[round_id]p[piece_id]/piece_d[dest_id]r[round_id]p[piece_id]_20220601T120000_20220602T120000 static int get_piece_inner_placeholder_file_path(const ObBackupDest &dest, const int64_t dest_id, - const int64_t round_id, const int64_t piece_id, const palf::SCN &start_scn, - const palf::SCN &end_scn, ObBackupPath &path); + const int64_t round_id, const int64_t piece_id, const SCN &start_scn, + const SCN &end_scn, ObBackupPath &path); // oss://archive/piece_d[dest_id]r[round_id]p[piece_id]/logstream_[ls_id] static int get_piece_ls_dir_path(const ObBackupDest &dest, const int64_t dest_id, diff --git a/src/share/backup/ob_archive_persist_helper.cpp b/src/share/backup/ob_archive_persist_helper.cpp index e4029bf80..94ef2688c 100644 --- a/src/share/backup/ob_archive_persist_helper.cpp +++ b/src/share/backup/ob_archive_persist_helper.cpp @@ -787,7 +787,7 @@ int ObArchivePersistHelper::get_frozen_pieces( return ret; } -int ObArchivePersistHelper::get_candidate_obsolete_backup_pieces(common::ObISQLClient &proxy, const palf::SCN &end_scn, +int ObArchivePersistHelper::get_candidate_obsolete_backup_pieces(common::ObISQLClient &proxy, const SCN &end_scn, const char *backup_dest_str, ObIArray &pieces) const { int ret = OB_SUCCESS; @@ -1142,8 +1142,8 @@ int ObArchivePersistHelper::do_parse_ls_archive_piece_summary_result_(sqlclient: piece.piece_id_ = 0; piece.incarnation_ = 0; piece.state_.set_invalid(); - piece.start_scn_ = palf::SCN::min_scn(); - piece.checkpoint_scn_ = palf::SCN::min_scn(); + piece.start_scn_ = SCN::min_scn(); + piece.checkpoint_scn_ = SCN::min_scn(); piece.min_lsn_ = 0; piece.max_lsn_ = 0; piece.input_bytes_ = 0; diff --git a/src/share/backup/ob_archive_persist_helper.h b/src/share/backup/ob_archive_persist_helper.h index 96c0f0590..eaa52401a 100644 --- a/src/share/backup/ob_archive_persist_helper.h +++ b/src/share/backup/ob_archive_persist_helper.h @@ -129,7 +129,7 @@ public: // Get all frozen pieces whose piece ids are smaller than `upper_piece_id`. int get_frozen_pieces(common::ObISQLClient &proxy, const int64_t dest_id, const int64_t upper_piece_id, common::ObIArray &piece_list) const; - int get_candidate_obsolete_backup_pieces(common::ObISQLClient &proxy, const palf::SCN &end_scn, + int get_candidate_obsolete_backup_pieces(common::ObISQLClient &proxy, const SCN &end_scn, const char *backup_dest_str, ObIArray &pieces) const; int insert_or_update_piece(common::ObISQLClient &proxy, const ObTenantArchivePieceAttr &piece) const; // Usually, we need do it in a transaction. diff --git a/src/share/backup/ob_archive_piece.cpp b/src/share/backup/ob_archive_piece.cpp index 66d6ab119..bc1d57856 100644 --- a/src/share/backup/ob_archive_piece.cpp +++ b/src/share/backup/ob_archive_piece.cpp @@ -19,7 +19,6 @@ namespace oceanbase namespace share { using namespace std; -using namespace palf; ObArchivePiece::ObArchivePiece() : interval_us_(0), @@ -84,7 +83,7 @@ int ObArchivePiece::get_piece_lower_limit(SCN &scn) int ret = OB_SUCCESS; int64_t ts = (piece_id_ - base_piece_id_) * interval_us_ + genesis_scn_.convert_to_ts(); if (OB_FAIL(scn.convert_from_ts(ts))) { - LOG_WARN("failed to convert_for_lsn_allocator", KPC(this), K(ret), K(ts)); + LOG_WARN("failed to convert_for_logservice", KPC(this), K(ret), K(ts)); } return ret; } diff --git a/src/share/backup/ob_archive_piece.h b/src/share/backup/ob_archive_piece.h index 063099a20..d37e65969 100644 --- a/src/share/backup/ob_archive_piece.h +++ b/src/share/backup/ob_archive_piece.h @@ -16,7 +16,7 @@ #include "lib/ob_define.h" // int64_t.. #include "lib/utility/ob_print_utils.h" // print #include -#include "logservice/palf/scn.h" //SCN +#include "share/scn.h" //SCN namespace oceanbase { @@ -52,15 +52,15 @@ public: public: ObArchivePiece(); - ObArchivePiece(const palf::SCN &scn, const int64_t interval_us, const palf::SCN &genesis_scn, const int64_t base_piece_id); + ObArchivePiece(const SCN &scn, const int64_t interval_us, const SCN &genesis_scn, const int64_t base_piece_id); ~ObArchivePiece(); public: int64_t get_piece_id() const { return piece_id_; } - int get_piece_lower_limit(palf::SCN &scn); + int get_piece_lower_limit(share::SCN &scn); bool is_valid() const; void reset(); - int set(const int64_t piece_id, const int64_t interval_us, const palf::SCN &genesis_scn, const int64_t base_piece_id); + int set(const int64_t piece_id, const int64_t interval_us, const SCN &genesis_scn, const int64_t base_piece_id); void inc(); ObArchivePiece &operator=(const ObArchivePiece &other); ObArchivePiece &operator++(); // 前置++ @@ -71,7 +71,7 @@ public: private: int64_t interval_us_; // piece时间长度 - palf::SCN genesis_scn_; // 归档基准SCN + SCN genesis_scn_; // 归档基准SCN int64_t base_piece_id_; // 基准piece id int64_t piece_id_; // piece目录 }; diff --git a/src/share/backup/ob_archive_store.cpp b/src/share/backup/ob_archive_store.cpp index 2bcecd343..aa2d250ff 100644 --- a/src/share/backup/ob_archive_store.cpp +++ b/src/share/backup/ob_archive_store.cpp @@ -34,7 +34,7 @@ ObRoundStartDesc::ObRoundStartDesc() { dest_id_ = 0; round_id_ = 0; - start_scn_ = palf::SCN::min_scn(); + start_scn_ = SCN::min_scn(); base_piece_id_ = 0; piece_switch_interval_ = 0; } @@ -56,8 +56,8 @@ ObRoundEndDesc::ObRoundEndDesc() { dest_id_ = 0; round_id_ = 0; - start_scn_ = palf::SCN::min_scn(); - checkpoint_scn_ = palf::SCN::min_scn(); + start_scn_ = SCN::min_scn(); + checkpoint_scn_ = SCN::min_scn(); base_piece_id_ = 0; piece_switch_interval_ = 0; } @@ -76,7 +76,7 @@ int ObRoundEndDesc::assign(const ObRoundStartDesc &round_start) start_scn_ = round_start.start_scn_; base_piece_id_ = round_start.base_piece_id_; piece_switch_interval_ = round_start.piece_switch_interval_; - checkpoint_scn_ = palf::SCN::min_scn(); + checkpoint_scn_ = SCN::min_scn(); return ret; } @@ -97,7 +97,7 @@ ObPieceStartDesc::ObPieceStartDesc() dest_id_ = 0; round_id_ = 0; piece_id_ = 0; - start_scn_ = palf::SCN::min_scn(); + start_scn_ = SCN::min_scn(); } bool ObPieceStartDesc::is_valid() const @@ -118,7 +118,7 @@ ObPieceEndDesc::ObPieceEndDesc() dest_id_ = 0; round_id_ = 0; piece_id_ = 0; - end_scn_ = palf::SCN::min_scn(); + end_scn_ = SCN::min_scn(); } bool ObPieceEndDesc::is_valid() const @@ -143,8 +143,8 @@ ObTenantArchivePieceInfosDesc::ObTenantArchivePieceInfosDesc() piece_id_ = 0; incarnation_ = 0; dest_no_ = -1; - start_scn_ = palf::SCN::min_scn(); - end_scn_ = palf::SCN::min_scn(); + start_scn_ = SCN::min_scn(); + end_scn_ = SCN::min_scn(); } bool ObTenantArchivePieceInfosDesc::is_valid() const @@ -183,10 +183,10 @@ ObPieceCheckpointDesc::ObPieceCheckpointDesc() round_id_ = 0; piece_id_ = 0; incarnation_ = 0; - start_scn_ = palf::SCN::min_scn(); - checkpoint_scn_ = palf::SCN::min_scn(); - max_scn_ = palf::SCN::min_scn(); - end_scn_ = palf::SCN::min_scn(); + start_scn_ = SCN::min_scn(); + checkpoint_scn_ = SCN::min_scn(); + max_scn_ = SCN::min_scn(); + end_scn_ = SCN::min_scn(); MEMSET(reserved_, 0, sizeof(reserved_)); } @@ -342,8 +342,8 @@ ObPieceInnerPlaceholderDesc::ObPieceInnerPlaceholderDesc() dest_id_ = 0; round_id_ = 0; piece_id_ = 0; - start_scn_ = palf::SCN::min_scn(); - checkpoint_scn_ = palf::SCN::min_scn(); + start_scn_ = SCN::min_scn(); + checkpoint_scn_ = SCN::min_scn(); } bool ObPieceInnerPlaceholderDesc::is_valid() const @@ -365,8 +365,8 @@ ObSingleLSInfoDesc::ObSingleLSInfoDesc() dest_id_ = 0; round_id_ = 0; piece_id_ = 0; - start_scn_ = palf::SCN::min_scn(); - checkpoint_scn_ = palf::SCN::min_scn(); + start_scn_ = SCN::min_scn(); + checkpoint_scn_ = SCN::min_scn(); min_lsn_ = 0; max_lsn_ = 0; } @@ -512,7 +512,7 @@ int ObArchiveStore::write_round_end(const int64_t dest_id, const int64_t round_i return ret; } -int ObArchiveStore::get_round_id(const int64_t dest_id, const palf::SCN &scn, int64_t &round_id) +int ObArchiveStore::get_round_id(const int64_t dest_id, const SCN &scn, int64_t &round_id) { int ret = OB_SUCCESS; ObBackupIoAdapter util; @@ -621,7 +621,7 @@ int ObArchiveStore::get_all_rounds(const int64_t dest_id, ObIArray &pieces) +int ObArchiveStore::get_piece_paths_in_range(const SCN &start_scn, const SCN &end_scn, ObIArray &pieces) { int ret = OB_SUCCESS; ObArray piece_keys; @@ -1340,7 +1340,7 @@ int ObArchiveStore:: get_file_list_in_piece(const int64_t dest_id, const int64_t } int ObArchiveStore::get_max_checkpoint_scn(const int64_t dest_id, int64_t &round_id, - int64_t &piece_id, palf::SCN &max_checkpoint_scn) + int64_t &piece_id, SCN &max_checkpoint_scn) { int ret = OB_SUCCESS; int64_t min_round_id = 0; @@ -1359,7 +1359,7 @@ int ObArchiveStore::get_max_checkpoint_scn(const int64_t dest_id, int64_t &round } int ObArchiveStore::get_round_max_checkpoint_scn(const int64_t dest_id, const int64_t round_id, - int64_t &piece_id, palf::SCN &max_checkpoint_scn) + int64_t &piece_id, SCN &max_checkpoint_scn) { int ret = OB_SUCCESS; int64_t min_piece_id = 0; @@ -1378,7 +1378,7 @@ int ObArchiveStore::get_round_max_checkpoint_scn(const int64_t dest_id, const in } int ObArchiveStore::get_piece_max_checkpoint_scn(const int64_t dest_id, const int64_t round_id, - const int64_t piece_id, palf::SCN &max_checkpoint_scn) + const int64_t piece_id, SCN &max_checkpoint_scn) { int ret = OB_SUCCESS; bool is_empty_piece = false; @@ -1389,7 +1389,7 @@ int ObArchiveStore::get_piece_max_checkpoint_scn(const int64_t dest_id, const in } else if (OB_FAIL(get_single_piece_info(dest_id, round_id, piece_id, is_empty_piece, single_piece_desc))) { LOG_WARN("failed to get single piece info", K(ret), K(dest_id), K(round_id), K(piece_id)); } else if (is_empty_piece) { - max_checkpoint_scn = palf::SCN::min_scn(); + max_checkpoint_scn = SCN::min_scn(); } else { max_checkpoint_scn = single_piece_desc.piece_.checkpoint_scn_; } @@ -1655,7 +1655,7 @@ ObArchiveStore::ObLocateRoundFilter::ObLocateRoundFilter() : is_inited_(false), store_(nullptr), scn_(), rounds_() {} -int ObArchiveStore::ObLocateRoundFilter::init(ObArchiveStore *store, const palf::SCN &scn) +int ObArchiveStore::ObLocateRoundFilter::init(ObArchiveStore *store, const SCN &scn) { int ret = OB_SUCCESS; if (IS_INIT) { diff --git a/src/share/backup/ob_archive_store.h b/src/share/backup/ob_archive_store.h index 2577c403f..61e1bf17e 100644 --- a/src/share/backup/ob_archive_store.h +++ b/src/share/backup/ob_archive_store.h @@ -71,7 +71,7 @@ struct ObRoundStartDesc final : public ObExternArchiveDesc public: int64_t dest_id_; int64_t round_id_; - palf::SCN start_scn_; // archive start time of the round + SCN start_scn_; // archive start time of the round int64_t base_piece_id_; int64_t piece_switch_interval_; // unit: us @@ -92,8 +92,8 @@ struct ObRoundEndDesc final : public ObExternArchiveDesc public: int64_t dest_id_; int64_t round_id_; - palf::SCN start_scn_; // archive start time of the round - palf::SCN checkpoint_scn_; + SCN start_scn_; // archive start time of the round + SCN checkpoint_scn_; int64_t base_piece_id_; int64_t piece_switch_interval_; // unit: ns @@ -121,7 +121,7 @@ public: int64_t dest_id_; int64_t round_id_; int64_t piece_id_; - palf::SCN start_scn_; + SCN start_scn_; ObPieceStartDesc(); @@ -141,7 +141,7 @@ public: int64_t dest_id_; int64_t round_id_; int64_t piece_id_; - palf::SCN end_scn_; + SCN end_scn_; ObPieceEndDesc(); @@ -167,8 +167,8 @@ public: int64_t incarnation_; int64_t dest_no_; ObArchiveCompatible compatible_; - palf::SCN start_scn_; - palf::SCN end_scn_; + SCN start_scn_; + SCN end_scn_; ObBackupPathString path_; // history frozen pieces, ordered by piece id desc. @@ -223,10 +223,10 @@ public: int64_t incarnation_; ObArchiveCompatible compatible_; // TODO: scn type need provide serialize method which returns fixed length. - palf::SCN start_scn_; // archive start time of the round - palf::SCN checkpoint_scn_; // archive end time of the round - palf::SCN max_scn_; - palf::SCN end_scn_; + SCN start_scn_; // archive start time of the round + SCN checkpoint_scn_; // archive end time of the round + SCN max_scn_; + SCN end_scn_; int64_t reserved_[7]; @@ -249,8 +249,8 @@ public: int64_t dest_id_; int64_t round_id_; int64_t piece_id_; - palf::SCN start_scn_; - palf::SCN checkpoint_scn_; + SCN start_scn_; + SCN checkpoint_scn_; ObPieceInnerPlaceholderDesc(); @@ -282,8 +282,8 @@ public: int64_t round_id_; int64_t piece_id_; ObLSID ls_id_; - palf::SCN start_scn_; - palf::SCN checkpoint_scn_; + SCN start_scn_; + SCN checkpoint_scn_; uint64_t min_lsn_; uint64_t max_lsn_; ObSArray filelist_; @@ -334,19 +334,19 @@ public: // oss://archive/pieces/piece_d[dest_id]r[round_id]p[piece_id]_start_20220601T120000 int is_piece_start_file_exist(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, - const palf::SCN &create_scn, bool &is_exist) const; + const SCN &create_scn, bool &is_exist) const; int read_piece_start(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, - const palf::SCN &create_scn, ObPieceStartDesc &desc) const; + const SCN &create_scn, ObPieceStartDesc &desc) const; int write_piece_start(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, - const palf::SCN &create_scn, const ObPieceStartDesc &desc) const; + const SCN &create_scn, const ObPieceStartDesc &desc) const; // oss://archive/pieces/piece_d[dest_id]r[round_id]p[piece_id]_end_20220601T120000 int is_piece_end_file_exist(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, - const palf::SCN &create_scn, bool &is_exist) const; + const SCN &create_scn, bool &is_exist) const; int read_piece_end(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, - const palf::SCN &create_scn, ObPieceEndDesc &desc) const; + const SCN &create_scn, ObPieceEndDesc &desc) const; int write_piece_end(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, - const palf::SCN &create_scn, const ObPieceEndDesc &desc) const; + const SCN &create_scn, const ObPieceEndDesc &desc) const; // oss://archive/d[dest_id]r[round_id]p[piece_id]/single_piece_info int is_single_piece_file_exist(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, bool &is_exist) const; @@ -359,10 +359,10 @@ public: int write_piece_checkpoint(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const int64_t file_id, const ObPieceCheckpointDesc &desc) const; // oss://archive/d[dest_id]r[round_id]p[piece_id]/piece_d[dest_id]r[round_id]p[piece_id]_20220601T120000_20220602T120000 - int is_piece_inner_placeholder_file_exist(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const palf::SCN &start_scn, - const palf::SCN &end_scn, bool &is_exist) const; - int read_piece_inner_placeholder(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const palf::SCN &start_scn, const palf::SCN &end_scn, ObPieceInnerPlaceholderDesc &desc) const; - int write_piece_inner_placeholder(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const palf::SCN &start_scn, const palf::SCN &end_scn, const ObPieceInnerPlaceholderDesc &desc) const; + int is_piece_inner_placeholder_file_exist(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const SCN &start_scn, + const SCN &end_scn, bool &is_exist) const; + int read_piece_inner_placeholder(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const SCN &start_scn, const SCN &end_scn, ObPieceInnerPlaceholderDesc &desc) const; + int write_piece_inner_placeholder(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const SCN &start_scn, const SCN &end_scn, const ObPieceInnerPlaceholderDesc &desc) const; // oss://archive/d[dest_id]r[round_id]p[piece_id]/[ls_id]/ls_file_info int is_single_ls_info_file_exist(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, @@ -383,7 +383,7 @@ public: int get_all_round_ids(const int64_t dest_id, ObIArray &roundid_array); // If end file not exist, set checkpoint_scn in ObRoundEndDesc to 0. int get_all_rounds(const int64_t dest_id, ObIArray &roundids); - int get_round_id(const int64_t dest_id, const palf::SCN &scn, int64_t &round_id); + int get_round_id(const int64_t dest_id, const SCN &scn, int64_t &round_id); int get_round_range(const int64_t dest_id, int64_t &min_round_id, int64_t &max_round_id); int get_piece_range(const int64_t dest_id, const int64_t round_id, int64_t &min_piece_id, int64_t &max_piece_id); @@ -399,7 +399,7 @@ public: // Get pieces needed in the specific interval indicated by 'start_scn' and 'end_scn'. // Return OB_ENTRY_NOT_EXIST if cannot find enough pieces. - int get_piece_paths_in_range(const palf::SCN &start_scn, const palf::SCN &end_scn, ObIArray &pieces); + int get_piece_paths_in_range(const SCN &start_scn, const SCN &end_scn, ObIArray &pieces); // Get archive file range in one piece // return OB_ENTRY_NOT_EXIST if no file exist @@ -408,9 +408,9 @@ public: // Get each file id and size for specific log stream under piece. int get_file_list_in_piece(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, const ObLSID &ls_id, ObIArray &filelist) const; - int get_max_checkpoint_scn(const int64_t dest_id, int64_t &round_id, int64_t &piece_id, palf::SCN &max_checkpoint_scn); - int get_round_max_checkpoint_scn(const int64_t dest_id, const int64_t round_id, int64_t &piece_id, palf::SCN &max_checkpoint_scn); - int get_piece_max_checkpoint_scn(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, palf::SCN &max_checkpoint_scn); + int get_max_checkpoint_scn(const int64_t dest_id, int64_t &round_id, int64_t &piece_id, SCN &max_checkpoint_scn); + int get_round_max_checkpoint_scn(const int64_t dest_id, const int64_t round_id, int64_t &piece_id, SCN &max_checkpoint_scn); + int get_piece_max_checkpoint_scn(const int64_t dest_id, const int64_t round_id, const int64_t piece_id, SCN &max_checkpoint_scn); private: @@ -483,7 +483,7 @@ private: public: ObLocateRoundFilter(); virtual ~ObLocateRoundFilter() {} - int init(ObArchiveStore *store, const palf::SCN &scn); + int init(ObArchiveStore *store, const SCN &scn); int func(const dirent *entry) override; ObArray &result() { return rounds_; } @@ -493,7 +493,7 @@ private: private: bool is_inited_; ObArchiveStore *store_; - palf::SCN scn_; + SCN scn_; ObArray rounds_; diff --git a/src/share/backup/ob_archive_struct.cpp b/src/share/backup/ob_archive_struct.cpp index b180eea79..46d703d6c 100644 --- a/src/share/backup/ob_archive_struct.cpp +++ b/src/share/backup/ob_archive_struct.cpp @@ -877,9 +877,9 @@ void ObLSArchivePersistInfo::reset() { key_.reset(); incarnation_ = OB_START_INCARNATION; - start_scn_ = palf::SCN::min_scn(); + start_scn_ = SCN::min_scn(); start_lsn_ = 0; - checkpoint_scn_.palf::SCN::min_scn(); + checkpoint_scn_.SCN::min_scn(); lsn_ = 0; archive_file_id_ = -1; archive_file_offset_ = -1; @@ -947,8 +947,8 @@ ObArchiveLSPieceSummary::ObArchiveLSPieceSummary() incarnation_ = OB_START_INCARNATION; state_.set_invalid(); - start_scn_ = palf::SCN::min_scn(); - checkpoint_scn_ = palf::SCN::min_scn(); + start_scn_ = SCN::min_scn(); + checkpoint_scn_ = SCN::min_scn(); min_lsn_ = 0; max_lsn_ = 0; input_bytes_ = 0; @@ -1074,7 +1074,7 @@ void ObLSDestRoundSummary::reset() ls_id_.reset(); is_deleted_ = false; state_.set_invalid(); - start_scn_ = palf::SCN::min_scn(); + start_scn_ = SCN::min_scn(); checkpoint_scn_.reset(); piece_list_.reset(); } diff --git a/src/share/backup/ob_archive_struct.h b/src/share/backup/ob_archive_struct.h index d8f9f61de..9e56afeee 100644 --- a/src/share/backup/ob_archive_struct.h +++ b/src/share/backup/ob_archive_struct.h @@ -19,7 +19,7 @@ #include "share/ob_inner_kv_table_operator.h" #include "share/backup/ob_backup_struct.h" #include "share/backup/ob_archive_compatible.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -152,9 +152,9 @@ struct ObTenantArchiveRoundAttr final : public ObIInnerTableRow int64_t dest_id_; // archive dest identification. int64_t round_id_; ObArchiveRoundState state_; - palf::SCN start_scn_; - palf::SCN checkpoint_scn_; - palf::SCN max_scn_; + SCN start_scn_; + SCN checkpoint_scn_; + SCN max_scn_; ObArchiveCompatible compatible_; int64_t base_piece_id_; @@ -179,9 +179,9 @@ struct ObTenantArchiveRoundAttr final : public ObIInnerTableRow base_piece_id_ = 0; used_piece_id_ = 0; piece_switch_interval_ = 0; - start_scn_ = palf::SCN::min_scn(); - checkpoint_scn_ = palf::SCN::min_scn(); - max_scn_ = palf::SCN::min_scn(); + start_scn_ = share::SCN::min_scn(); + checkpoint_scn_ = share::SCN::min_scn(); + max_scn_ = share::SCN::min_scn(); frozen_input_bytes_ = 0; frozen_output_bytes_ = 0; active_input_bytes_ = 0; @@ -316,9 +316,9 @@ struct ObTenantArchiveHisRoundAttr final : public ObIInnerTableRow Key key_; int64_t incarnation_; int64_t dest_id_; - palf::SCN start_scn_; - palf::SCN checkpoint_scn_; - palf::SCN max_scn_; + SCN start_scn_; + SCN checkpoint_scn_; + SCN max_scn_; ObArchiveCompatible compatible_; int64_t base_piece_id_; @@ -340,9 +340,9 @@ struct ObTenantArchiveHisRoundAttr final : public ObIInnerTableRow base_piece_id_ = 0; used_piece_id_ = 0; piece_switch_interval_ = 0; - start_scn_ = palf::SCN::min_scn(); - checkpoint_scn_ = palf::SCN::min_scn(); - max_scn_ = palf::SCN::min_scn(); + start_scn_ = share::SCN::min_scn(); + checkpoint_scn_ = share::SCN::min_scn(); + max_scn_ = share::SCN::min_scn(); input_bytes_ = 0; output_bytes_ = 0; deleted_input_bytes_ = 0; @@ -581,10 +581,10 @@ public: int64_t incarnation_; int64_t dest_no_; int64_t file_count_; - palf::SCN start_scn_; - palf::SCN checkpoint_scn_; - palf::SCN max_scn_; - palf::SCN end_scn_; + SCN start_scn_; + SCN checkpoint_scn_; + SCN max_scn_; + SCN end_scn_; ObArchiveCompatible compatible_; int64_t input_bytes_; @@ -606,10 +606,10 @@ public: output_bytes_ = 0; cp_file_id_ = 0; cp_file_offset_ = 0; - start_scn_ = palf::SCN::min_scn(); - checkpoint_scn_ = palf::SCN::min_scn(); - max_scn_ = palf::SCN::min_scn(); - end_scn_ = palf::SCN::min_scn(); + start_scn_ = share::SCN::min_scn(); + checkpoint_scn_ = share::SCN::min_scn(); + max_scn_ = share::SCN::min_scn(); + end_scn_ = share::SCN::min_scn(); } // Return if primary key valid. @@ -744,9 +744,9 @@ struct ObLSArchivePersistInfo final : public ObIInnerTableRow Key key_; int64_t incarnation_; - palf::SCN start_scn_; // piece start ts + SCN start_scn_; // piece start ts uint64_t start_lsn_; // piece start lsn - palf::SCN checkpoint_scn_; + SCN checkpoint_scn_; uint64_t lsn_; int64_t archive_file_id_; int64_t archive_file_offset_; @@ -814,8 +814,8 @@ struct ObArchiveLSPieceSummary int64_t incarnation_; ObArchiveRoundState state_; - palf::SCN start_scn_; - palf::SCN checkpoint_scn_; + SCN start_scn_; + SCN checkpoint_scn_; uint64_t min_lsn_; uint64_t max_lsn_; int64_t input_bytes_; @@ -834,8 +834,8 @@ struct ObLSDestRoundSummary struct OnePiece { int64_t piece_id_; - palf::SCN start_scn_; - palf::SCN checkpoint_scn_; + SCN start_scn_; + SCN checkpoint_scn_; uint64_t min_lsn_; uint64_t max_lsn_; int64_t input_bytes_; @@ -851,8 +851,8 @@ struct ObLSDestRoundSummary ObLSID ls_id_; bool is_deleted_; // mark deleted ls. ObArchiveRoundState state_; - palf::SCN start_scn_; - palf::SCN checkpoint_scn_; + SCN start_scn_; + SCN checkpoint_scn_; // Ordered by pieceid. common::ObArray piece_list_; diff --git a/src/share/backup/ob_backup_data_store.cpp b/src/share/backup/ob_backup_data_store.cpp index ee71df7f5..aeb95f1dd 100644 --- a/src/share/backup/ob_backup_data_store.cpp +++ b/src/share/backup/ob_backup_data_store.cpp @@ -526,8 +526,8 @@ int ObBackupDataStore::write_backup_set_placeholder( const bool is_inner, const bool is_start, const bool is_succeed, - const palf::SCN &replay_scn, - const palf::SCN &min_restore_scn) + const SCN &replay_scn, + const SCN &min_restore_scn) { int ret = OB_SUCCESS; ObExternBackupSetPlaceholderDesc placeholder; @@ -672,8 +672,8 @@ int ObBackupDataStore::get_backup_set_placeholder_path_( const bool is_inner, const bool is_start, const bool is_succeed, - const palf::SCN &replay_scn, - const palf::SCN &min_restore_scn, + const SCN &replay_scn, + const SCN &min_restore_scn, share::ObBackupPath &path) { int ret = OB_SUCCESS; @@ -704,8 +704,8 @@ int ObBackupDataStore::get_backup_set_placeholder_path_( int ObBackupDataStore::get_backup_set_array( const common::ObString &passwd_array, - const palf::SCN &restore_scn, - palf::SCN &restore_start_scn, + const SCN &restore_scn, + SCN &restore_start_scn, common::ObIArray &backup_set_list) { int ret = OB_SUCCESS; @@ -734,9 +734,9 @@ int ObBackupDataStore::get_backup_set_array( } int ObBackupDataStore::do_get_backup_set_array_(const common::ObString &passwd_array, - const palf::SCN &restore_scn, const ObBackupSetFilter &op, + const SCN &restore_scn, const ObBackupSetFilter &op, common::ObIArray &tmp_backup_set_list, - int64_t &cur_max_backup_set_id, palf::SCN &restore_start_scn) + int64_t &cur_max_backup_set_id, SCN &restore_start_scn) { int ret = OB_SUCCESS; const int64_t OB_BACKUP_MAX_BACKUP_SET_ID = 5000; diff --git a/src/share/backup/ob_backup_data_store.h b/src/share/backup/ob_backup_data_store.h index 645fbea44..8c8877476 100644 --- a/src/share/backup/ob_backup_data_store.h +++ b/src/share/backup/ob_backup_data_store.h @@ -59,7 +59,7 @@ public: bool is_valid() const override; INHERIT_TO_STRING_KV("ObExternBackupDataDesc", ObExternBackupDataDesc, K(backup_scn_), K_(ls_attr_array)); public: - palf::SCN backup_scn_; + SCN backup_scn_; ObSArray ls_attr_array_; private: DISALLOW_COPY_AND_ASSIGN(ObBackupDataLSAttrDesc); @@ -97,7 +97,7 @@ public: bool is_valid() const override; INHERIT_TO_STRING_KV("ObExternBackupDataDesc", ObExternBackupDataDesc, K_(backup_scn), K_(tablet_to_ls)); public: - palf::SCN backup_scn_; + SCN backup_scn_; ObSArray tablet_to_ls_; private: DISALLOW_COPY_AND_ASSIGN(ObBackupDataTabletToLSDesc); @@ -279,7 +279,7 @@ public: // write backup set place holder int write_backup_set_placeholder(const bool is_inner, const bool is_start, const bool is_succeed, - const palf::SCN &replay_scn, const palf::SCN &min_restore_scn); + const SCN &replay_scn, const SCN &min_restore_scn); // write and read tenant locality info int write_tenant_locality_info(const ObExternTenantLocalityInfoDesc &locality_info); @@ -293,8 +293,8 @@ public: int write_backup_set_info(const ObExternBackupSetInfoDesc &backup_set_info); int read_backup_set_info(ObExternBackupSetInfoDesc &backup_set_info); - int get_backup_set_array(const common::ObString &passwd_array, const palf::SCN &restore_scn, - palf::SCN &restore_start_scn, common::ObIArray &backup_set_list); + int get_backup_set_array(const common::ObString &passwd_array, const SCN &restore_scn, + SCN &restore_start_scn, common::ObIArray &backup_set_list); int get_max_sys_ls_retry_id(const share::ObBackupPath &backup_path, const ObLSID &ls_id, int64_t &retry_id); TO_STRING_KV(K_(backup_desc)); @@ -312,11 +312,11 @@ public: } }; private: - int do_get_backup_set_array_(const common::ObString &passwd_array, const palf::SCN &restore_scn, + int do_get_backup_set_array_(const common::ObString &passwd_array, const SCN &restore_scn, const ObBackupSetFilter &op, common::ObIArray &tmp_backup_set_list, - int64_t &cur_max_backup_set_id, palf::SCN &restore_start_scn); + int64_t &cur_max_backup_set_id, SCN &restore_start_scn); int get_backup_set_placeholder_path_(const bool is_inner, const bool is_start, const bool is_succeed, - const palf::SCN &replay_scn, const palf::SCN &min_restore_scn, share::ObBackupPath &path); + const SCN &replay_scn, const SCN &min_restore_scn, share::ObBackupPath &path); private: share::ObBackupSetDesc backup_desc_; share::ObBackupDest backup_set_dest_; diff --git a/src/share/backup/ob_backup_data_table_operator.cpp b/src/share/backup/ob_backup_data_table_operator.cpp index dc47b1136..be4d6a44e 100644 --- a/src/share/backup/ob_backup_data_table_operator.cpp +++ b/src/share/backup/ob_backup_data_table_operator.cpp @@ -1325,7 +1325,7 @@ int ObBackupTaskOperator::advance_task_status( const ObBackupSetTaskAttr &set_task_attr, const ObBackupStatus &next_status, const int result, - const palf::SCN &end_scn, + const SCN &end_scn, const int64_t end_ts) { int ret = OB_SUCCESS; @@ -1471,7 +1471,7 @@ int ObBackupTaskOperator::update_meta_turn_id(common::ObISQLClient &proxy, const } int ObBackupTaskOperator::update_user_ls_start_scn(common::ObISQLClient &proxy, const int64_t task_id, - const uint64_t tenant_id, const palf::SCN &scn) + const uint64_t tenant_id, const SCN &scn) { int ret = OB_SUCCESS; ObSqlString sql; diff --git a/src/share/backup/ob_backup_data_table_operator.h b/src/share/backup/ob_backup_data_table_operator.h index 557ce2ed5..cfadc2e08 100644 --- a/src/share/backup/ob_backup_data_table_operator.h +++ b/src/share/backup/ob_backup_data_table_operator.h @@ -127,14 +127,14 @@ public: static int get_backup_task(common::ObISQLClient &proxy, const int64_t job_id, const uint64_t tenant_id, ObBackupSetTaskAttr &set_task_attr); static int advance_task_status(common::ObISQLClient &proxy, const ObBackupSetTaskAttr &set_task_attr, - const ObBackupStatus &next_status, const int result, const palf::SCN &end_scn, const int64_t end_ts); + const ObBackupStatus &next_status, const int result, const SCN &end_scn, const int64_t end_ts); static int move_task_to_his(common::ObISQLClient &proxy, const uint64_t tenant_id, const int64_t job_id); static int update_stats(common::ObISQLClient &proxy, const int64_t task_id, const uint64_t tenant_id, const ObBackupStats &stats); static int update_meta_turn_id(common::ObISQLClient &proxy, const int64_t task_id, const uint64_t tenant_id, const int64_t turn_id); static int update_user_ls_start_scn(common::ObISQLClient &proxy, const int64_t task_id, const uint64_t tenant_id, - const palf::SCN &scn); + const SCN &scn); private: static int fill_dml_with_backup_task_(const ObBackupSetTaskAttr &backup_set_task, ObDMLSqlSplicer &dml); }; diff --git a/src/share/backup/ob_backup_path.cpp b/src/share/backup/ob_backup_path.cpp index 9a59f995f..58a7d1cb4 100644 --- a/src/share/backup/ob_backup_path.cpp +++ b/src/share/backup/ob_backup_path.cpp @@ -415,7 +415,7 @@ bool ObBackupPath::operator ==(const ObBackupPath &path) const int ObBackupPathUtil::get_tenant_data_backup_set_placeholder_path_( const uint64_t backup_set_id, const ObBackupType backup_type, - const palf::SCN &min_restore_scn, + const SCN &min_restore_scn, const ObString &suffix, ObBackupPath &path) { @@ -471,7 +471,7 @@ int ObBackupPathUtil::get_backup_set_placeholder_start_path( int ObBackupPathUtil::get_backup_set_placeholder_end_success_path( const share::ObBackupDest &backup_tenant_dest, const share::ObBackupSetDesc &backup_set_desc, - const palf::SCN &min_restore_scn, share::ObBackupPath &backup_path) + const SCN &min_restore_scn, share::ObBackupPath &backup_path) { int ret = OB_SUCCESS; ObString suffix("end_success"); @@ -486,7 +486,7 @@ int ObBackupPathUtil::get_backup_set_placeholder_end_success_path( int ObBackupPathUtil::get_backup_set_placeholder_end_failed_path( const share::ObBackupDest &backup_tenant_dest, const share::ObBackupSetDesc &backup_set_desc, - const palf::SCN &min_restore_scn, share::ObBackupPath &backup_path) + const SCN &min_restore_scn, share::ObBackupPath &backup_path) { int ret = OB_SUCCESS; ObString suffix("end_failed"); @@ -545,7 +545,7 @@ int ObBackupPathUtil::get_backup_set_inner_placeholder_prefix( // file:///obbackup/backup_set_1_full/backup_set_1_full_xxxx_xxxxx int ObBackupPathUtil::get_backup_set_inner_placeholder( const share::ObBackupDest &backup_set_dest, const share::ObBackupSetDesc &backup_set_desc, - const palf::SCN &replay_scn, const palf::SCN &min_restore_scn, + const SCN &replay_scn, const SCN &min_restore_scn, share::ObBackupPath &backup_path) { int ret = OB_SUCCESS; diff --git a/src/share/backup/ob_backup_path.h b/src/share/backup/ob_backup_path.h index fa4db9bba..c02283d52 100644 --- a/src/share/backup/ob_backup_path.h +++ b/src/share/backup/ob_backup_path.h @@ -74,12 +74,12 @@ struct ObBackupPathUtil // file:///backup/backup_sets/backup_set_1_[full|inc]_end_success_20220601T120000 static int get_backup_set_placeholder_end_success_path(const share::ObBackupDest &backup_tenant_dest, const share::ObBackupSetDesc &backup_set_desc, - const palf::SCN &min_restore_scn, share::ObBackupPath &backup_path); + const SCN &min_restore_scn, share::ObBackupPath &backup_path); // file:///backup/backup_sets/backup_set_1_[full|inc]_end_failed_20220601T120000 static int get_backup_set_placeholder_end_failed_path(const share::ObBackupDest &backup_tenant_dest, const share::ObBackupSetDesc &backup_set_desc, - const palf::SCN &min_restore_scn, share::ObBackupPath &backup_path); + const SCN &min_restore_scn, share::ObBackupPath &backup_path); // file:///obbackup/backup_set_1_full/ static int get_backup_set_dir_path(const share::ObBackupDest &backup_set_dest, @@ -95,8 +95,8 @@ struct ObBackupPathUtil // file:///obbackup/backup_set_1_full/backup_set_1_full_xxxx_xxxxx static int get_backup_set_inner_placeholder(const share::ObBackupDest &backup_set_dest, - const share::ObBackupSetDesc &backup_set_desc, const palf::SCN &replay_scn, - const palf::SCN &min_restore_scn, share::ObBackupPath &backup_path); + const share::ObBackupSetDesc &backup_set_desc, const SCN &replay_scn, + const SCN &min_restore_scn, share::ObBackupPath &backup_path); // file:///obbackup/backup_set_1_full/log_stream_1/ static int get_ls_backup_dir_path(const share::ObBackupDest &backup_set_dest, @@ -263,7 +263,7 @@ private: static int get_tenant_data_backup_set_placeholder_path_( const uint64_t backup_set_id, const ObBackupType backup_type, - const palf::SCN &min_restore_scn, + const SCN &min_restore_scn, const ObString &suffix, share::ObBackupPath &path); }; diff --git a/src/share/backup/ob_backup_struct.cpp b/src/share/backup/ob_backup_struct.cpp index 7581a5dd6..77831e1b5 100644 --- a/src/share/backup/ob_backup_struct.cpp +++ b/src/share/backup/ob_backup_struct.cpp @@ -3517,8 +3517,8 @@ void ObBackupSetFileDesc::reset() passwd_.reset(); file_status_ = ObBackupFileStatus::BACKUP_FILE_MAX; backup_path_.reset(); - start_replay_scn_ = palf::SCN::min_scn(); - min_restore_scn_ = palf::SCN::min_scn(); + start_replay_scn_ = SCN::min_scn(); + min_restore_scn_ = SCN::min_scn(); tenant_compatible_ = 0; backup_compatible_ = Compatible::MAX_COMPATIBLE_VERSION; data_turn_id_ = 0; @@ -4058,7 +4058,7 @@ int share::trim_right_backslash(ObBackupPathString &path) } // Convert a scn to time string. -int share::backup_scn_to_strftime(const palf::SCN &scn, char *buf, +int share::backup_scn_to_strftime(const SCN &scn, char *buf, const int64_t buf_len, int64_t &pos, const char concat) { int ret = OB_SUCCESS; @@ -4085,7 +4085,7 @@ int share::backup_scn_to_strftime(const palf::SCN &scn, char *buf, return ret; } -int share::backup_scn_to_time_tag(const palf::SCN &scn, char *buf, const int64_t buf_len, int64_t &pos) +int share::backup_scn_to_time_tag(const SCN &scn, char *buf, const int64_t buf_len, int64_t &pos) { int ret = OB_SUCCESS; if (OB_FAIL(share::backup_scn_to_strftime(scn, buf, buf_len, pos, 'T'/* concat */))) { diff --git a/src/share/backup/ob_backup_struct.h b/src/share/backup/ob_backup_struct.h index f774555db..b34dcbab6 100644 --- a/src/share/backup/ob_backup_struct.h +++ b/src/share/backup/ob_backup_struct.h @@ -27,7 +27,7 @@ #include "share/ob_encryption_util.h" #include "share/schema/ob_schema_getter_guard.h" #include "share/ob_dml_sql_splicer.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -1302,9 +1302,9 @@ public: int64_t backup_set_id_; int64_t start_ts_; int64_t end_ts_; - palf::SCN start_scn_; - palf::SCN end_scn_; - palf::SCN user_ls_start_scn_; + SCN start_scn_; + SCN end_scn_; + SCN user_ls_start_scn_; int64_t data_turn_id_; int64_t meta_turn_id_; ObBackupStatus status_; @@ -1432,8 +1432,8 @@ public: common::ObFixedLengthString passwd_; ObBackupFileStatus::STATUS file_status_; common::ObFixedLengthString backup_path_; - palf::SCN start_replay_scn_; - palf::SCN min_restore_scn_; + SCN start_replay_scn_; + SCN min_restore_scn_; uint64_t tenant_compatible_; Compatible backup_compatible_; int64_t data_turn_id_; @@ -1592,12 +1592,12 @@ struct ObLogArchiveDestAtrr final int trim_right_backslash(ObBackupPathString &path); // Convert a scn to time string, return like '2022-05-31 12:00:00' if concat is ' '. -int backup_scn_to_strftime(const palf::SCN &scn, char *buf, const int64_t buf_len, int64_t &pos, const char concat); +int backup_scn_to_strftime(const SCN &scn, char *buf, const int64_t buf_len, int64_t &pos, const char concat); // Convert a scn to time tag, return like '20220531T120000' -int backup_scn_to_time_tag(const palf::SCN &scn, char *buf, const int64_t buf_len, int64_t &pos); +int backup_scn_to_time_tag(const SCN &scn, char *buf, const int64_t buf_len, int64_t &pos); -inline uint64_t trans_scn_to_second(const palf::SCN &scn) { return scn.convert_to_ts() / 1000 / 1000; } +inline uint64_t trans_scn_to_second(const SCN &scn) { return scn.convert_to_ts() / 1000 / 1000; } }//share }//oceanbase diff --git a/src/share/backup/ob_tenant_archive_mgr.cpp b/src/share/backup/ob_tenant_archive_mgr.cpp index d9fad8896..513225cc2 100644 --- a/src/share/backup/ob_tenant_archive_mgr.cpp +++ b/src/share/backup/ob_tenant_archive_mgr.cpp @@ -71,10 +71,10 @@ int ObTenantArchiveMgr::is_archive_running( // piece op int ObTenantArchiveMgr::decide_piece_id( - const palf::SCN &piece_start_scn, + const SCN &piece_start_scn, const int64_t start_piece_id, const int64_t piece_switch_interval, - const palf::SCN &scn, + const SCN &scn, int64_t &piece_id) { int ret = OB_SUCCESS; @@ -95,17 +95,17 @@ int ObTenantArchiveMgr::decide_piece_id( } int ObTenantArchiveMgr::decide_piece_start_scn( - const palf::SCN &piece_start_scn, + const SCN &piece_start_scn, const int64_t start_piece_id, const int64_t piece_switch_interval, const int64_t piece_id, - palf::SCN &start_scn) + SCN &start_scn) { int ret = OB_SUCCESS; if (piece_id < start_piece_id) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid piece id", K(ret), K(piece_start_scn), K(start_piece_id), K(piece_switch_interval), K(piece_id)); - } else if (palf::SCN::min_scn() >= piece_start_scn) { + } else if (SCN::min_scn() >= piece_start_scn) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid piece_start_scn", K(ret), K(piece_start_scn), K(start_piece_id), K(piece_switch_interval), K(piece_id)); } else if (0 >= piece_switch_interval) { @@ -119,11 +119,11 @@ int ObTenantArchiveMgr::decide_piece_start_scn( } int ObTenantArchiveMgr::decide_piece_end_scn( - const palf::SCN &piece_start_scn, + const SCN &piece_start_scn, const int64_t start_piece_id, const int64_t piece_switch_interval, const int64_t piece_id, - palf::SCN &end_scn) + SCN &end_scn) { int ret = OB_SUCCESS; // piece end scn is the start of next piece. @@ -135,9 +135,9 @@ int ObTenantArchiveMgr::decide_piece_end_scn( } int ObTenantArchiveMgr::decide_first_piece_start_scn( - const palf::SCN &start_scn, + const SCN &start_scn, const int64_t piece_switch_interval, - palf::SCN &piece_start_scn) + SCN &piece_start_scn) { int ret = OB_SUCCESS; diff --git a/src/share/backup/ob_tenant_archive_mgr.h b/src/share/backup/ob_tenant_archive_mgr.h index 44f5e56cd..2d959e9bf 100644 --- a/src/share/backup/ob_tenant_archive_mgr.h +++ b/src/share/backup/ob_tenant_archive_mgr.h @@ -39,27 +39,27 @@ public: // piece op static int decide_piece_id( - const palf::SCN &piece_start_scn, + const SCN &piece_start_scn, const int64_t start_piece_id, const int64_t piece_switch_interval, - const palf::SCN &scn, + const SCN &scn, int64_t &piece_id); static int decide_piece_start_scn( - const palf::SCN & piece_start_scn, + const SCN & piece_start_scn, const int64_t start_piece_id, const int64_t piece_switch_interval, const int64_t piece_id, - palf::SCN &start_scn); + SCN &start_scn); static int decide_piece_end_scn( - const palf::SCN &piece_start_scn, + const SCN &piece_start_scn, const int64_t start_piece_id, const int64_t piece_switch_interval, const int64_t piece_id, - palf::SCN &end_scn); + SCN &end_scn); static int decide_first_piece_start_scn( - const palf::SCN &start_scn, + const SCN &start_scn, const int64_t piece_switch_interval, - palf::SCN &piece_start_scn); + SCN &piece_start_scn); // 'ts' must be with unit us. static int timestamp_to_day(const int64_t ts, int64_t &day); }; diff --git a/src/share/backup/ob_tenant_archive_round.cpp b/src/share/backup/ob_tenant_archive_round.cpp index 057a640df..67a0a13a8 100644 --- a/src/share/backup/ob_tenant_archive_round.cpp +++ b/src/share/backup/ob_tenant_archive_round.cpp @@ -21,7 +21,7 @@ #include "share/ob_tenant_info_proxy.h" #include "rootserver/ob_rs_event_history_table_operator.h" #include "share/ls/ob_ls_i_life_manager.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" using namespace oceanbase; using namespace share; @@ -99,7 +99,7 @@ int ObArchiveRoundHandler::prepare_beginning_dest_round_(const ObTenantArchiveRo { int ret = OB_SUCCESS; - palf::SCN start_scn = palf::SCN::min_scn(); + SCN start_scn = SCN::min_scn(); ObArchiveRoundState next_state = ObArchiveRoundState::beginning(); if (!round.state_.is_prepare()) { ret = OB_INVALID_ARGUMENT; @@ -131,7 +131,7 @@ bool ObArchiveRoundHandler::can_stop_archive(const ObTenantArchiveRoundAttr &rou } -int ObArchiveRoundHandler::decide_start_scn_(palf::SCN &start_scn) +int ObArchiveRoundHandler::decide_start_scn_(SCN &start_scn) { int ret = OB_SUCCESS; ObAllTenantInfo tenant_info; @@ -139,7 +139,7 @@ int ObArchiveRoundHandler::decide_start_scn_(palf::SCN &start_scn) if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id_, sql_proxy_, for_update, tenant_info))) { LOG_WARN("failed to get tenant info", K(ret), K_(tenant_id)); } else if (OB_FALSE_IT(start_scn = tenant_info.get_standby_scn())){ - } else if (palf::SCN::base_scn() >= start_scn) { + } else if (SCN::base_scn() >= start_scn) { ret = OB_EAGAIN; LOG_WARN("start_scn not valid， need wait", K(ret), K(tenant_info)); } diff --git a/src/share/backup/ob_tenant_archive_round.h b/src/share/backup/ob_tenant_archive_round.h index 345d44f26..112bcf8ac 100644 --- a/src/share/backup/ob_tenant_archive_round.h +++ b/src/share/backup/ob_tenant_archive_round.h @@ -64,7 +64,7 @@ private: int start_trans_(common::ObMySQLTransaction &trans); int prepare_new_dest_round_(const int64_t dest_no, ObMySQLTransaction &trans, ObTenantArchiveRoundAttr &round); int prepare_beginning_dest_round_(const ObTenantArchiveRoundAttr &round, ObTenantArchiveRoundAttr &new_round); - int decide_start_scn_(palf::SCN &start_ts); + int decide_start_scn_(share::SCN &start_ts); bool is_inited_; uint64_t tenant_id_; // user tenant id diff --git a/src/share/inner_table/ob_inner_table_schema.12301_12350.cpp b/src/share/inner_table/ob_inner_table_schema.12301_12350.cpp index 219b55f66..f74c3e745 100644 --- a/src/share/inner_table/ob_inner_table_schema.12301_12350.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12301_12350.cpp @@ -969,6 +969,21 @@ int ObInnerTableSchema::all_virtual_ash_schema(ObTableSchema &table_schema) true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PLAN_ID", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); diff --git a/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp b/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp index 9f70d38d2..3ae7b890a 100644 --- a/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp @@ -12801,6 +12801,21 @@ int ObInnerTableSchema::all_virtual_ash_ora_schema(ObTableSchema &table_schema) true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PLAN_ID", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); diff --git a/src/share/inner_table/ob_inner_table_schema.21251_21300.cpp b/src/share/inner_table/ob_inner_table_schema.21251_21300.cpp index f0c3e428c..64ea22781 100644 --- a/src/share/inner_table/ob_inner_table_schema.21251_21300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21251_21300.cpp @@ -909,7 +909,7 @@ int ObInnerTableSchema::gv_active_session_history_schema(ObTableSchema &table_sc table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(SVR_IP AS CHAR(46)) AS SVR_IP, CAST(SVR_PORT AS SIGNED) AS SVR_PORT, CAST(SAMPLE_ID AS SIGNED) AS SAMPLE_ID, CAST(SAMPLE_TIME AS DATETIME) AS SAMPLE_TIME, CAST(TENANT_ID AS SIGNED) AS CON_ID, CAST(USER_ID AS SIGNED) AS USER_ID, CAST(SESSION_ID AS SIGNED) AS SESSION_ID, CAST(IF (SESSION_TYPE = 0, 'FOREGROUND', 'BACKGROUND') AS CHAR(10)) AS SESSION_TYPE, CAST(IF (EVENT_NO = 0, 'ON CPU', 'WAITING') AS CHAR(7)) AS SESSION_STATE, CAST(SQL_ID AS CHAR(32)) AS SQL_ID, CAST(TRACE_ID AS CHAR(64)) AS TRACE_ID, CAST(NAME AS CHAR(64)) AS EVENT, CAST(EVENT_NO AS SIGNED) AS EVENT_NO, CAST(PARAMETER1 AS CHAR(64)) AS P1TEXT, CAST(P1 AS SIGNED) AS P1, CAST(PARAMETER2 AS CHAR(64)) AS P2TEXT, CAST(P2 AS SIGNED) AS P2, CAST(PARAMETER3 AS CHAR(64)) AS P3TEXT, CAST(P3 AS SIGNED) AS P3, CAST(WAIT_CLASS AS CHAR(64)) AS WAIT_CLASS, CAST(WAIT_CLASS_ID AS SIGNED) AS WAIT_CLASS_ID, CAST(TIME_WAITED AS SIGNED) AS TIME_WAITED, CAST(SQL_PLAN_LINE_ID AS SIGNED) SQL_PLAN_LINE_ID, CAST(IF (IN_PARSE = 1, 'Y', 'N') AS CHAR(1)) AS IN_PARSE, CAST(IF (IN_PL_PARSE = 1, 'Y', 'N') AS CHAR(1)) AS IN_PL_PARSE, CAST(IF (IN_PLAN_CACHE = 1, 'Y', 'N') AS CHAR(1)) AS IN_PLAN_CACHE, CAST(IF (IN_SQL_OPTIMIZE = 1, 'Y', 'N') AS CHAR(1)) AS IN_SQL_OPTIMIZE, CAST(IF (IN_SQL_EXECUTION = 1, 'Y', 'N') AS CHAR(1)) AS IN_SQL_EXECUTION, CAST(IF (IN_PX_EXECUTION = 1, 'Y', 'N') AS CHAR(1)) AS IN_PX_EXECUTION, CAST(IF (IN_SEQUENCE_LOAD = 1, 'Y', 'N') AS CHAR(1)) AS IN_SEQUENCE_LOAD, CAST(MODULE AS CHAR(64)) AS MODULE, CAST(ACTION AS CHAR(64)) AS ACTION, CAST(CLIENT_ID AS CHAR(64)) AS CLIENT_ID, CAST(BACKTRACE AS CHAR(512)) AS BACKTRACE FROM oceanbase.__all_virtual_ash LEFT JOIN oceanbase.v$event_name on EVENT_NO = `event#` )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(SVR_IP AS CHAR(46)) AS SVR_IP, CAST(SVR_PORT AS SIGNED) AS SVR_PORT, CAST(SAMPLE_ID AS SIGNED) AS SAMPLE_ID, CAST(SAMPLE_TIME AS DATETIME) AS SAMPLE_TIME, CAST(TENANT_ID AS SIGNED) AS CON_ID, CAST(USER_ID AS SIGNED) AS USER_ID, CAST(SESSION_ID AS SIGNED) AS SESSION_ID, CAST(IF (SESSION_TYPE = 0, 'FOREGROUND', 'BACKGROUND') AS CHAR(10)) AS SESSION_TYPE, CAST(IF (EVENT_NO = 0, 'ON CPU', 'WAITING') AS CHAR(7)) AS SESSION_STATE, CAST(SQL_ID AS CHAR(32)) AS SQL_ID, CAST(PLAN_ID AS SIGNED) AS PLAN_ID, CAST(TRACE_ID AS CHAR(64)) AS TRACE_ID, CAST(NAME AS CHAR(64)) AS EVENT, CAST(EVENT_NO AS SIGNED) AS EVENT_NO, CAST(PARAMETER1 AS CHAR(64)) AS P1TEXT, CAST(P1 AS SIGNED) AS P1, CAST(PARAMETER2 AS CHAR(64)) AS P2TEXT, CAST(P2 AS SIGNED) AS P2, CAST(PARAMETER3 AS CHAR(64)) AS P3TEXT, CAST(P3 AS SIGNED) AS P3, CAST(WAIT_CLASS AS CHAR(64)) AS WAIT_CLASS, CAST(WAIT_CLASS_ID AS SIGNED) AS WAIT_CLASS_ID, CAST(TIME_WAITED AS SIGNED) AS TIME_WAITED, CAST(SQL_PLAN_LINE_ID AS SIGNED) SQL_PLAN_LINE_ID, CAST(IF (IN_PARSE = 1, 'Y', 'N') AS CHAR(1)) AS IN_PARSE, CAST(IF (IN_PL_PARSE = 1, 'Y', 'N') AS CHAR(1)) AS IN_PL_PARSE, CAST(IF (IN_PLAN_CACHE = 1, 'Y', 'N') AS CHAR(1)) AS IN_PLAN_CACHE, CAST(IF (IN_SQL_OPTIMIZE = 1, 'Y', 'N') AS CHAR(1)) AS IN_SQL_OPTIMIZE, CAST(IF (IN_SQL_EXECUTION = 1, 'Y', 'N') AS CHAR(1)) AS IN_SQL_EXECUTION, CAST(IF (IN_PX_EXECUTION = 1, 'Y', 'N') AS CHAR(1)) AS IN_PX_EXECUTION, CAST(IF (IN_SEQUENCE_LOAD = 1, 'Y', 'N') AS CHAR(1)) AS IN_SEQUENCE_LOAD, CAST(MODULE AS CHAR(64)) AS MODULE, CAST(ACTION AS CHAR(64)) AS ACTION, CAST(CLIENT_ID AS CHAR(64)) AS CLIENT_ID, CAST(BACKTRACE AS CHAR(512)) AS BACKTRACE FROM oceanbase.__all_virtual_ash LEFT JOIN oceanbase.v$event_name on EVENT_NO = `event#` )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp b/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp index a21e23d01..384d966b5 100644 --- a/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp @@ -2009,7 +2009,7 @@ int ObInnerTableSchema::gv_active_session_history_ora_schema(ObTableSchema &tabl table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(SVR_IP AS VARCHAR2(46)) AS SVR_IP, CAST(SVR_PORT AS NUMBER) AS SVR_PORT, CAST(SAMPLE_ID AS NUMBER) AS SAMPLE_ID, CAST(SAMPLE_TIME AS TIMESTAMP) AS SAMPLE_TIME, CAST(TENANT_ID AS NUMBER) AS CON_ID, CAST(USER_ID AS NUMBER) AS USER_ID, CAST(SESSION_ID AS NUMBER) AS SESSION_ID, CAST(DECODE(SESSION_TYPE, 0, 'FOREGROUND', 'BACKGROUND') AS VARCHAR2(10)) AS SESSION_TYPE, CAST(DECODE(EVENT_NO, 0, 'ON CPU', 'WAITING') AS VARCHAR2(7)) AS SESSION_STATE, CAST(SQL_ID AS VARCHAR(32)) AS SQL_ID, CAST(TRACE_ID AS VARCHAR(64)) AS TRACE_ID, CAST(NAME AS VARCHAR2(64)) AS EVENT, CAST(EVENT_NO AS NUMBER) AS EVENT_NO, CAST(PARAMETER1 AS VARCHAR2(64)) AS P1TEXT, CAST(P1 AS NUMBER) AS P1, CAST(PARAMETER2 AS VARCHAR2(64)) AS P2TEXT, CAST(P2 AS NUMBER) AS P2, CAST(PARAMETER3 AS VARCHAR2(64)) AS P3TEXT, CAST(P3 AS NUMBER) AS P3, CAST(WAIT_CLASS AS VARCHAR2(64)) AS WAIT_CLASS, CAST(WAIT_CLASS_ID AS NUMBER) AS WAIT_CLASS_ID, CAST(TIME_WAITED AS NUMBER) AS TIME_WAITED, CAST(SQL_PLAN_LINE_ID AS NUMBER) SQL_PLAN_LINE_ID, CAST(DECODE(IN_PARSE, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_PARSE, CAST(DECODE(IN_PL_PARSE, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_PL_PARSE, CAST(DECODE(IN_PLAN_CACHE, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_PLAN_CACHE, CAST(DECODE(IN_SQL_OPTIMIZE, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_SQL_OPTIMIZE, CAST(DECODE(IN_SQL_EXECUTION, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_SQL_EXECUTION, CAST(DECODE(IN_PX_EXECUTION, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_PX_EXECUTION, CAST(DECODE(IN_SEQUENCE_LOAD, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_SEQUENCE_LOAD, CAST(MODULE AS VARCHAR2(64)) AS MODULE, CAST(ACTION AS VARCHAR2(64)) AS ACTION, CAST(CLIENT_ID AS VARCHAR2(64)) AS CLIENT_ID, CAST(BACKTRACE AS VARCHAR2(512)) AS BACKTRACE FROM SYS.ALL_VIRTUAL_ASH LEFT JOIN SYS.V$EVENT_NAME on EVENT_NO = "EVENT#" )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__(SELECT CAST(SVR_IP AS VARCHAR2(46)) AS SVR_IP, CAST(SVR_PORT AS NUMBER) AS SVR_PORT, CAST(SAMPLE_ID AS NUMBER) AS SAMPLE_ID, CAST(SAMPLE_TIME AS TIMESTAMP) AS SAMPLE_TIME, CAST(TENANT_ID AS NUMBER) AS CON_ID, CAST(USER_ID AS NUMBER) AS USER_ID, CAST(SESSION_ID AS NUMBER) AS SESSION_ID, CAST(DECODE(SESSION_TYPE, 0, 'FOREGROUND', 'BACKGROUND') AS VARCHAR2(10)) AS SESSION_TYPE, CAST(DECODE(EVENT_NO, 0, 'ON CPU', 'WAITING') AS VARCHAR2(7)) AS SESSION_STATE, CAST(SQL_ID AS VARCHAR(32)) AS SQL_ID, CAST(PLAN_ID AS NUMBER) AS PLAN_ID, CAST(TRACE_ID AS VARCHAR(64)) AS TRACE_ID, CAST(NAME AS VARCHAR2(64)) AS EVENT, CAST(EVENT_NO AS NUMBER) AS EVENT_NO, CAST(PARAMETER1 AS VARCHAR2(64)) AS P1TEXT, CAST(P1 AS NUMBER) AS P1, CAST(PARAMETER2 AS VARCHAR2(64)) AS P2TEXT, CAST(P2 AS NUMBER) AS P2, CAST(PARAMETER3 AS VARCHAR2(64)) AS P3TEXT, CAST(P3 AS NUMBER) AS P3, CAST(WAIT_CLASS AS VARCHAR2(64)) AS WAIT_CLASS, CAST(WAIT_CLASS_ID AS NUMBER) AS WAIT_CLASS_ID, CAST(TIME_WAITED AS NUMBER) AS TIME_WAITED, CAST(SQL_PLAN_LINE_ID AS NUMBER) SQL_PLAN_LINE_ID, CAST(DECODE(IN_PARSE, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_PARSE, CAST(DECODE(IN_PL_PARSE, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_PL_PARSE, CAST(DECODE(IN_PLAN_CACHE, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_PLAN_CACHE, CAST(DECODE(IN_SQL_OPTIMIZE, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_SQL_OPTIMIZE, CAST(DECODE(IN_SQL_EXECUTION, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_SQL_EXECUTION, CAST(DECODE(IN_PX_EXECUTION, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_PX_EXECUTION, CAST(DECODE(IN_SEQUENCE_LOAD, 1, 'Y', 'N') AS VARCHAR2(1)) AS IN_SEQUENCE_LOAD, CAST(MODULE AS VARCHAR2(64)) AS MODULE, CAST(ACTION AS VARCHAR2(64)) AS ACTION, CAST(CLIENT_ID AS VARCHAR2(64)) AS CLIENT_ID, CAST(BACKTRACE AS VARCHAR2(512)) AS BACKTRACE FROM SYS.ALL_VIRTUAL_ASH LEFT JOIN SYS.V$EVENT_NAME on EVENT_NO = "EVENT#" )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 233f7f31b..28feb42c7 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -10382,7 +10382,8 @@ def_table_schema( ('MODULE', 'varchar:64', 'true'), ('ACTION', 'varchar:64', 'true'), ('CLIENT_ID', 'varchar:64', 'true'), - ('BACKTRACE', 'varchar:512', 'true') + ('BACKTRACE', 'varchar:512', 'true'), + ('PLAN_ID', 'int') ], partition_columns = ['SVR_IP', 'SVR_PORT'], vtable_route_policy = 'distributed', @@ -20790,6 +20791,7 @@ def_table_schema( CAST(IF (SESSION_TYPE = 0, 'FOREGROUND', 'BACKGROUND') AS CHAR(10)) AS SESSION_TYPE, CAST(IF (EVENT_NO = 0, 'ON CPU', 'WAITING') AS CHAR(7)) AS SESSION_STATE, CAST(SQL_ID AS CHAR(32)) AS SQL_ID, + CAST(PLAN_ID AS SIGNED) AS PLAN_ID, CAST(TRACE_ID AS CHAR(64)) AS TRACE_ID, CAST(NAME AS CHAR(64)) AS EVENT, CAST(EVENT_NO AS SIGNED) AS EVENT_NO, @@ -44239,6 +44241,7 @@ def_table_schema( CAST(DECODE(SESSION_TYPE, 0, 'FOREGROUND', 'BACKGROUND') AS VARCHAR2(10)) AS SESSION_TYPE, CAST(DECODE(EVENT_NO, 0, 'ON CPU', 'WAITING') AS VARCHAR2(7)) AS SESSION_STATE, CAST(SQL_ID AS VARCHAR(32)) AS SQL_ID, + CAST(PLAN_ID AS NUMBER) AS PLAN_ID, CAST(TRACE_ID AS VARCHAR(64)) AS TRACE_ID, CAST(NAME AS VARCHAR2(64)) AS EVENT, CAST(EVENT_NO AS NUMBER) AS EVENT_NO, diff --git a/src/share/inner_table/sys_package/dbms_ash_internal.sql b/src/share/inner_table/sys_package/dbms_ash_internal.sql new file mode 100644 index 000000000..e6c078578 --- /dev/null +++ b/src/share/inner_table/sys_package/dbms_ash_internal.sql @@ -0,0 +1,13 @@ +-- package_name:dbms_ash_internal +-- author:xiaochu.yh + + +CREATE OR REPLACE PACKAGE dbms_ash_internal AUTHID CURRENT_USER AS + + FUNCTION IN_MEMORY_ASH_VIEW_SQL + RETURN VARCHAR2; + + FUNCTION ASH_VIEW_SQL + RETURN VARCHAR2; + +END dbms_ash_internal; diff --git a/src/share/inner_table/sys_package/dbms_ash_internal_body.sql b/src/share/inner_table/sys_package/dbms_ash_internal_body.sql new file mode 100644 index 000000000..df236c5c7 --- /dev/null +++ b/src/share/inner_table/sys_package/dbms_ash_internal_body.sql @@ -0,0 +1,69 @@ +-- package_name:dbms_ash_internal +-- author:xiaochu.yh + +CREATE OR REPLACE PACKAGE BODY dbms_ash_internal AS + + FUNCTION IN_MEMORY_ASH_VIEW_SQL + RETURN VARCHAR2 + IS + RETVAL VARCHAR2(30000); + BEGIN + RETVAL := 'SELECT a.sample_id, a.sample_time, ' || + ' a.svr_ip, ' || + ' a.svr_port, ' || + ' a.con_id, ' || + ' a.user_id, ' || + ' a.session_id, ' || + ' a.session_type, ' || + ' a.session_state, ' || + ' a.sql_id, ' || + ' plan_id, ' || + ' a.trace_id, ' || + ' nvl(a.event, ''CPU + Wait for CPU'') as event, ' || + ' nvl(a.event_no, 1) as event_no, ' || + ' a.p1, a.p1text, ' || + ' a.p2, a.p2text, ' || + ' a.p3, a.p3text, ' || + ' nvl(a.wait_class, ''CPU'') as wait_class, ' || + ' nvl(a.wait_class_id, 9999) as wait_class_id, ' || + ' a.time_waited, ' || + ' a.sql_plan_line_id, ' || + ' a.in_parse, ' || + ' a.in_pl_parse, ' || + ' a.in_plan_cache, ' || + ' a.in_sql_optimize, ' || + ' a.in_sql_execution, ' || + ' a.in_px_execution, ' || + ' a.in_sequence_load, ' || + ' a.module, a.action, a.client_id ' || + 'FROM GV$ACTIVE_SESSION_HISTORY a ' || + 'WHERE 1=1 '; + RETURN RETVAL; + END IN_MEMORY_ASH_VIEW_SQL; + + FUNCTION ASH_VIEW_SQL + RETURN VARCHAR2 + IS + RETVAL VARCHAR2(30000); + BEGIN + RETVAL := 'SELECT * FROM (' || IN_MEMORY_ASH_VIEW_SQL || + ' and a.sample_time between :ash_mem_btime and :ash_mem_etime ' || + ' ) unified_ash ' || + 'WHERE sample_time between :ash_begin_time ' || + ' and :ash_end_time ' || + ' AND (:ash_sql_id IS NULL ' || + ' OR sql_id like :ash_sql_id) ' || + ' AND (:ash_trace_id IS NULL ' || + ' OR trace_id like :ash_trace_id) ' || + ' AND (:ash_wait_class IS NULL ' || + ' OR wait_class like :ash_wait_class) ' || + ' AND (:ash_module IS NULL ' || + ' OR module like :ash_module) ' || + ' AND (:ash_action IS NULL ' || + ' OR action like :ash_action) ' || + ' AND (:ash_client_id IS NULL ' || + ' OR client_id like :ash_client_id)'; + RETURN RETVAL; + END ASH_VIEW_SQL; + +END dbms_ash_internal; diff --git a/src/share/inner_table/sys_package/dbms_workload_repository.sql b/src/share/inner_table/sys_package/dbms_workload_repository.sql new file mode 100644 index 000000000..5036ad3eb --- /dev/null +++ b/src/share/inner_table/sys_package/dbms_workload_repository.sql @@ -0,0 +1,33 @@ +-- package_name:dbms_workload_repository +-- author:xiaochu.yh + + +-- +-- 完整 PL 包参考文档：https://docs.oracle.com/database/121/ARPLS/d_workload_repos.htm#ARPLS093 +-- +CREATE OR REPLACE PACKAGE dbms_workload_repository AUTHID CURRENT_USER AS + + -- Type declare + -- SUBTYPE OUTPUT_TYPE IS VARCHAR2(4000 CHAR); + TYPE awrrpt_text_type_table IS TABLE OF VARCHAR2(4096 CHAR) INDEX BY BINARY_INTEGER; + RPT_ROWS awrrpt_text_type_table; + + + TYPE SEC_REC IS RECORD( + TITLE VARCHAR2(4000 CHAR), + TITLE_TYPE VARCHAR2(1 CHAR) + ); + + TYPE SEC_REC_TAB IS TABLE OF SEC_REC + INDEX BY BINARY_INTEGER; + + TEST_ROW SEC_REC_TAB; + + FUNCTION ash_report_text(l_btime IN DATE, + l_etime IN DATE, + l_sql_id IN VARCHAR2 DEFAULT NULL, + l_wait_class IN VARCHAR2 DEFAULT NULL + ) + RETURN awrrpt_text_type_table; + +END dbms_workload_repository; diff --git a/src/share/inner_table/sys_package/dbms_workload_repository_body.sql b/src/share/inner_table/sys_package/dbms_workload_repository_body.sql new file mode 100644 index 000000000..131f0ad19 --- /dev/null +++ b/src/share/inner_table/sys_package/dbms_workload_repository_body.sql @@ -0,0 +1,591 @@ +--package_name:dbms_workload_repository +--author:xiaochu.yh + +CREATE OR REPLACE PACKAGE BODY dbms_workload_repository AS + +TYPE COLUMN_CONTENT_ARRAY IS VARRAY(20) OF VARCHAR2(4096); +TYPE COLUMN_WIDTH_ARRAY IS VARRAY(20) OF INTEGER; + +DIG_3_FM VARCHAR2(20) := 'FM999999990.000'; +DIG_2_FM VARCHAR2(20) := 'FM999999990.00'; + +-- helper functions +PROCEDURE APPEND_ROW(ROW IN VARCHAR2) +IS +BEGIN + RPT_ROWS(RPT_ROWS.COUNT) := ROW; +END APPEND_ROW; + +PROCEDURE REPORT_CLEANUP +IS +BEGIN + IF (RPT_ROWS.COUNT > 0) THEN + RPT_ROWS.DELETE; + END IF; +END REPORT_CLEANUP; + +FUNCTION FORMAT_ROW(column_content IN COLUMN_CONTENT_ARRAY, + column_width IN COLUMN_WIDTH_ARRAY, + pad IN VARCHAR2, + sep IN VARCHAR2) +RETURN VARCHAR2 +IS + RES VARCHAR2(4000 CHAR); +BEGIN + RES := ''; + FOR i IN 1 .. column_content.count LOOP + RES := RES || LPAD(column_content(i), column_width(i), pad) || sep; + END LOOP; + RETURN RES; +END FORMAT_ROW; + + +-- main function +FUNCTION ASH_REPORT_TEXT(l_btime IN DATE, + l_etime IN DATE, + l_sql_id IN VARCHAR2 DEFAULT NULL, + l_wait_class IN VARCHAR2 DEFAULT NULL + ) +RETURN awrrpt_text_type_table +IS + DYN_SQL VARCHAR2(15000); + NULL_NUM NUMBER := NULL; + NULL_CHAR VARCHAR2(10) := NULL; + + TYPE TopEventCursor IS REF CURSOR; + top_event_cv TopEventCursor; + + TYPE TopEventRecord IS RECORD ( + EVENT SYS.V$ACTIVE_SESSION_HISTORY.EVENT%TYPE, + WAIT_CLASS SYS.V$ACTIVE_SESSION_HISTORY.WAIT_CLASS%TYPE, + EVENT_CNT NUMBER + ); + top_event_rec TopEventRecord; + + TYPE TopEventPvalRecord IS RECORD ( + EVENT SYS.V$ACTIVE_SESSION_HISTORY.EVENT%TYPE, + EVENT_CNT NUMBER, + SAMPLE_CNT NUMBER, + P1 SYS.V$ACTIVE_SESSION_HISTORY.P1%TYPE, + P2 SYS.V$ACTIVE_SESSION_HISTORY.P2%TYPE, + P3 SYS.V$ACTIVE_SESSION_HISTORY.P3%TYPE, + P1TEXT SYS.V$ACTIVE_SESSION_HISTORY.P1TEXT%TYPE, + P2TEXT SYS.V$ACTIVE_SESSION_HISTORY.P2TEXT%TYPE, + P3TEXT SYS.V$ACTIVE_SESSION_HISTORY.P3TEXT%TYPE + ); + top_event_pval_rec TopEventPvalRecord; + + TYPE TopAppInfoRecord IS RECORD ( + MODULE SYS.V$ACTIVE_SESSION_HISTORY.MODULE%TYPE, + ACTION SYS.V$ACTIVE_SESSION_HISTORY.ACTION%TYPE, + SAMPLE_CNT NUMBER + ); + top_appinfo_rec TopAppInfoRecord; + + TYPE TopPhaseOfExecutionRecord IS RECORD ( + EXECUTION_PHASE VARCHAR2(40), + SAMPLE_CNT NUMBER + ); + top_phase_rec TopPhaseOfExecutionRecord; + + TYPE TopSQLRecord IS RECORD ( + SQL_ID SYS.V$ACTIVE_SESSION_HISTORY.SQL_ID%TYPE, + PLAN_ID NUMBER, + EVENT_CNT NUMBER, + EVENT SYS.V$ACTIVE_SESSION_HISTORY.EVENT%TYPE, + QUERY_SQL SYS.V$OB_PLAN_CACHE_PLAN_STAT.QUERY_SQL%TYPE + ); + top_sql_rec TopSQLRecord; + + TYPE CompleteSQLRecord IS RECORD ( + SQL_ID SYS.V$ACTIVE_SESSION_HISTORY.SQL_ID%TYPE, + QUERY_SQL SYS.V$OB_PLAN_CACHE_PLAN_STAT.QUERY_SQL%TYPE + ); + complete_sql_rec CompleteSQLRecord; + + + TYPE TopSessionRecord IS RECORD ( + SESSION_ID SYS.V$ACTIVE_SESSION_HISTORY.SESSION_ID%TYPE, + EVENT SYS.V$ACTIVE_SESSION_HISTORY.EVENT%TYPE, + EVENT_CNT NUMBER, + SAMPLE_CNT NUMBER, + USER_NAME SYS.ALL_USERS.USERNAME%TYPE + ); + top_sess_rec TopSessionRecord; + + TYPE TopLatchRecord IS RECORD ( + EVENT SYS.V$ACTIVE_SESSION_HISTORY.EVENT%TYPE, + SAMPLE_CNT NUMBER + ); + top_latch_rec TopLatchRecord; + + + + + column_content COLUMN_CONTENT_ARRAY; + column_widths COLUMN_WIDTH_ARRAY; + + ASH_END_TIME Date; + ASH_BEGIN_TIME Date; + DUR_ELAPSED Number; + NUM_SAMPLES Number; + NUM_EVENTS Number; -- One event may cross many samples + + FILTER_EVENT_STR CONSTANT VARCHAR2(100) := 'CASE WHEN wait_class_id = 100 OR TIME_WAITED != 0 THEN 1 ELSE 0 END'; +BEGIN + REPORT_CLEANUP(); + + DBMS_OUTPUT.PUT_LINE(''); + DBMS_OUTPUT.PUT_LINE('# ' || l_sql_id); + DBMS_OUTPUT.PUT_LINE(''); + + DYN_SQL := 'SELECT MIN(SAMPLE_TIME) ASH_BEGIN_TIME, MAX(SAMPLE_TIME) ASH_END_TIME, COUNT(1) NUM_SAMPLES, SUM(' || FILTER_EVENT_STR || ') NUM_EVENTS ' || + 'FROM (' || DBMS_ASH_INTERNAL.ASH_VIEW_SQL || ') top_event '; + EXECUTE IMMEDIATE DYN_SQL + INTO ASH_BEGIN_TIME, ASH_END_TIME, NUM_SAMPLES, NUM_EVENTS + USING L_BTIME, L_ETIME, + L_BTIME, L_ETIME, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR; + DUR_ELAPSED := (ASH_END_TIME - ASH_BEGIN_TIME) * 24 * 60 * 60; -- in seconds + IF DUR_ELAPSED <= 0 THEN + DUR_ELAPSED := 1; -- avoid zero division + END IF; + IF NUM_SAMPLES <= 0 THEN + NUM_SAMPLES := 1; + END IF; + IF NUM_EVENTS <= 0 THEN + NUM_EVENTS := 1; + END IF; + + APPEND_ROW('----'); + APPEND_ROW(' Sample Begin: ' || TO_CHAR(L_BTIME, 'yyyy-mm-dd HH24:MI:SS')); + APPEND_ROW(' Sample End: ' || TO_CHAR(L_ETIME, 'yyyy-mm-dd HH24:MI:SS')); + APPEND_ROW(' ----------'); + APPEND_ROW(' Analysis Begin Time: ' || TO_CHAR(ASH_BEGIN_TIME, 'yyyy-mm-dd HH24:MI:SS')); + APPEND_ROW(' Analysis End Time: ' || TO_CHAR(ASH_END_TIME, 'yyyy-mm-dd HH24:MI:SS')); + APPEND_ROW(' Elapsed Time: ' || TO_CHAR(DUR_ELAPSED) || '(secs)'); + APPEND_ROW(' Num of Sample: ' || TO_CHAR(NUM_SAMPLES)); + APPEND_ROW(' Num of Events: ' || TO_CHAR(NUM_EVENTS)); + APPEND_ROW('Average Active Sessions: ' || TO_CHAR(ROUND(NUM_SAMPLES/DUR_ELAPSED,2), DIG_3_FM)); + APPEND_ROW('----'); + + APPEND_ROW(' '); + APPEND_ROW('## Top User Events:'); + column_widths := COLUMN_WIDTH_ARRAY(40, 20, 10, 9); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + column_content := COLUMN_CONTENT_ARRAY('Event', 'WAIT_CLASS', 'EVENT_CNT', '% Event'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, ' ', '|')); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + DYN_SQL := 'SELECT EVENT, WAIT_CLASS, COUNT(1) EVENT_CNT FROM (' || DBMS_ASH_INTERNAL.ASH_VIEW_SQL || ') top_event ' || 'GROUP BY EVENT, WAIT_CLASS'; + OPEN top_event_cv FOR DYN_SQL + USING L_BTIME, L_ETIME, + L_BTIME, L_ETIME, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR; + LOOP + FETCH top_event_cv INTO top_event_rec; + EXIT WHEN top_event_cv%NOTFOUND; + APPEND_ROW(FORMAT_ROW(COLUMN_CONTENT_ARRAY( + top_event_rec.EVENT, + top_event_rec.WAIT_CLASS, + TO_CHAR(top_event_rec.EVENT_CNT), + TO_CHAR(ROUND(100 * top_event_rec.EVENT_CNT/NUM_EVENTS,2), DIG_2_FM) || '%' + ), column_widths, ' ', '|')); + END LOOP; + CLOSE top_event_cv; + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + + + APPEND_ROW(' '); + APPEND_ROW('## Top Events P1/P2/P3 Value:'); + column_widths := COLUMN_WIDTH_ARRAY(40, 10, 12, 50, 20, 20, 20); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + column_content := COLUMN_CONTENT_ARRAY('Event', '% Event', '% Activity', 'Max P1/P2/P3', 'Parameter 1', 'Parameter 2', 'Parameter 3'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, ' ', '|')); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + DYN_SQL := 'SELECT * FROM (SELECT EVENT, SUM(' || FILTER_EVENT_STR || ') EVENT_CNT, COUNT(1) SAMPLE_CNT, MAX(P1) P1, MAX(P2) P2, MAX(P3) P3, MAX(P1TEXT) P1TEXT, MAX(P2TEXT) P2TEXT, MAX(P3TEXT) P3TEXT ' || + 'FROM (' || DBMS_ASH_INTERNAL.ASH_VIEW_SQL || ') top_event ' || + 'GROUP BY EVENT, WAIT_CLASS ORDER BY 2 DESC) WHERE ROWNUM < 10'; + OPEN top_event_cv FOR DYN_SQL + USING L_BTIME, L_ETIME, + L_BTIME, L_ETIME, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR; + LOOP + FETCH top_event_cv INTO top_event_pval_rec; + EXIT WHEN top_event_cv%NOTFOUND; + APPEND_ROW( + FORMAT_ROW( + COLUMN_CONTENT_ARRAY( + top_event_pval_rec.EVENT, + TO_CHAR(ROUND(100 * top_event_pval_rec.EVENT_CNT/NUM_EVENTS,2), DIG_2_FM) || '%', + TO_CHAR(ROUND(100 * top_event_pval_rec.SAMPLE_CNT/DUR_ELAPSED,3), DIG_3_FM) || '%', + '"' || TO_CHAR(top_event_pval_rec.P1) || '","' || TO_CHAR(top_event_pval_rec.P2) || '","' || TO_CHAR(top_event_pval_rec.P3) || '"', + NVL(top_event_pval_rec.P1TEXT, ' '), + NVL(top_event_pval_rec.P2TEXT, ' '), + NVL(top_event_pval_rec.P3TEXT, ' ') + ), + column_widths, ' ', '|' + ) + ); + END LOOP; + CLOSE top_event_cv; + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + + + -- Not implemented yet + -- APPEND_ROW(' '); + -- APPEND_ROW('## Top Service/Module:'); + -- column_widths := COLUMN_WIDTH_ARRAY(40, 40, 12, 40, 12); + -- column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-'); + -- APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + -- column_content := COLUMN_CONTENT_ARRAY('Service', 'Module', '% Activity', 'Action', '% Action'); + -- APPEND_ROW(FORMAT_ROW(column_content, column_widths, ' ', '|')); + -- column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-'); + -- APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + -- DYN_SQL := 'SELECT * FROM (SELECT MODULE, ACTION, COUNT(1) SAMPLE_CNT ' || + -- 'FROM (' || DBMS_ASH_INTERNAL.ASH_VIEW_SQL || ') top_event ' || + -- 'GROUP BY MODULE, ROLLUP(ACTION) ORDER BY MODULE, SAMPLE_CNT DESC) ' || + -- 'WHERE SAMPLE_CNT / :num_samples_param > -0.01'; + -- OPEN top_event_cv FOR DYN_SQL + -- USING L_BTIME, L_ETIME, + -- L_BTIME, L_ETIME, + -- NULL_CHAR, NULL_CHAR, + -- NULL_CHAR, NULL_CHAR, + -- NULL_CHAR, NULL_CHAR, + -- NULL_CHAR, NULL_CHAR, + -- NULL_CHAR, NULL_CHAR, + -- NULL_CHAR, NULL_CHAR, + -- NUM_SAMPLES; + -- LOOP + -- FETCH top_event_cv INTO top_appinfo_rec; + -- EXIT WHEN top_event_cv%NOTFOUND; + -- APPEND_ROW( + -- FORMAT_ROW( + -- COLUMN_CONTENT_ARRAY( + -- '*', + -- NVL(top_appinfo_rec.MODULE, ' '), + -- TO_CHAR(ROUND(100 * top_appinfo_rec.SAMPLE_CNT/DUR_ELAPSED, 3), DIG_3_FM) || '%', + -- NVL(top_appinfo_rec.ACTION, ' '), + -- TO_CHAR(ROUND(100 * top_appinfo_rec.SAMPLE_CNT/NUM_SAMPLES,2), DIG_2_FM) || '%' + -- ), + -- column_widths, ' ', '|' + -- ) + -- ); + -- END LOOP; + -- CLOSE top_event_cv; + -- column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-'); + -- APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + + + + APPEND_ROW(' '); + APPEND_ROW('## Top Phase of Execution:'); + column_widths := COLUMN_WIDTH_ARRAY(40, 12, 14, 40); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + column_content := COLUMN_CONTENT_ARRAY('Phase of Execution', '% Activity', 'Sample Count', 'Avg Active Sessions'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, ' ', '|')); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + DYN_SQL := 'SELECT * FROM (SELECT ' || + ' SUM(CASE IN_PARSE WHEN ''N'' THEN 0 ELSE 1 END) IN_PARSE, ' || + ' SUM(CASE IN_PL_PARSE WHEN ''N'' THEN 0 ELSE 1 END) IN_PL_PARSE, ' || + ' SUM(CASE IN_PLAN_CACHE WHEN ''N'' THEN 0 ELSE 1 END) IN_PLAN_CACHE, ' || + ' SUM(CASE IN_SQL_OPTIMIZE WHEN ''N'' THEN 0 ELSE 1 END) IN_SQL_OPTIMIZE, ' || + ' SUM(CASE IN_SQL_EXECUTION WHEN ''N'' THEN 0 ELSE 1 END) IN_SQL_EXECUTION, ' || + ' SUM(CASE IN_PX_EXECUTION WHEN ''N'' THEN 0 ELSE 1 END) IN_PX_EXECUTION, ' || + ' SUM(CASE IN_SEQUENCE_LOAD WHEN ''N'' THEN 0 ELSE 1 END) IN_SEQUENCE_LOAD ' || + 'FROM (' || DBMS_ASH_INTERNAL.ASH_VIEW_SQL || ') top_event ) phases ' || + ' unpivot ' || + ' (' || + ' SAMPLES_CNT FOR EXECUTION_PHASE IN (IN_PARSE, IN_PL_PARSE, IN_PLAN_CACHE, IN_SQL_OPTIMIZE, IN_SQL_EXECUTION,IN_PX_EXECUTION, IN_SEQUENCE_LOAD )' || + ' ) ORDER BY SAMPLES_CNT DESC'; + OPEN top_event_cv FOR DYN_SQL + USING L_BTIME, L_ETIME, + L_BTIME, L_ETIME, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR; + LOOP + FETCH top_event_cv INTO top_phase_rec; + EXIT WHEN top_event_cv%NOTFOUND; + APPEND_ROW( + FORMAT_ROW( + COLUMN_CONTENT_ARRAY( + top_phase_rec.EXECUTION_PHASE, + TO_CHAR(top_phase_rec.SAMPLE_CNT), + TO_CHAR(ROUND(100 * top_phase_rec.SAMPLE_CNT/NUM_SAMPLES, 3), DIG_3_FM) || '%', + TO_CHAR(ROUND(top_phase_rec.SAMPLE_CNT/DUR_ELAPSED,2), DIG_2_FM) + ), + column_widths, ' ', '|' + ) + ); + END LOOP; + CLOSE top_event_cv; + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + + + + APPEND_ROW(' '); + APPEND_ROW('## Top SQL with Top Events'); + APPEND_ROW(' - All events included.'); + APPEND_ROW(' - Empty ''SQL Text'' if it is PL/SQL query'); + column_widths := COLUMN_WIDTH_ARRAY(40, 12, 25, 40, 12, 60); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + column_content := COLUMN_CONTENT_ARRAY('SQL ID', 'PLAN ID', 'Sampled # of Executions', 'Event', '% Event', 'SQL Text'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, ' ', '|')); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + DYN_SQL := 'SELECT SQL_ID, PLAN_ID, EVENT_CNT, EVENT, QUERY_SQL FROM (SELECT ash.*, SUBSTR(TRIM(REPLACE(pc.QUERY_SQL, CHR(10), '''')), 0, 55) QUERY_SQL ' || + 'FROM (SELECT SQL_ID, PLAN_ID, SUM(' || FILTER_EVENT_STR || ') EVENT_CNT, EVENT FROM (' || + DBMS_ASH_INTERNAL.ASH_VIEW_SQL || + ') top_event GROUP BY SQL_ID, PLAN_ID, EVENT) ash ' || + 'LEFT JOIN SYS.GV$OB_PLAN_CACHE_PLAN_STAT pc ON ash.sql_id = pc.sql_id AND ash.plan_id = pc.plan_id ORDER BY EVENT_CNT DESC) v1 WHERE ROWNUM < 100'; + OPEN top_event_cv FOR DYN_SQL + USING L_BTIME, L_ETIME, + L_BTIME, L_ETIME, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR; + LOOP + FETCH top_event_cv INTO top_sql_rec; + EXIT WHEN top_event_cv%NOTFOUND; + APPEND_ROW(FORMAT_ROW(COLUMN_CONTENT_ARRAY( + top_sql_rec.SQL_ID, + TO_CHAR(top_sql_rec.PLAN_ID), + TO_CHAR(top_sql_rec.EVENT_CNT), + top_sql_rec.EVENT, + TO_CHAR(ROUND(100 * top_sql_rec.EVENT_CNT/NUM_EVENTS, 2), DIG_2_FM) || '%', + NVL(top_sql_rec.QUERY_SQL, ' ') + ), column_widths, ' ', '|')); + END LOOP; + CLOSE top_event_cv; + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + + + + + APPEND_ROW(' '); + APPEND_ROW('## Top SQL with Top Blocking Events'); + APPEND_ROW(' - Empty result if no event other than On CPU sampled'); + APPEND_ROW(' - Empty ''SQL Text'' if it is PL/SQL query'); + column_widths := COLUMN_WIDTH_ARRAY(40, 12, 25, 40, 12, 60); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + column_content := COLUMN_CONTENT_ARRAY('SQL ID', 'PLAN ID', 'Sampled # of Executions', 'Event', '% Event', 'SQL Text'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, ' ', '|')); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + DYN_SQL := 'SELECT SQL_ID, PLAN_ID, EVENT_CNT, EVENT, QUERY_SQL ' || + 'FROM (SELECT ash.*, SUBSTR(REPLACE(pc.QUERY_SQL, CHR(10), '' ''), 0, 55) QUERY_SQL ' || + ' FROM (SELECT SQL_ID, PLAN_ID, SUM(' || FILTER_EVENT_STR || ') EVENT_CNT, EVENT FROM (' || + DBMS_ASH_INTERNAL.ASH_VIEW_SQL || + ' ) top_event WHERE wait_class_id != 100 GROUP BY SQL_ID, PLAN_ID, EVENT) ash ' || + 'LEFT JOIN GV$OB_PLAN_CACHE_PLAN_STAT pc ON ash.sql_id = pc.sql_id AND ash.plan_id = pc.plan_id ORDER BY EVENT_CNT DESC) WHERE ROWNUM < 100'; + OPEN top_event_cv FOR DYN_SQL + USING L_BTIME, L_ETIME, + L_BTIME, L_ETIME, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR; + LOOP + FETCH top_event_cv INTO top_sql_rec; + EXIT WHEN top_event_cv%NOTFOUND; + APPEND_ROW(FORMAT_ROW(COLUMN_CONTENT_ARRAY( + top_sql_rec.SQL_ID, + TO_CHAR(top_sql_rec.PLAN_ID), + TO_CHAR(top_sql_rec.EVENT_CNT), + top_sql_rec.EVENT, + TO_CHAR(ROUND(100 * top_sql_rec.EVENT_CNT/NUM_EVENTS, 2), DIG_2_FM) || '%', + NVL(top_sql_rec.QUERY_SQL, ' ') + ), column_widths, ' ', '|')); + END LOOP; + CLOSE top_event_cv; + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + + -- complete List of SQL Text + APPEND_ROW(' '); + APPEND_ROW('## Complete List of SQL Text'); + DYN_SQL := 'SELECT SQL_ID, QUERY_SQL FROM (SELECT pc.SQL_ID SQL_ID, pc.QUERY_SQL QUERY_SQL ' || + 'FROM (SELECT SQL_ID, PLAN_ID, COUNT(1) EVENT_CNT FROM (' || DBMS_ASH_INTERNAL.ASH_VIEW_SQL || ') top_event GROUP BY SQL_ID, PLAN_ID, EVENT) ash ' || + 'LEFT JOIN GV$OB_PLAN_CACHE_PLAN_STAT pc ON ash.sql_id = pc.sql_id AND ash.plan_id = pc.plan_id ORDER BY EVENT_CNT DESC) WHERE QUERY_SQL IS NOT NULL AND ROWNUM < 100'; + OPEN top_event_cv FOR DYN_SQL + USING L_BTIME, L_ETIME, + L_BTIME, L_ETIME, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR; + LOOP + FETCH top_event_cv INTO complete_sql_rec; + EXIT WHEN top_event_cv%NOTFOUND; + APPEND_ROW(' SQL ID: ' || NVL(complete_sql_rec.SQL_ID, ' ')); + APPEND_ROW('SQL Text: ' || NVL(SUBSTR(complete_sql_rec.QUERY_SQL, 0, 4000), ' ')); + APPEND_ROW(''); + END LOOP; + CLOSE top_event_cv; + + + APPEND_ROW(' '); + APPEND_ROW('## Top Sessions:'); + APPEND_ROW(' - ''# Samples Active'' shows the number of ASH samples in which the session was found waiting for that particular event. The percentage shown in this column is calculated with respect to wall time.'); + column_widths := COLUMN_WIDTH_ARRAY(20, 22, 40, 12, 12, 20, '20'); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + column_content := COLUMN_CONTENT_ARRAY('Sid', '% Activity', 'Event', 'Event Count', '% Event', 'User', '# Samples Active'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, ' ', '|')); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + DYN_SQL := 'SELECT SESSION_ID, EVENT, EVENT_CNT, SAMPLE_CNT, USERNAME USER_NAME ' || + ' FROM (SELECT * FROM (SELECT SESSION_ID, USER_ID, EVENT, SUM(' || FILTER_EVENT_STR || ') EVENT_CNT, COUNT(1) SAMPLE_CNT FROM (' || DBMS_ASH_INTERNAL.ASH_VIEW_SQL || ') top_event ' || + ' GROUP BY SESSION_ID, USER_ID, EVENT HAVING COUNT(1) / :num_samples > 0.005 ORDER BY SAMPLE_CNT DESC) WHERE ROWNUM < 100) ash ' || + ' LEFT JOIN SYS.ALL_USERS u ON u.USERID = ash.USER_ID'; + OPEN top_event_cv FOR DYN_SQL + USING L_BTIME, L_ETIME, + L_BTIME, L_ETIME, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, NUM_SAMPLES; + LOOP + FETCH top_event_cv INTO top_sess_rec; + EXIT WHEN top_event_cv%NOTFOUND; + APPEND_ROW(FORMAT_ROW(COLUMN_CONTENT_ARRAY( + TO_CHAR(top_sess_rec.SESSION_ID), + TO_CHAR(ROUND(100 * top_sess_rec.SAMPLE_CNT/NUM_SAMPLES, 2), DIG_2_FM) || '%', + top_sess_rec.EVENT, + TO_CHAR(top_sess_rec.EVENT_CNT), + TO_CHAR(ROUND(100 * top_sess_rec.EVENT_CNT/NUM_EVENTS, 2), DIG_2_FM) || '%', + NVL(top_sess_rec.USER_NAME, ' '), + TO_CHAR(top_sess_rec.EVENT_CNT) || '/' || TO_CHAR(DUR_ELAPSED) || '[' || TO_CHAR(ROUND(100*top_sess_rec.EVENT_CNT/DUR_ELAPSED, 2), DIG_2_FM) || '%]' + ), column_widths, ' ', '|')); + END LOOP; + CLOSE top_event_cv; + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + + + + + APPEND_ROW(' '); + APPEND_ROW('## Top Blocking Sessions:'); + APPEND_ROW(' - Blocking session activity percentages are calculated with respect to waits on latches and locks only.'); + APPEND_ROW(' - ''# Samples Active'' shows the number of ASH samples in which the blocking session was found active.'); + column_widths := COLUMN_WIDTH_ARRAY(20, 22, 40, 12, 12, 20, '20'); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + column_content := COLUMN_CONTENT_ARRAY('Blocking Sid', '% Activity', 'Event Caused', 'Event Count', '% Event', 'User', '# Samples Active'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, ' ', '|')); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + DYN_SQL := 'SELECT SESSION_ID, EVENT, EVENT_CNT, SAMPLE_CNT, USERNAME USER_NAME ' || + ' FROM (SELECT * FROM (SELECT SESSION_ID, USER_ID, EVENT, SUM(' || FILTER_EVENT_STR || ') EVENT_CNT, COUNT(1) SAMPLE_CNT FROM (' || DBMS_ASH_INTERNAL.ASH_VIEW_SQL || ') top_event ' || + ' WHERE wait_class_id != 100 GROUP BY SESSION_ID, USER_ID, EVENT HAVING COUNT(1) / :num_samples > 0.005 ORDER BY SAMPLE_CNT DESC) WHERE ROWNUM < 100) ash ' || + ' LEFT JOIN SYS.ALL_USERS u ON u.USERID = ash.USER_ID'; + OPEN top_event_cv FOR DYN_SQL + USING L_BTIME, L_ETIME, + L_BTIME, L_ETIME, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, NUM_SAMPLES; + LOOP + FETCH top_event_cv INTO top_sess_rec; + EXIT WHEN top_event_cv%NOTFOUND; + DBMS_OUTPUT.PUT_LINE(TO_CHAR(top_sess_rec.EVENT_CNT)); + APPEND_ROW(FORMAT_ROW(COLUMN_CONTENT_ARRAY( + TO_CHAR(top_sess_rec.SESSION_ID), + TO_CHAR(ROUND(100 * top_sess_rec.SAMPLE_CNT/NUM_SAMPLES, 2), DIG_2_FM) || '%', + top_sess_rec.EVENT, + TO_CHAR(top_sess_rec.EVENT_CNT), + TO_CHAR(ROUND(100 * top_sess_rec.EVENT_CNT/NUM_EVENTS, 2), DIG_2_FM) || '%', + top_sess_rec.USER_NAME, + TO_CHAR(top_sess_rec.EVENT_CNT) || '/' || TO_CHAR(DUR_ELAPSED) || '[' || TO_CHAR(ROUND(100*top_sess_rec.EVENT_CNT/DUR_ELAPSED, 2), DIG_2_FM) || '%]' + ), column_widths, ' ', '|')); + END LOOP; + CLOSE top_event_cv; + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-', '-', '-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + + + + APPEND_ROW(' '); + APPEND_ROW('## Top latches:'); + column_widths := COLUMN_WIDTH_ARRAY(40, 20, 20); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + column_content := COLUMN_CONTENT_ARRAY('Latch', 'Sampled Count', '% Activity'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, ' ', '|')); + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + DYN_SQL := 'SELECT * FROM (SELECT EVENT, COUNT(1) SAMPLE_CNT FROM (' || DBMS_ASH_INTERNAL.ASH_VIEW_SQL || ') top_event ' || + ' WHERE wait_class_id = 104 AND SUBSTR(event, 0, 6) = ''latch:'' GROUP BY EVENT HAVING COUNT(1) / :num_samples > 0.005 ORDER BY SAMPLE_CNT DESC) WHERE ROWNUM < 100'; + OPEN top_event_cv FOR DYN_SQL + USING L_BTIME, L_ETIME, + L_BTIME, L_ETIME, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, + NULL_CHAR, NULL_CHAR, NUM_SAMPLES; + LOOP + FETCH top_event_cv INTO top_latch_rec; + EXIT WHEN top_event_cv%NOTFOUND; + APPEND_ROW(FORMAT_ROW(COLUMN_CONTENT_ARRAY( + TO_CHAR(top_latch_rec.EVENT), + TO_CHAR(top_latch_rec.SAMPLE_CNT), + TO_CHAR(ROUND(100 * top_latch_rec.SAMPLE_CNT/NUM_SAMPLES, 2), DIG_2_FM) || '%' + ), column_widths, ' ', '|')); + END LOOP; + CLOSE top_event_cv; + column_content := COLUMN_CONTENT_ARRAY('-', '-', '-'); + APPEND_ROW(FORMAT_ROW(column_content, column_widths, '-', '+')); + + + return RPT_ROWS; + +END ASH_REPORT_TEXT; + + + +END dbms_workload_repository; diff --git a/src/share/io/ob_io_define.cpp b/src/share/io/ob_io_define.cpp index 62285c1db..e15415b8c 100644 --- a/src/share/io/ob_io_define.cpp +++ b/src/share/io/ob_io_define.cpp @@ -709,7 +709,6 @@ void ObIORequest::dec_out_ref() ObPhyQueue::ObPhyQueue() : is_inited_(false), - phy_queue_(), reservation_ts_(INT_MAX64), category_limitation_ts_(INT_MAX64), tenant_limitation_ts_(INT_MAX64), @@ -720,7 +719,8 @@ ObPhyQueue::ObPhyQueue() reservation_pos_(-1), category_limitation_pos_(-1), tenant_limitation_pos_(-1), - proportion_pos_(-1) + proportion_pos_(-1), + req_list_() { } @@ -736,8 +736,6 @@ int ObPhyQueue::init(const int index) if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("phy queue init twice", K(ret), K(is_inited_)); - } else if (OB_FAIL(phy_queue_.init(SINGLE_QUEUE_DEPTH))) { - LOG_WARN("phy queue init failed", K(ret)); } else if (index < 0 || index > static_cast(ObIOCategory::MAX_CATEGORY)){ ret = OB_INVALID_ARGUMENT; LOG_WARN("index out of boundary", K(ret), K(index)); @@ -754,7 +752,6 @@ int ObPhyQueue::init(const int index) void ObPhyQueue::destroy() { is_inited_ = false; - phy_queue_.destroy(); reservation_ts_ = INT_MAX64; category_limitation_ts_ = INT_MAX64; tenant_limitation_ts_ = INT_MAX64; @@ -1293,32 +1290,29 @@ int ObMClockQueue::pop_phyqueue(ObIORequest *&req, int64_t &deadline_ts) if (OB_ISNULL(tmp_phy_queue)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("phy_queue is null", K(ret), KP(tmp_phy_queue)); - } else if (tmp_phy_queue->phy_queue_.get_total() <= 0) { + } else if (tmp_phy_queue->req_list_.is_empty()) { ret = OB_ENTRY_NOT_EXIST; } else if (tmp_phy_queue->reservation_ts_ <= current_ts) { //R schedule if(OB_FAIL(remove_from_heap(tmp_phy_queue))) { LOG_WARN("remove phy queue from heap failed(R schedule)", K(ret)); } else { - if (OB_FAIL(tmp_phy_queue->phy_queue_.pop(req))) { - LOG_WARN("pop req from r_heap failed", K(ret)); - } else if (OB_ISNULL(req)) { + req = tmp_phy_queue->req_list_.remove_first(); + if (OB_ISNULL(req)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("request is null", K(ret), KP(req)); } else { req->time_log_.dequeue_ts_ = ObTimeUtility::fast_current_time(); - if (tmp_phy_queue->phy_queue_.get_total() <= 0) { + if (tmp_phy_queue->req_list_.is_empty()) { tmp_phy_queue->reset_time_info(); } else if (OB_NOT_NULL(req->tenant_io_mgr_.get_ptr())) { ObTenantIOClock *io_clock = static_cast(req->tenant_io_mgr_.get_ptr()->get_io_clock()); - ObIORequest *next_req = nullptr; - if (OB_FAIL(tmp_phy_queue->phy_queue_.head_unsafe(next_req))) { - LOG_WARN("get next req failed", K(ret), KP(next_req)); - } else if (OB_ISNULL(next_req)) { + ObIORequest *next_req = tmp_phy_queue->req_list_.get_first(); + if (OB_ISNULL(next_req)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get null next_req", KP(next_req)); } else if (OB_FAIL(io_clock->calc_phyqueue_clock(tmp_phy_queue, *next_req))) { - LOG_WARN("calc phyqueue clock failed", K(ret)); + LOG_WARN("calc phyqueue clock failed", K(ret), KPC(next_req)); } } } @@ -1372,7 +1366,7 @@ int ObMClockQueue::pop_with_ready_queue(const int64_t current_ts, ObIORequest *& int64_t iter_count = 0; ObPhyQueue *tmp_phy_queue = nullptr; req = nullptr; - while (OB_SUCC(ret) && !cl_heap_.empty() && cl_heap_.top()->phy_queue_.get_total() > 0) { + while (OB_SUCC(ret) && !cl_heap_.empty() && !cl_heap_.top()->req_list_.is_empty()) { tmp_phy_queue = cl_heap_.top(); deadline_ts = 0 == iter_count ? tmp_phy_queue->tenant_limitation_ts_ : deadline_ts; ++iter_count; @@ -1395,7 +1389,7 @@ int ObMClockQueue::pop_with_ready_queue(const int64_t current_ts, ObIORequest *& } } iter_count = 0; - while (OB_SUCC(ret) && !tl_heap_.empty() && tl_heap_.top()->phy_queue_.get_total() > 0) { + while (OB_SUCC(ret) && !tl_heap_.empty() && !tl_heap_.top()->req_list_.is_empty()) { tmp_phy_queue = tl_heap_.top(); if (0 == iter_count) { if (0 == deadline_ts) { @@ -1421,26 +1415,23 @@ int ObMClockQueue::pop_with_ready_queue(const int64_t current_ts, ObIORequest *& if (OB_ISNULL(tmp_phy_queue)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("phy_queue is null", K(ret)); - } else if (tmp_phy_queue->phy_queue_.get_total() > 0) { + } else if (!tmp_phy_queue->req_list_.is_empty()) { if (OB_FAIL(remove_from_heap(tmp_phy_queue))) { LOG_WARN("remove phy queue from heap failed(P schedule)", K(ret)); } else { - if (OB_FAIL(tmp_phy_queue->phy_queue_.pop(req))) { - LOG_WARN("pop req from ready_heap failed"); - } else if (OB_ISNULL(req)) { + req = tmp_phy_queue->req_list_.remove_first(); + if (OB_ISNULL(req)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("req is null", K(ret), KP(req)); } else { req->time_log_.dequeue_ts_ = ObTimeUtility::fast_current_time(); LOG_DEBUG("req pop from phy queue succcess(P schedule)", KP(req), K(iter_count), "time_cost", ObTimeUtility::fast_current_time() - current_ts, K(ready_heap_.count()), K(current_ts)); - if (tmp_phy_queue->phy_queue_.get_total() <= 0) { + if (tmp_phy_queue->req_list_.is_empty()) { tmp_phy_queue->reset_time_info(); } else if (OB_NOT_NULL(req->tenant_io_mgr_.get_ptr())) { ObTenantIOClock *io_clock = static_cast(req->tenant_io_mgr_.get_ptr()->get_io_clock()); - ObIORequest *next_req = nullptr; - if (OB_FAIL(tmp_phy_queue->phy_queue_.head_unsafe(next_req))) { - LOG_WARN("get next req failed", K(ret), K(next_req)); - } else if (OB_ISNULL(next_req)) { + ObIORequest *next_req = tmp_phy_queue->req_list_.get_first(); + if (OB_ISNULL(next_req)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get null next_req", KP(next_req)); } else { @@ -1462,7 +1453,7 @@ int ObMClockQueue::pop_with_ready_queue(const int64_t current_ts, ObIORequest *& } } else { ret = OB_EAGAIN; - if (!r_heap_.empty() && r_heap_.top()->phy_queue_.get_total() > 0) { + if (!r_heap_.empty() && !r_heap_.top()->req_list_.is_empty()) { ObPhyQueue *next_tmp_phy_queue = r_heap_.top(); if (0 == deadline_ts) { deadline_ts = next_tmp_phy_queue->reservation_ts_; diff --git a/src/share/io/ob_io_define.h b/src/share/io/ob_io_define.h index b53dbd216..a86f08d53 100644 --- a/src/share/io/ob_io_define.h +++ b/src/share/io/ob_io_define.h @@ -18,7 +18,7 @@ #include "lib/lock/ob_thread_cond.h" #include "lib/container/ob_rbtree.h" #include "common/storage/ob_io_device.h" -#include "lib/queue/ob_fixed_queue.h" +#include "lib/list/ob_list.h" #include "lib/container/ob_heap.h" namespace oceanbase @@ -207,7 +207,7 @@ class ObIOChannel; class ObIOSender; class ObIOUsage; -class ObIORequest +class ObIORequest : public common::ObDLinkBase { public: ObIORequest(); @@ -272,10 +272,9 @@ public: void reset_time_info(); void reset_queue_info(); public: + typedef common::ObDList IOReqList; TO_STRING_KV(K_(reservation_ts), K_(category_limitation_ts), K_(tenant_limitation_ts)); bool is_inited_; - ObFixedQueue phy_queue_; - static const int32_t SINGLE_QUEUE_DEPTH = 10000; int64_t reservation_ts_; int64_t category_limitation_ts_; int64_t tenant_limitation_ts_; @@ -287,6 +286,7 @@ public: int64_t category_limitation_pos_; int64_t tenant_limitation_pos_; int64_t proportion_pos_; + IOReqList req_list_; }; class ObIOHandle final diff --git a/src/share/io/ob_io_struct.cpp b/src/share/io/ob_io_struct.cpp index 7c9141b51..05aa809b4 100644 --- a/src/share/io/ob_io_struct.cpp +++ b/src/share/io/ob_io_struct.cpp @@ -634,12 +634,40 @@ void ObIOTuner::print_io_status() } } -/****************** ObPhyQueue **********************/ -ObTenantPhyQueues::ObTenantPhyQueues() +/****************** ObIOCategoryQueues **********************/ +ObIOCategoryQueues::ObIOCategoryQueues() + : is_inited_(false) { } +ObIOCategoryQueues::~ObIOCategoryQueues() +{ + destroy(); +} + +int ObIOCategoryQueues::init() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("phy queue init twice", K(ret), K(is_inited_)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < static_cast(ObIOCategory::MAX_CATEGORY) + 1; ++i) { + if (OB_FAIL(phy_queues_[i].init(i))){ + LOG_WARN("phy queue init failed", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + is_inited_ = true; + } + return ret; +} +void ObIOCategoryQueues::destroy() +{ + is_inited_ = false; +} /****************** IOScheduleQueue **********************/ ObIOSender::ObIOSender(ObIAllocator &allocator) : is_inited_(false), @@ -693,11 +721,9 @@ struct DestroyPhyqueueMapFn { public: DestroyPhyqueueMapFn(ObIAllocator &allocator) : allocator_(allocator) {} - int operator () (hash::HashMapPair &entry) { + int operator () (hash::HashMapPair &entry) { if (nullptr != entry.second) { - for (int64_t i = 0; i < static_cast(ObIOCategory::MAX_CATEGORY) + 1; ++i) { - entry.second->phy_queues_[i].destroy(); - } + entry.second->~ObIOCategoryQueues(); allocator_.free(entry.second); } return OB_SUCCESS; @@ -797,7 +823,7 @@ int ObIOSender::alloc_mclock_queue(ObIAllocator &allocator, ObMClockQueue *&io_q int ObIOSender::enqueue_request(ObIORequest &req) { int ret = OB_SUCCESS; - ObTenantPhyQueues *tenant_phy_queues = nullptr; + ObIOCategoryQueues *io_category_queues = nullptr; ObIORequest *tmp_req = &req; if (!is_inited_) { ret = OB_NOT_INIT; @@ -807,18 +833,19 @@ int ObIOSender::enqueue_request(ObIORequest &req) if (OB_FAIL(cond_guard.get_ret())) { LOG_ERROR("guard queue condition failed", K(ret)); } else { - if (OB_FAIL(tenant_map_.get_refactored(tmp_req->io_info_.tenant_id_, tenant_phy_queues))) { + if (OB_FAIL(tenant_map_.get_refactored(tmp_req->io_info_.tenant_id_, io_category_queues))) { LOG_WARN("get_refactored tenant_map failed", K(ret), K(req)); } else { const int index = static_cast(tmp_req->get_category()); - ObPhyQueue *tmp_phy_queue = &(tenant_phy_queues->phy_queues_[index]); - if (tmp_phy_queue->phy_queue_.get_total() <= 0) { + ObPhyQueue *tmp_phy_queue = &(io_category_queues->phy_queues_[index]); + if (tmp_phy_queue->req_list_.is_empty()) { //new request if (OB_FAIL(io_queue_->remove_from_heap(tmp_phy_queue))) { LOG_WARN("remove phy queue from heap failed", K(ret), K(index)); } else { req.inc_ref("phyqueue_inc"); //ref for phy_queue - if (OB_FAIL(tmp_phy_queue->phy_queue_.push(tmp_req))) { + if (OB_UNLIKELY(!tmp_phy_queue->req_list_.add_last(tmp_req))) { + ret = OB_ERR_UNEXPECTED; req.dec_ref("phyqueue_dec"); //ref for phy_queue tmp_phy_queue->reset_time_info(); LOG_WARN("push new req into phy queue failed", K(ret)); @@ -846,7 +873,8 @@ int ObIOSender::enqueue_request(ObIORequest &req) } else { //not new req, into phy_queue and line up req.inc_ref("phyqueue_inc"); //ref for phy_queue - if (OB_FAIL(tmp_phy_queue->phy_queue_.push(tmp_req))) { + if (OB_UNLIKELY(!tmp_phy_queue->req_list_.add_last(tmp_req))) { + ret = OB_ERR_UNEXPECTED; LOG_WARN("req line up failed", K(req)); req.dec_ref("phyqueue_dec"); //ref for phy_queue } else { @@ -865,7 +893,7 @@ int ObIOSender::enqueue_request(ObIORequest &req) return ret; } -int ObIOSender::enqueue_phy_queue(ObPhyQueue *phyqueue) +int ObIOSender::enqueue_phy_queue(ObPhyQueue &phyqueue) { int ret = OB_SUCCESS; if (!is_inited_) { @@ -876,7 +904,7 @@ int ObIOSender::enqueue_phy_queue(ObPhyQueue *phyqueue) if (OB_FAIL(cond_guard.get_ret())) { LOG_ERROR("guard queue condition failed", K(ret)); } else { - if (OB_FAIL(io_queue_->push_phyqueue(phyqueue))) { + if (OB_FAIL(io_queue_->push_phyqueue(&phyqueue))) { LOG_WARN("push phyqueue into queue failed", K(ret)); } else { if (OB_FAIL(queue_cond_.signal())) { @@ -930,20 +958,19 @@ int ObIOSender::remove_phy_queue(const uint64_t tenant_id) if (OB_FAIL(cond_guard.get_ret())) { LOG_ERROR("guard queue condition failed", K(ret)); } else { - ObTenantPhyQueues *tenant_phy_queues = nullptr; - if (OB_FAIL(tenant_map_.erase_refactored(tenant_id, &tenant_phy_queues))) { + ObIOCategoryQueues *io_category_queues = nullptr; + if (OB_FAIL(tenant_map_.erase_refactored(tenant_id, &io_category_queues))) { LOG_WARN("erase phy_queues failed", K(ret), K(tenant_id)); - } else if (nullptr != tenant_phy_queues) { + } else if (nullptr != io_category_queues) { for (int64_t j = 0; OB_SUCC(ret) && j < static_cast(ObIOCategory::MAX_CATEGORY) + 1; ++j) { - ObPhyQueue *tmp_phy_queue = &(tenant_phy_queues->phy_queues_[j]); + ObPhyQueue *tmp_phy_queue = &(io_category_queues->phy_queues_[j]); if (OB_FAIL(io_queue_->remove_from_heap(tmp_phy_queue))) { LOG_WARN("remove phy queue from heap failed", K(ret)); - } else { - tmp_phy_queue->~ObPhyQueue(); } } if (OB_SUCC(ret)) { - allocator_.free(tenant_phy_queues); + io_category_queues->~ObIOCategoryQueues(); + allocator_.free(io_category_queues); } } } @@ -1202,29 +1229,29 @@ int ObIOScheduler::add_tenant_map(uint64_t tenant_id) int ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret) && i < senders_.count(); ++i) { ObIOSender *cur_sender = senders_.at(i); - ObTenantPhyQueues *tenant_phy_queues = nullptr; + ObIOCategoryQueues *io_category_queues = nullptr; void *buf_queues = nullptr; - if (OB_ISNULL(buf_queues = cur_sender->allocator_.alloc(sizeof(ObTenantPhyQueues)))) { + if (OB_ISNULL(buf_queues = cur_sender->allocator_.alloc(sizeof(ObIOCategoryQueues)))) { ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("allocate tenant phyqueues memory failed", K(ret)); + LOG_WARN("allocate phyqueues memory failed", K(ret)); } else { - tenant_phy_queues = new (buf_queues) ObTenantPhyQueues; - for(int64_t j = 0; OB_SUCC(ret) && j < static_cast(ObIOCategory::MAX_CATEGORY) + 1; j++) { - if (OB_FAIL(tenant_phy_queues->phy_queues_[j].init(j))){ - LOG_WARN("phy queue init failed", K(ret)); - } else if (OB_FAIL(cur_sender->enqueue_phy_queue(&(tenant_phy_queues->phy_queues_[j])))) { - LOG_WARN("new phy_queue into send_queue failed", K(ret)); + io_category_queues = new (buf_queues) ObIOCategoryQueues(); + if (OB_FAIL(io_category_queues->init())) { + LOG_WARN("init phyqueues failed", K(ret)); + } else { + for(int64_t j = 0; OB_SUCC(ret) && j < static_cast(ObIOCategory::MAX_CATEGORY) + 1; j++) { + if (OB_FAIL(cur_sender->enqueue_phy_queue(io_category_queues->phy_queues_[j]))) { + LOG_WARN("new phy_queue into send_queue failed", K(ret)); + } } } if (OB_SUCC(ret)) { - if (OB_FAIL(cur_sender->tenant_map_.set_refactored(tenant_id, tenant_phy_queues))) { + if (OB_FAIL(cur_sender->tenant_map_.set_refactored(tenant_id, io_category_queues))) { LOG_WARN("init tenant map failed", K(ret), K(i)); } } else { - for(int64_t j = 0; OB_SUCC(ret) && j < static_cast(ObIOCategory::MAX_CATEGORY) + 1; j++) { - tenant_phy_queues->phy_queues_[j].~ObPhyQueue(); - } - cur_sender->allocator_.free(tenant_phy_queues); + io_category_queues->~ObIOCategoryQueues(); + cur_sender->allocator_.free(io_category_queues); } } } diff --git a/src/share/io/ob_io_struct.h b/src/share/io/ob_io_struct.h index e11b357d4..a573e4f84 100644 --- a/src/share/io/ob_io_struct.h +++ b/src/share/io/ob_io_struct.h @@ -187,11 +187,14 @@ private: ObIOScheduler &io_scheduler_; }; -struct ObTenantPhyQueues final { +struct ObIOCategoryQueues final { public: - ObTenantPhyQueues(); - ~ObTenantPhyQueues(); + ObIOCategoryQueues(); + ~ObIOCategoryQueues(); + int init(); + void destroy(); public: + bool is_inited_; ObPhyQueue phy_queues_[static_cast(ObIOCategory::MAX_CATEGORY) + 1]; }; @@ -212,7 +215,7 @@ public: int alloc_mclock_queue(ObIAllocator &allocator, ObMClockQueue *&io_queue); int enqueue_request(ObIORequest &req); - int enqueue_phy_queue(ObPhyQueue *phyqueue); + int enqueue_phy_queue(ObPhyQueue &phyqueue); int dequeue_request(ObIORequest *&req); int remove_phy_queue(const uint64_t tenant_id); int notify(); @@ -229,7 +232,7 @@ public: int tg_id_; // thread group id ObMClockQueue *io_queue_; ObThreadCond queue_cond_; - hash::ObHashMap tenant_map_; + hash::ObHashMap tenant_map_; int64_t sender_req_count_; }; diff --git a/src/share/ls/ob_ls_creator.cpp b/src/share/ls/ob_ls_creator.cpp index 18d7c06af..0b3f987f7 100644 --- a/src/share/ls/ob_ls_creator.cpp +++ b/src/share/ls/ob_ls_creator.cpp @@ -25,7 +25,7 @@ #include "share/ob_unit_table_operator.h" //ObUnitTableOperator #include "logservice/leader_coordinator/table_accessor.h" #include "logservice/palf/palf_base_info.h"//palf::PalfBaseInfo -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ls/ob_ls_life_manager.h" using namespace oceanbase::common; diff --git a/src/share/ls/ob_ls_creator.h b/src/share/ls/ob_ls_creator.h index d41324971..1d0de1467 100644 --- a/src/share/ls/ob_ls_creator.h +++ b/src/share/ls/ob_ls_creator.h @@ -40,11 +40,11 @@ class ObSchemaGetterGuard; namespace palf { struct PalfBaseInfo; -class SCN; } namespace share { +class SCN; class ObServerManager; struct ObLSReplicaAddr { @@ -105,7 +105,7 @@ public: int create_user_ls(const share::ObLSStatusInfo &status_info, const int64_t paxos_replica_num, const share::schema::ZoneLocalityIArray &zone_locality, - const palf::SCN &create_scn, + const SCN &create_scn, const common::ObCompatibilityMode &compat_mode, const bool create_with_palf, const palf::PalfBaseInfo &palf_base_info); @@ -115,7 +115,7 @@ public: private: int do_create_ls_(const ObLSAddr &addr, const share::ObLSStatusInfo &info, const int64_t paxos_replica_num, - const palf::SCN &create_scn, + const SCN &create_scn, const common::ObCompatibilityMode &compat_mode, common::ObMemberList &member_list, const bool create_with_palf, @@ -124,7 +124,7 @@ private: const int64_t paxos_replica_num); int create_ls_(const ObILSAddr &addr, const int64_t paxos_replica_num, const share::ObAllTenantInfo &tenant_info, - const palf::SCN &create_scn, + const SCN &create_scn, const common::ObCompatibilityMode &compat_mode, const bool create_with_palf, const palf::PalfBaseInfo &palf_base_info, diff --git a/src/share/ls/ob_ls_election_reference_info_operator.cpp b/src/share/ls/ob_ls_election_reference_info_operator.cpp index 6dd597d64..896ac8212 100644 --- a/src/share/ls/ob_ls_election_reference_info_operator.cpp +++ b/src/share/ls/ob_ls_election_reference_info_operator.cpp @@ -21,7 +21,7 @@ #include "common/ob_timeout_ctx.h" #include "share/ob_share_util.h" #include "share/ls/ob_ls_status_operator.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -31,7 +31,7 @@ namespace share using namespace common; int ObLsElectionReferenceInfoOperator::create_new_ls(const ObLSStatusInfo &ls_info, - const palf::SCN &create_ls_scn, + const SCN &create_ls_scn, const common::ObString &zone_priority, ObMySQLTransaction &trans) { @@ -80,7 +80,7 @@ int ObLsElectionReferenceInfoOperator::drop_ls(const uint64_t &tenant_id, int ObLsElectionReferenceInfoOperator::set_ls_offline(const uint64_t &, const share::ObLSID &, const ObLSStatus &, - const palf::SCN &, + const SCN &, ObMySQLTransaction &) { return OB_SUCCESS; } diff --git a/src/share/ls/ob_ls_election_reference_info_operator.h b/src/share/ls/ob_ls_election_reference_info_operator.h index 44beb7604..e52a52512 100644 --- a/src/share/ls/ob_ls_election_reference_info_operator.h +++ b/src/share/ls/ob_ls_election_reference_info_operator.h @@ -36,12 +36,9 @@ namespace sqlclient class ObMySQLResult; } } -namespace palf -{ -class SCN; -} namespace share { +class SCN; /* * description: operation of __all_ls_election_reference_info * insert into __all_ls_election_reference_info while create new ls @@ -60,7 +57,7 @@ public: * @param[in] create_ls_scn: ls's create scn * @param[in] trans:*/ virtual int create_new_ls(const ObLSStatusInfo &ls_info, - const palf::SCN &create_ls_scn, + const SCN &create_ls_scn, const common::ObString &zone_priority, ObMySQLTransaction &trans) override final; /* @@ -82,7 +79,7 @@ public: virtual int set_ls_offline(const uint64_t &tenant_id, const share::ObLSID &ls_id, const ObLSStatus &ls_status, - const palf::SCN &drop_scn, + const SCN &drop_scn, ObMySQLTransaction &trans) override final; /* * description: update ls primary zone, need update __all_ls_status and __all_ls_election_reference diff --git a/src/share/ls/ob_ls_i_life_manager.h b/src/share/ls/ob_ls_i_life_manager.h index 9f3b60672..2e87fca23 100644 --- a/src/share/ls/ob_ls_i_life_manager.h +++ b/src/share/ls/ob_ls_i_life_manager.h @@ -18,7 +18,7 @@ #include "share/config/ob_server_config.h" #include "lib/mysqlclient/ob_mysql_proxy.h" #include "lib/string/ob_sql_string.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { namespace common @@ -65,7 +65,7 @@ public: virtual ~ObLSLifeIAgent () {} //create new ls virtual int create_new_ls(const ObLSStatusInfo &ls_info, - const palf::SCN &create_scn, + const SCN &create_scn, const common::ObString &zone_priority, ObMySQLTransaction &trans) = 0; //drop ls @@ -76,7 +76,7 @@ public: virtual int set_ls_offline(const uint64_t &tenant_id, const share::ObLSID &ls_id, const share::ObLSStatus &ls_status, - const palf::SCN &drop_scn, + const SCN &drop_scn, ObMySQLTransaction &trans) = 0; //update ls primary zone virtual int update_ls_primary_zone( diff --git a/src/share/ls/ob_ls_life_manager.cpp b/src/share/ls/ob_ls_life_manager.cpp index 1ce93aee3..cec339ea8 100644 --- a/src/share/ls/ob_ls_life_manager.cpp +++ b/src/share/ls/ob_ls_life_manager.cpp @@ -14,7 +14,7 @@ #include "ob_ls_life_manager.h" #include "lib/mysqlclient/ob_mysql_transaction.h"//ObMySQLTransaction -#include "logservice/palf/scn.h"//SCN +#include "share/scn.h"//SCN using namespace oceanbase; using namespace oceanbase::common; diff --git a/src/share/ls/ob_ls_life_manager.h b/src/share/ls/ob_ls_life_manager.h index 47721a6e1..338f57eee 100644 --- a/src/share/ls/ob_ls_life_manager.h +++ b/src/share/ls/ob_ls_life_manager.h @@ -20,10 +20,6 @@ namespace oceanbase { -namespace palf -{ -class SCN; -} namespace common { class ObMySQLTransaction; @@ -35,6 +31,7 @@ class ObMySQLResult; } namespace share { +class SCN; struct ObLSStatusInfo; class ObLSID; /* @@ -74,7 +71,7 @@ public: * @param[in]zone_priority: the primary_zone of OB_ALL_LS_ELECTION_REFERENCE_INFO * */ int create_new_ls(const ObLSStatusInfo &ls_info, - const palf::SCN &create_ls_scn, + const SCN &create_ls_scn, const common::ObString &zone_priority); /* * description: for primary cluster and GC of standby, delete ls from each inner_table @@ -92,7 +89,7 @@ public: int set_ls_offline(const uint64_t &tenant_id, const share::ObLSID &ls_id, const ObLSStatus &ls_status, - const palf::SCN &drop_scn); + const SCN &drop_scn); /* * description: update ls primary zone, need update __all_ls_status and __all_ls_election_reference * @param[in] tenant_id: tenant_id @@ -111,7 +108,7 @@ public: * description: for standby cluster, create new ls */ int create_new_ls_in_trans(const ObLSStatusInfo &ls_info, - const palf::SCN &create_ls_scn, + const SCN &create_ls_scn, const common::ObString &zone_priority, ObMySQLTransaction &trans); /* @@ -126,7 +123,7 @@ public: int set_ls_offline_in_trans(const uint64_t &tenant_id, const share::ObLSID &ls_id, const ObLSStatus &ls_status, - const palf::SCN &drop_scn, + const SCN &drop_scn, ObMySQLTransaction &trans); private: diff --git a/src/share/ls/ob_ls_log_stat_info.h b/src/share/ls/ob_ls_log_stat_info.h index a589096d5..e579d14a9 100644 --- a/src/share/ls/ob_ls_log_stat_info.h +++ b/src/share/ls/ob_ls_log_stat_info.h @@ -17,7 +17,7 @@ #include "lib/net/ob_addr.h" // ObAddr #include "common/ob_role.h" // ObRole #include "share/ls/ob_ls_info.h" // MemberList -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // SCN namespace oceanbase { diff --git a/src/share/ls/ob_ls_operator.cpp b/src/share/ls/ob_ls_operator.cpp index 8ed33c602..6f983fa5b 100644 --- a/src/share/ls/ob_ls_operator.cpp +++ b/src/share/ls/ob_ls_operator.cpp @@ -31,7 +31,7 @@ #include "share/ob_max_id_fetcher.h"//ObMaxIdFetcher #include "share/ob_global_stat_proxy.h"//get gc #include "logservice/palf/log_define.h"//SCN -#include "logservice/palf/scn.h"//SCN +#include "share/scn.h"//SCN using namespace oceanbase; using namespace oceanbase::common; diff --git a/src/share/ls/ob_ls_operator.h b/src/share/ls/ob_ls_operator.h index 5780ed163..461e92bcc 100644 --- a/src/share/ls/ob_ls_operator.h +++ b/src/share/ls/ob_ls_operator.h @@ -19,7 +19,7 @@ #include "share/ls/ob_ls_status_operator.h" //ObLSStatus #include "share/ls/ob_ls_i_life_manager.h"//ObLSTemplateOperator #include "logservice/palf/log_define.h"//SCN -#include "logservice/palf/scn.h"//SCN +#include "share/scn.h"//SCN namespace oceanbase @@ -36,12 +36,9 @@ namespace sqlclient class ObMySQLResult; } } -namespace palf -{ -class SCN; -} namespace share { +class SCN; //TODO for duplicate ls enum ObLSFlag { @@ -89,7 +86,7 @@ struct ObLSAttr const ObLSFlag &flag, const ObLSStatus &status, const ObLSOperationType &type, - const palf::SCN &create_scn); + const SCN &create_scn); void reset(); int assign(const ObLSAttr &other); bool ls_is_creating() const @@ -135,7 +132,7 @@ struct ObLSAttr { return flag_; } - palf::SCN get_create_scn() const + SCN get_create_scn() const { return create_scn_; } @@ -148,7 +145,7 @@ private: ObLSFlag flag_; ObLSStatus status_; ObLSOperationType operation_type_; - palf::SCN create_scn_; + SCN create_scn_; }; typedef common::ObArray ObLSAttrArray; @@ -187,8 +184,8 @@ public: * @param[out] read_scn:the snapshot of read_version * @param[out] ObLSAttrIArray ls_info in __all_ls * */ - int load_all_ls_and_snapshot(palf::SCN &read_scn, ObLSAttrIArray &ls_array); - static int get_tenant_gts(const uint64_t &tenant_id, palf::SCN >s_scn); + int load_all_ls_and_snapshot(share::SCN &read_scn, ObLSAttrIArray &ls_array); + static int get_tenant_gts(const uint64_t &tenant_id, SCN >s_scn); private: int process_sub_trans_(const ObLSAttr &ls_attr, ObMySQLTransaction &trans); diff --git a/src/share/ls/ob_ls_recovery_stat_operator.cpp b/src/share/ls/ob_ls_recovery_stat_operator.cpp index 0b1b5814e..3f15a79b0 100644 --- a/src/share/ls/ob_ls_recovery_stat_operator.cpp +++ b/src/share/ls/ob_ls_recovery_stat_operator.cpp @@ -22,7 +22,7 @@ #include "common/ob_timeout_ctx.h" #include "share/ob_share_util.h" #include "share/ls/ob_ls_status_operator.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" using namespace oceanbase; using namespace oceanbase::common; @@ -235,7 +235,7 @@ int ObLSRecoveryStatOperator::update_ls_recovery_stat_in_trans( int ObLSRecoveryStatOperator::set_ls_offline(const uint64_t &tenant_id, const share::ObLSID &ls_id, const ObLSStatus &ls_status, - const palf::SCN &drop_scn, + const SCN &drop_scn, ObMySQLTransaction &trans) { int ret = OB_SUCCESS; diff --git a/src/share/ls/ob_ls_recovery_stat_operator.h b/src/share/ls/ob_ls_recovery_stat_operator.h index a0d6ea84b..153a37f7c 100644 --- a/src/share/ls/ob_ls_recovery_stat_operator.h +++ b/src/share/ls/ob_ls_recovery_stat_operator.h @@ -19,7 +19,7 @@ #include "lib/container/ob_array.h"//ObArray #include "lib/container/ob_iarray.h"//ObIArray #include "logservice/palf/log_define.h"//SCN -#include "logservice/palf/scn.h"//SCN +#include "share/scn.h"//SCN namespace oceanbase { @@ -37,12 +37,9 @@ namespace sqlclient class ObMySQLResult; } } -namespace palf -{ -class SCN; -} namespace share { +class SCN; struct ObLSRecoveryStat { ObLSRecoveryStat() @@ -56,13 +53,13 @@ struct ObLSRecoveryStat bool is_valid() const; int init(const uint64_t tenant_id, const ObLSID &id, - const palf::SCN &sync_scn, - const palf::SCN &readable_scn, - const palf::SCN &create_scn, - const palf::SCN &drop_scn); + const SCN &sync_scn, + const SCN &readable_scn, + const SCN &create_scn, + const SCN &drop_scn); int init_only_recovery_stat(const uint64_t tenant_id, const ObLSID &id, - const palf::SCN &sync_scn, - const palf::SCN &readable_scn); + const SCN &sync_scn, + const SCN &readable_scn); void reset(); int assign(const ObLSRecoveryStat &other); uint64_t get_tenant_id() const @@ -73,19 +70,19 @@ struct ObLSRecoveryStat { return ls_id_; } - const palf::SCN get_sync_scn() const + const SCN get_sync_scn() const { return sync_scn_; } - const palf::SCN get_readable_scn() const + const SCN get_readable_scn() const { return readable_scn_; } - const palf::SCN get_create_scn() const + const SCN get_create_scn() const { return create_scn_; } - const palf::SCN get_drop_scn() const + const SCN get_drop_scn() const { return drop_scn_; } @@ -95,10 +92,10 @@ struct ObLSRecoveryStat private: uint64_t tenant_id_; ObLSID ls_id_; - palf::SCN sync_scn_;//clog sync ts - palf::SCN readable_scn_;//min weak read timestamp TODO need different majorty replicas and all replicas - palf::SCN create_scn_;//ts less than first clog ts - palf::SCN drop_scn_; //ts larger than last user data's clog and before offline + SCN sync_scn_;//clog sync ts + SCN readable_scn_;//min weak read timestamp TODO need different majorty replicas and all replicas + SCN create_scn_;//ts less than first clog ts + SCN drop_scn_; //ts larger than last user data's clog and before offline }; /* @@ -119,7 +116,7 @@ public: * @param[in] create_ls_scn: ls's create ts * @param[in] trans:*/ virtual int create_new_ls(const ObLSStatusInfo &ls_info, - const palf::SCN &create_ls_scn, + const SCN &create_ls_scn, const common::ObString &zone_priority, ObMySQLTransaction &trans) override; /* @@ -141,7 +138,7 @@ public: virtual int set_ls_offline(const uint64_t &tenant_id, const share::ObLSID &ls_id, const ObLSStatus &ls_status, - const palf::SCN &drop_scn, + const SCN &drop_scn, ObMySQLTransaction &trans) override; /* * description: update ls primary zone, need update __all_ls_status and __all_ls_election_reference @@ -200,8 +197,8 @@ public: * */ int get_tenant_recovery_stat(const uint64_t tenant_id, ObISQLClient &client, - palf::SCN &sync_scn, - palf::SCN &min_wrs); + SCN &sync_scn, + SCN &min_wrs); /* * description: get user ls sync scn, for recovery ls * @param[in] tenant_id @@ -210,15 +207,15 @@ public: * */ int get_user_ls_sync_scn(const uint64_t tenant_id, ObISQLClient &client, - palf::SCN &sync_scn); + SCN &sync_scn); private: bool need_update_ls_recovery_(const ObLSRecoveryStat &old_recovery, const ObLSRecoveryStat &new_recovery); int get_all_ls_recovery_stat_(const uint64_t tenant_id, const common::ObSqlString &sql, ObISQLClient &client, - palf::SCN &sync_scn, - palf::SCN &min_wrs); + SCN &sync_scn, + SCN &min_wrs); }; } diff --git a/src/share/ls/ob_ls_status_operator.cpp b/src/share/ls/ob_ls_status_operator.cpp index afd08d2ee..09eb31675 100644 --- a/src/share/ls/ob_ls_status_operator.cpp +++ b/src/share/ls/ob_ls_status_operator.cpp @@ -27,7 +27,7 @@ #include "rootserver/ob_root_utils.h" // majority #include "logservice/palf/log_define.h" // INVALID_PROPOSAL_ID #include "share/schema/ob_multi_version_schema_service.h" // ObMultiVersionSchemaService -#include "logservice/palf/scn.h" // SCN +#include "share/scn.h" // SCN using namespace oceanbase; using namespace oceanbase::common; @@ -240,7 +240,7 @@ const char* ObLSStatusOperator::ls_status_to_str(const ObLSStatus &status) } int ObLSStatusOperator::create_new_ls(const ObLSStatusInfo &ls_info, - const palf::SCN ¤t_tenant_scn, + const SCN ¤t_tenant_scn, const common::ObString &zone_priority, ObMySQLTransaction &trans) { @@ -289,7 +289,7 @@ int ObLSStatusOperator::drop_ls(const uint64_t &tenant_id, int ObLSStatusOperator::set_ls_offline(const uint64_t &tenant_id, const share::ObLSID &ls_id, const ObLSStatus &ls_status, - const palf::SCN &drop_scn, + const SCN &drop_scn, ObMySQLTransaction &trans) { UNUSEDx(drop_scn); diff --git a/src/share/ls/ob_ls_status_operator.h b/src/share/ls/ob_ls_status_operator.h index 1430bf481..642e11f20 100644 --- a/src/share/ls/ob_ls_status_operator.h +++ b/src/share/ls/ob_ls_status_operator.h @@ -39,7 +39,7 @@ namespace sqlclient class ObMySQLResult; } } -namespace palf +namespace share { class SCN; } @@ -208,7 +208,7 @@ public: * @param[in] zone_priority: for __all_ls_election_reference_info * @param[in] trans:*/ virtual int create_new_ls(const ObLSStatusInfo &ls_info, - const palf::SCN ¤t_tenant_scn, + const SCN ¤t_tenant_scn, const common::ObString &zone_priority, ObMySQLTransaction &trans) override; /* @@ -230,7 +230,7 @@ public: virtual int set_ls_offline(const uint64_t &tenant_id, const share::ObLSID &ls_id, const ObLSStatus &ls_status, - const palf::SCN &drop_scn, + const SCN &drop_scn, ObMySQLTransaction &trans) override; /* * description: update ls primary zone, need update __all_ls_status and __all_ls_election_reference diff --git a/src/share/ob_column_checksum_error_operator.cpp b/src/share/ob_column_checksum_error_operator.cpp index 27d28dc92..25c16360e 100644 --- a/src/share/ob_column_checksum_error_operator.cpp +++ b/src/share/ob_column_checksum_error_operator.cpp @@ -98,7 +98,7 @@ int ObColumnChecksumErrorOperator::insert_column_checksum_err_info_( int ObColumnChecksumErrorOperator::delete_column_checksum_err_info( ObISQLClient &sql_client, const uint64_t tenant_id, - const palf::SCN &min_frozen_scn) + const SCN &min_frozen_scn) { int ret = OB_SUCCESS; ObSqlString sql; diff --git a/src/share/ob_column_checksum_error_operator.h b/src/share/ob_column_checksum_error_operator.h index 4c3bd0c28..9edd1e019 100644 --- a/src/share/ob_column_checksum_error_operator.h +++ b/src/share/ob_column_checksum_error_operator.h @@ -16,7 +16,7 @@ #include "lib/container/ob_iarray.h" #include "lib/mysqlclient/ob_isql_client.h" #include "common/ob_tablet_id.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -32,7 +32,7 @@ public: index_column_checksum_(-1) {} virtual ~ObColumnChecksumErrorInfo() = default; - ObColumnChecksumErrorInfo(const uint64_t tenant_id, const palf::SCN &frozen_scn, const bool is_global_index, + ObColumnChecksumErrorInfo(const uint64_t tenant_id, const SCN &frozen_scn, const bool is_global_index, const int64_t data_table_id, const int64_t index_table_id, const common::ObTabletID &data_tablet_id, const common::ObTabletID &index_tablet_id) : tenant_id_(tenant_id), frozen_scn_(frozen_scn), is_global_index_(is_global_index), @@ -47,7 +47,7 @@ public: K_(data_tablet_id), K_(index_tablet_id), K_(column_id), K_(data_column_checksum), K_(index_column_checksum)); uint64_t tenant_id_; - palf::SCN frozen_scn_; + SCN frozen_scn_; bool is_global_index_; int64_t data_table_id_; int64_t index_table_id_; @@ -70,7 +70,7 @@ public: static int delete_column_checksum_err_info( common::ObISQLClient &sql_client, const uint64_t tenant_id, - const palf::SCN &min_frozen_scn); + const SCN &min_frozen_scn); private: static int insert_column_checksum_err_info_( diff --git a/src/share/ob_common_rpc_proxy.h b/src/share/ob_common_rpc_proxy.h index 8cd0a9fa7..7a9db4ab9 100644 --- a/src/share/ob_common_rpc_proxy.h +++ b/src/share/ob_common_rpc_proxy.h @@ -214,6 +214,7 @@ public: RPC_S(PRD admin_upgrade_virtual_schema, obrpc::OB_ADMIN_UPGRADE_VIRTUAL_SCHEMA); RPC_S(PRD run_job, obrpc::OB_RUN_JOB, (ObRunJobArg)); RPC_S(PRD run_upgrade_job, obrpc::OB_RUN_UPGRADE_JOB, (ObUpgradeJobArg)); + RPC_S(PRD upgrade_table_schema, obrpc::OB_UPGRADE_TABLE_SCHEMA, (ObUpgradeTableSchemaArg)); RPC_S(PR5 admin_flush_cache, obrpc::OB_ADMIN_FLUSH_CACHE, (ObAdminFlushCacheArg)); RPC_S(PR5 admin_upgrade_cmd, obrpc::OB_ADMIN_UPGRADE_CMD, (Bool)); RPC_S(PR5 admin_rolling_upgrade_cmd, obrpc::OB_ADMIN_ROLLING_UPGRADE_CMD, (ObAdminRollingUpgradeArg)); diff --git a/src/share/ob_freeze_info_proxy.cpp b/src/share/ob_freeze_info_proxy.cpp index 4f792c76d..b6ac8542d 100644 --- a/src/share/ob_freeze_info_proxy.cpp +++ b/src/share/ob_freeze_info_proxy.cpp @@ -401,7 +401,7 @@ int ObFreezeInfoProxy::construct_frozen_status_( int ObFreezeInfoProxy::get_freeze_schema_info( ObISQLClient &sql_proxy, const uint64_t tenant_id, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, TenantIdAndSchemaVersion &schema_version_info) { int ret = OB_SUCCESS; diff --git a/src/share/ob_freeze_info_proxy.h b/src/share/ob_freeze_info_proxy.h index be6a84edc..2b2a0d400 100644 --- a/src/share/ob_freeze_info_proxy.h +++ b/src/share/ob_freeze_info_proxy.h @@ -22,7 +22,7 @@ #include "lib/queue/ob_fixed_queue.h" #include "common/storage/ob_freeze_define.h" #include "share/ob_cluster_version.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_tenant_id_schema_version.h" namespace oceanbase @@ -52,7 +52,7 @@ struct ObSimpleFrozenStatus schema_version_(INVALID_SCHEMA_VERSION), cluster_version_(0) {} - ObSimpleFrozenStatus(const palf::SCN &frozen_scn, + ObSimpleFrozenStatus(const SCN &frozen_scn, const int64_t schema_version, const int64_t cluster_version) : schema_version_(schema_version), @@ -79,7 +79,7 @@ struct ObSimpleFrozenStatus { schema_version_ = ORIGIN_SCHEMA_VERSION; cluster_version_ = cluster_version; - frozen_scn_ = palf::SCN::base_scn(); + frozen_scn_ = share::SCN::base_scn(); } bool is_valid() const @@ -106,7 +106,7 @@ struct ObSimpleFrozenStatus static const uint64_t INITIAL_FROZEN_SCN_VAL = 1; static const uint64_t INVALID_FROZEN_SCN_VAL = UINT64_MAX; - palf::SCN frozen_scn_; + SCN frozen_scn_; int64_t schema_version_; int64_t cluster_version_; @@ -121,7 +121,7 @@ public: public: int get_freeze_info(common::ObISQLClient &sql_proxy, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObSimpleFrozenStatus &frozen_status); // not include initial_freeze_info @@ -130,7 +130,7 @@ public: int get_freeze_info_larger_or_equal_than( common::ObISQLClient &sql_proxy, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, common::ObIArray &frozen_statuses); int set_freeze_info(common::ObISQLClient &sql_proxy, @@ -140,24 +140,24 @@ public: // 1. get min frozen_scn, as @min_frozen_scn // 2. get all frozen status whose frozen_scn is larger than @frozen_scn int get_min_major_available_and_larger_info(common::ObISQLClient &sql_proxy, - const palf::SCN &frozen_scn, - palf::SCN &min_frozen_scn, + const SCN &frozen_scn, + SCN &min_frozen_scn, common::ObIArray &frozen_statuses); // batch delete freeze info: // frozen_scn <= upper_frozen_scn && frozen_scn > 1 int batch_delete(common::ObISQLClient &sql_proxy, - const palf::SCN &upper_frozen_scn); + const SCN &upper_frozen_scn); // for frozen_status whose frozen_scn is less than @frozen_scn, get the one with biggest frozen_scn int get_frozen_info_less_than(common::ObISQLClient &sql_proxy, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, ObSimpleFrozenStatus &frozen_status); // get frozen_status whose frozen_scn <= @frozen_scn // If @get_all = true, means 'get all matched'; Else, only get one record with highest frozen_scn. int get_frozen_info_less_than(common::ObISQLClient &sql_proxy, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, common::ObIArray &frozen_status_arr, bool get_all = true); @@ -167,13 +167,13 @@ public: int get_freeze_schema_info(common::ObISQLClient &sql_proxy, const uint64_t tenant_id, - const palf::SCN &frozen_scn, + const SCN &frozen_scn, TenantIdAndSchemaVersion &schema_version_info); private: int get_min_major_available_and_larger_info_inner_(common::ObISQLClient &sql_proxy, - const palf::SCN &frozen_scn, - palf::SCN &min_frozen_scn, + const SCN &frozen_scn, + SCN &min_frozen_scn, common::ObIArray &frozen_statuses); int construct_frozen_status_(common::sqlclient::ObMySQLResult &result, diff --git a/src/share/ob_global_autoinc_service.h b/src/share/ob_global_autoinc_service.h index c06a48598..9166ca300 100644 --- a/src/share/ob_global_autoinc_service.h +++ b/src/share/ob_global_autoinc_service.h @@ -18,7 +18,7 @@ #include "lib/lock/ob_spin_lock.h" #include "lib/mysqlclient/ob_mysql_proxy.h" #include "logservice/ob_log_base_type.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_autoincrement_param.h" #include "share/ob_autoincrement_service.h" #include "share/ob_gais_msg.h" @@ -142,7 +142,7 @@ public: int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, - const palf::SCN &scn) override final + const SCN &scn) override final { int ret = OB_SUCCESS; UNUSED(buffer); @@ -153,12 +153,12 @@ public: } // for checkpoint, do nothing - palf::SCN get_rec_scn() override final + SCN get_rec_scn() override final { - return palf::SCN::max_scn();; + return share::SCN::max_scn();; } - int flush(palf::SCN &scn) override final + int flush(share::SCN &scn) override final { int ret = OB_SUCCESS; UNUSED(scn); diff --git a/src/share/ob_global_stat_proxy.cpp b/src/share/ob_global_stat_proxy.cpp index 4772aeb0d..ab89643a1 100644 --- a/src/share/ob_global_stat_proxy.cpp +++ b/src/share/ob_global_stat_proxy.cpp @@ -31,7 +31,7 @@ const char *ObGlobalStatProxy::TENANT_ID_CNAME = "tenant_id"; int ObGlobalStatProxy::set_init_value(const int64_t core_schema_version, const int64_t baseline_schema_version, const int64_t rootservice_epoch, - const palf::SCN &snapshot_gc_scn, + const SCN &snapshot_gc_scn, const int64_t gc_schema_version) { int ret = OB_SUCCESS; @@ -58,7 +58,7 @@ int ObGlobalStatProxy::set_init_value(const int64_t core_schema_version, int ObGlobalStatProxy::set_tenant_init_global_stat( const int64_t core_schema_version, const int64_t baseline_schema_version, - const palf::SCN &snapshot_gc_scn) + const SCN &snapshot_gc_scn) { int ret = OB_SUCCESS; if (!is_valid() || core_schema_version <= 0 || baseline_schema_version < OB_INVALID_VERSION @@ -114,7 +114,7 @@ int ObGlobalStatProxy::set_baseline_schema_version(const int64_t baseline_schema return ret; } -int ObGlobalStatProxy::set_snapshot_gc_scn(const palf::SCN &snapshot_gc_scn) +int ObGlobalStatProxy::set_snapshot_gc_scn(const SCN &snapshot_gc_scn) { int ret = OB_SUCCESS; if (!is_valid() || !snapshot_gc_scn.is_valid()) { @@ -130,7 +130,7 @@ int ObGlobalStatProxy::set_snapshot_gc_scn(const palf::SCN &snapshot_gc_scn) return ret; } -int ObGlobalStatProxy::get_snapshot_gc_scn(palf::SCN &snapshot_gc_scn) +int ObGlobalStatProxy::get_snapshot_gc_scn(SCN &snapshot_gc_scn) { int ret = OB_SUCCESS; ObGlobalStatItem::ItemList list; @@ -347,7 +347,7 @@ int ObGlobalStatProxy::get(ObGlobalStatItem::ItemList &list) int ObGlobalStatProxy::select_snapshot_gc_scn_for_update( common::ObISQLClient &sql_client, const uint64_t tenant_id, - palf::SCN &snapshot_gc_scn) + SCN &snapshot_gc_scn) { int ret = OB_SUCCESS; uint64_t snapshot_gc_scn_val = 0; @@ -411,7 +411,7 @@ int ObGlobalStatProxy::select_snapshot_gc_scn_for_update( int ObGlobalStatProxy::update_snapshot_gc_scn( common::ObISQLClient &sql_client, const uint64_t tenant_id, - const palf::SCN &snapshot_gc_scn, + const SCN &snapshot_gc_scn, int64_t &affected_rows) { int ret = OB_SUCCESS; diff --git a/src/share/ob_global_stat_proxy.h b/src/share/ob_global_stat_proxy.h index 5e0d7987e..490018637 100644 --- a/src/share/ob_global_stat_proxy.h +++ b/src/share/ob_global_stat_proxy.h @@ -17,7 +17,7 @@ #include "lib/list/ob_dlist.h" #include "lib/mysqlclient/ob_mysql_proxy.h" #include "share/ob_core_table_proxy.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -50,12 +50,12 @@ public: virtual int set_init_value(const int64_t core_schema_version, const int64_t baseline_schema_version, const int64_t rootservice_epoch, - const palf::SCN &snapshot_gc_scn, + const SCN &snapshot_gc_scn, const int64_t gc_schema_version); virtual int set_tenant_init_global_stat(const int64_t core_schema_version, const int64_t baseline_schema_version, - const palf::SCN &snapshot_gc_scn); + const SCN &snapshot_gc_scn); virtual int set_core_schema_version(const int64_t core_schema_version); virtual int set_baseline_schema_version(const int64_t baseline_schema_version); @@ -69,14 +69,14 @@ public: int64_t &gc_schema_version); static int select_snapshot_gc_scn_for_update(common::ObISQLClient &sql_client, const uint64_t tenant_id, - palf::SCN &snapshot_gc_scn); + SCN &snapshot_gc_scn); static int update_snapshot_gc_scn(common::ObISQLClient &sql_client, const uint64_t tenant_id, - const palf::SCN &snapshot_gc_scn, + const SCN &snapshot_gc_scn, int64_t &affected_rows); - int get_snapshot_gc_scn(palf::SCN &snapshot_gc_scn); + int get_snapshot_gc_scn(share::SCN &snapshot_gc_scn); //interface of standby - int set_snapshot_gc_scn(const palf::SCN &snapshot_gc_scn); + int set_snapshot_gc_scn(const SCN &snapshot_gc_scn); private: int update(const ObGlobalStatItem::ItemList &list, const bool is_incremental = false); int get(ObGlobalStatItem::ItemList &list); diff --git a/src/share/ob_i_tablet_scan.h b/src/share/ob_i_tablet_scan.h index de272bd0f..6703e16c8 100644 --- a/src/share/ob_i_tablet_scan.h +++ b/src/share/ob_i_tablet_scan.h @@ -266,7 +266,7 @@ ObVTableScanParam() : ObIAllocator *scan_allocator_; ObTableScanStatistic main_table_scan_stat_; ObTableScanStatistic idx_table_scan_stat_; - palf::SCN fb_snapshot_; + share::SCN fb_snapshot_; bool is_get_; bool force_refresh_lc_; diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 763469a6e..b574ae240 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -2589,7 +2589,7 @@ ObCheckFrozenScnArg::ObCheckFrozenScnArg() bool ObCheckFrozenScnArg::is_valid() const { - return frozen_scn_.is_valid() && frozen_scn_ > palf::SCN::min_scn(); + return frozen_scn_.is_valid() && frozen_scn_ > SCN::min_scn(); } void ObCreateTabletBatchRes::reset() @@ -3830,7 +3830,8 @@ ObUpgradeJobArg::ObUpgradeJobArg() {} bool ObUpgradeJobArg::is_valid() const { - return INVALID_ACTION != action_ && version_ > 0; + return INVALID_ACTION != action_ + && (UPGRADE_POST_ACTION != action_ || version_ > 0); } int ObUpgradeJobArg::assign(const ObUpgradeJobArg &other) { @@ -3841,6 +3842,41 @@ int ObUpgradeJobArg::assign(const ObUpgradeJobArg &other) } OB_SERIALIZE_MEMBER(ObUpgradeJobArg, action_, version_); +int ObUpgradeTableSchemaArg::init( + const uint64_t tenant_id, + const uint64_t table_id) +{ + int ret = OB_SUCCESS; + ObDDLArg::reset(); + exec_tenant_id_ = tenant_id; + tenant_id_ = tenant_id; + table_id_ = table_id; + return ret; +} + +bool ObUpgradeTableSchemaArg::is_valid() const +{ + return common::OB_INVALID_TENANT_ID != exec_tenant_id_ + && common::OB_INVALID_TENANT_ID != tenant_id_ + /*index、lob table will be created with related system table*/ + && is_system_table(table_id_); +} + +int ObUpgradeTableSchemaArg::assign(const ObUpgradeTableSchemaArg &other) +{ + int ret = OB_SUCCESS; + if (this == &other) { + } else if (OB_FAIL(ObDDLArg::assign(other))) { + LOG_WARN("fail to assign ObDDLArg", KR(ret)); + } else { + tenant_id_ = other.tenant_id_; + table_id_ = other.table_id_; + } + return ret; +} + +OB_SERIALIZE_MEMBER((ObUpgradeTableSchemaArg, ObDDLArg), tenant_id_, table_id_); + int ObAdminFlushCacheArg::assign(const ObAdminFlushCacheArg &other) { int ret = OB_SUCCESS; @@ -5682,7 +5718,7 @@ int ObCreateLSArg::init(const int64_t tenant_id, const share::ObLSID &id, const ObReplicaType replica_type, const common::ObReplicaProperty &replica_property, const share::ObAllTenantInfo &tenant_info, - const palf::SCN &create_scn, + const SCN &create_scn, const lib::Worker::CompatMode &mode, const bool create_with_palf, const palf::PalfBaseInfo &palf_base_info) @@ -5828,7 +5864,7 @@ int ObLSAccessModeInfo::init( uint64_t tenant_id, const ObLSID &ls_id, const int64_t mode_version, const palf::AccessMode &access_mode, - const palf::SCN &ref_scn) + const SCN &ref_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id @@ -6101,7 +6137,7 @@ int ObContextDDLArg::assign(const ObContextDDLArg &other) return ret; } -int ObBatchCreateTabletArg::init_create_tablet(const share::ObLSID &id, const palf::SCN &major_frozen_scn) +int ObBatchCreateTabletArg::init_create_tablet(const share::ObLSID &id, const SCN &major_frozen_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!id.is_valid() || !major_frozen_scn.is_valid())) { @@ -6555,7 +6591,7 @@ ObRpcRemoteWriteDDLPrepareLogArg::ObRpcRemoteWriteDDLPrepareLogArg() int ObRpcRemoteWriteDDLPrepareLogArg::init(const uint64_t tenant_id, const share::ObLSID &ls_id, const storage::ObITable::TableKey &table_key, - const palf::SCN &start_scn, + const SCN &start_scn, const int64_t table_id, const int64_t execution_id, const int64_t ddl_task_id) @@ -6588,8 +6624,8 @@ ObRpcRemoteWriteDDLCommitLogArg::ObRpcRemoteWriteDDLCommitLogArg() int ObRpcRemoteWriteDDLCommitLogArg::init(const uint64_t tenant_id, const share::ObLSID &ls_id, const storage::ObITable::TableKey &table_key, - const palf::SCN &start_scn, - const palf::SCN &prepare_scn) + const SCN &start_scn, + const SCN &prepare_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(tenant_id == OB_INVALID_ID || !ls_id.is_valid() || !table_key.is_valid() || !start_scn.is_valid() diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 1dca9438b..5c10c47f6 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -62,10 +62,9 @@ #include "logservice/palf/palf_options.h"//access mode #include "logservice/palf/palf_base_info.h"//PalfBaseInfo #include "logservice/palf/log_define.h"//INVALID_PROPOSAL_ID -#include "logservice/palf/scn.h"//SCN +#include "share/scn.h"//SCN #include "share/location_cache/ob_vtable_location_service.h" // share::ObVtableLocationType #include "logservice/palf/log_meta_info.h"//LogConfigVersion -#include "logservice/palf/scn.h"//SCN namespace oceanbase { @@ -913,7 +912,7 @@ public: bool is_valid() const; TO_STRING_KV(K_(frozen_scn)); public: - palf::SCN frozen_scn_; + share::SCN frozen_scn_; }; struct ObGetMinSSTableSchemaVersionArg @@ -2421,7 +2420,7 @@ public: const ObReplicaType replica_type, const common::ObReplicaProperty &replica_property, const share::ObAllTenantInfo &tenant_info, - const palf::SCN &create_scn, + const share::SCN &create_scn, const lib::Worker::CompatMode &mode, const bool create_with_palf, const palf::PalfBaseInfo &palf_base_info); @@ -2442,7 +2441,7 @@ public: { return replica_property_; } - const palf::SCN &get_create_scn() const + const share::SCN &get_create_scn() const { return create_scn_; } @@ -2471,7 +2470,7 @@ private: ObReplicaType replica_type_; common::ObReplicaProperty replica_property_; share::ObAllTenantInfo tenant_info_; - palf::SCN create_scn_; + share::SCN create_scn_; lib::Worker::CompatMode compat_mode_; CreateLSType create_ls_type_; palf::PalfBaseInfo palf_base_info_; @@ -2607,7 +2606,7 @@ public: int init(uint64_t tenant_id, const share::ObLSID &ls_idd, const int64_t mode_version, const palf::AccessMode &access_mode, - const palf::SCN &ref_scn); + const share::SCN &ref_scn); int assign(const ObLSAccessModeInfo &other); TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(mode_version), K_(access_mode), K_(ref_scn)); @@ -2627,7 +2626,7 @@ public: { return mode_version_; } - const palf::SCN &get_ref_scn() const + const share::SCN &get_ref_scn() const { return ref_scn_; } @@ -2638,7 +2637,7 @@ private: share::ObLSID ls_id_; int64_t mode_version_; palf::AccessMode access_mode_; - palf::SCN ref_scn_; + share::SCN ref_scn_; }; struct ObChangeLSAccessModeRes @@ -2709,13 +2708,13 @@ public: void reset(); int assign(const ObBatchCreateTabletArg &arg); int init_create_tablet(const share::ObLSID &id_, - const palf::SCN &major_frozen_scn); + const share::SCN &major_frozen_scn); int64_t get_tablet_count() const; DECLARE_TO_STRING; public: share::ObLSID id_; - palf::SCN major_frozen_scn_; + share::SCN major_frozen_scn_; common::ObSArray table_schemas_; common::ObSArray tablets_; }; @@ -3409,8 +3408,8 @@ public: share::ObLSID ls_id_; common::ObAddr dst_server_; share::ObBackupPathString backup_path_; - palf::SCN start_scn_; - palf::SCN end_scn_; + share::SCN start_scn_; + share::SCN end_scn_; }; struct ObBackupBuildIdxArg @@ -3507,7 +3506,7 @@ public: share::ObLSID ls_id_; int64_t turn_id_; int64_t retry_id_; - palf::SCN start_scn_; + share::SCN start_scn_; common::ObAddr dst_server_; share::ObBackupPathString backup_path_; }; @@ -3528,7 +3527,7 @@ public: public: uint64_t tenant_id_; share::ObLSID ls_id_; - palf::SCN backup_scn_; + share::SCN backup_scn_; ObSArray tablet_ids_; }; @@ -4353,7 +4352,7 @@ public: common::ObString tenant_name_; common::ObString uri_; common::ObString restore_option_; - palf::SCN restore_scn_; + share::SCN restore_scn_; common::ObString kms_info_; //Encryption use common::ObString passwd_array_; // Password verification common::ObSArray table_items_; @@ -4453,8 +4452,10 @@ struct ObUpgradeJobArg public: enum Action { INVALID_ACTION, - RUN_UPGRADE_JOB, - STOP_UPGRADE_JOB + UPGRADE_POST_ACTION, + STOP_UPGRADE_JOB, + UPGRADE_SYSTEM_VARIABLE, + UPGRADE_SYSTEM_TABLE, }; public: ObUpgradeJobArg(); @@ -4466,6 +4467,26 @@ public: int64_t version_; }; +struct ObUpgradeTableSchemaArg : public ObDDLArg +{ + OB_UNIS_VERSION(1); +public: + ObUpgradeTableSchemaArg() + : ObDDLArg(), + tenant_id_(common::OB_INVALID_TENANT_ID), + table_id_(common::OB_INVALID_ID) {} + ~ObUpgradeTableSchemaArg() {} + int init(const uint64_t tenant_id, const uint64_t table_id); + bool is_valid() const; + int assign(const ObUpgradeTableSchemaArg &other); + uint64_t get_tenant_id() const { return tenant_id_; } + uint64_t get_table_id() const { return table_id_; } + TO_STRING_KV(K_(tenant_id), K_(table_id)); +private: + uint64_t tenant_id_; + uint64_t table_id_; +}; + struct ObAdminMergeArg { OB_UNIS_VERSION(1); @@ -6030,7 +6051,7 @@ public: } bool is_valid() const; public: - palf::SCN frozen_scn_; + share::SCN frozen_scn_; TO_STRING_KV(K_(frozen_scn)); }; @@ -7421,7 +7442,7 @@ public: int init(const uint64_t tenant_id, const share::ObLSID &ls_id, const storage::ObITable::TableKey &table_key, - const palf::SCN &start_scn, + const share::SCN &start_scn, const int64_t table_id, const int64_t execution_id, const int64_t ddl_task_id); @@ -7436,7 +7457,7 @@ public: uint64_t tenant_id_; share::ObLSID ls_id_; storage::ObITable::TableKey table_key_; - palf::SCN start_scn_; + share::SCN start_scn_; int64_t table_id_; int64_t execution_id_; int64_t ddl_task_id_; @@ -7453,8 +7474,8 @@ public: int init(const uint64_t tenant_id, const share::ObLSID &ls_id, const storage::ObITable::TableKey &table_key, - const palf::SCN &start_scn, - const palf::SCN &prepare_scn); + const share::SCN &start_scn, + const share::SCN &prepare_scn); bool is_valid() const { return tenant_id_ != OB_INVALID_ID && ls_id_.is_valid() && table_key_.is_valid() && start_scn_.is_valid() @@ -7465,8 +7486,8 @@ public: uint64_t tenant_id_; share::ObLSID ls_id_; storage::ObITable::TableKey table_key_; - palf::SCN start_scn_; - palf::SCN prepare_scn_; + share::SCN start_scn_; + share::SCN prepare_scn_; private: DISALLOW_COPY_AND_ASSIGN(ObRpcRemoteWriteDDLCommitLogArg); }; diff --git a/src/share/ob_snapshot_table_proxy.cpp b/src/share/ob_snapshot_table_proxy.cpp index 3eea8ad3d..978dfce56 100644 --- a/src/share/ob_snapshot_table_proxy.cpp +++ b/src/share/ob_snapshot_table_proxy.cpp @@ -53,7 +53,7 @@ int ObSnapshotInfo::init( const uint64_t tenant_id, const uint64_t tablet_id, const ObSnapShotType &snapshot_type, - const palf::SCN &snapshot_scn, + const SCN &snapshot_scn, const int64_t schema_version, const char* comment) { @@ -179,7 +179,7 @@ int ObSnapshotTableProxy::batch_add_snapshot( const share::ObSnapShotType snapshot_type, const uint64_t tenant_id, const int64_t schema_version, - const palf::SCN &snapshot_scn, + const SCN &snapshot_scn, const char *comment, const common::ObIArray &tablet_id_array) { @@ -194,7 +194,7 @@ int ObSnapshotTableProxy::batch_add_snapshot( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(tenant_id), K(schema_version), K(snapshot_scn), K(tablet_id_array)); } else { - palf::SCN snapshot_gc_scn = palf::SCN::min_scn(); + SCN snapshot_gc_scn = SCN::min_scn(); int64_t report_idx = 0; const int64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id); ObSnapshotInfo info; @@ -307,7 +307,7 @@ int ObSnapshotTableProxy::batch_remove_snapshots( share::ObSnapShotType snapshot_type, const uint64_t tenant_id, const int64_t schema_version, - const palf::SCN &snapshot_scn, + const SCN &snapshot_scn, const common::ObIArray &tablet_ids) { int ret = OB_SUCCESS; @@ -363,8 +363,8 @@ int extract_snapshot(const ObMySQLResult &result, ObSnapshotInfo &snapshot, cons { int ret = OB_SUCCESS; - palf::SCN snapshot_scn; - uint64_t snapshot_scn_val = palf::OB_INVALID_SCN_VAL; + SCN snapshot_scn; + uint64_t snapshot_scn_val = share::OB_INVALID_SCN_VAL; ObSnapShotType snapshot_type = ObSnapShotType::MAX_SNAPSHOT_TYPE; uint64_t tablet_id = UINT64_MAX; int64_t schema_version = -1; @@ -432,7 +432,7 @@ int ObSnapshotTableProxy::get_all_snapshots( } int ObSnapshotTableProxy::check_snapshot_valid( - const palf::SCN &snapshot_gc_scn, + const SCN &snapshot_gc_scn, const ObSnapshotInfo &info, bool &is_valid) const { @@ -597,7 +597,7 @@ int ObSnapshotTableProxy::get_snapshot( ObISQLClient &proxy, const uint64_t tenant_id, const ObSnapShotType snapshot_type, - const palf::SCN &snapshot_scn, + const SCN &snapshot_scn, ObSnapshotInfo &snapshot_info) { int ret = OB_SUCCESS; diff --git a/src/share/ob_snapshot_table_proxy.h b/src/share/ob_snapshot_table_proxy.h index d91bdef8d..390809cf9 100644 --- a/src/share/ob_snapshot_table_proxy.h +++ b/src/share/ob_snapshot_table_proxy.h @@ -16,7 +16,7 @@ #include "lib/container/ob_iarray.h" #include "lib/lock/ob_mutex.h" #include "share/ob_define.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -43,7 +43,7 @@ struct ObSnapshotInfo { public: ObSnapShotType snapshot_type_; - palf::SCN snapshot_scn_; + SCN snapshot_scn_; int64_t schema_version_; uint64_t tenant_id_; //tenant_id=OB_INVALID_ID represent all tenants uint64_t tablet_id_; //tablet_id=OB_INVALID_ID represent all tablets of tenant @@ -51,7 +51,7 @@ public: ObSnapshotInfo(); ~ObSnapshotInfo() {} int init(const uint64_t tenant_id, const uint64_t tablet_id, - const ObSnapShotType &snapshot_type, const palf::SCN &snapshot_scn, + const ObSnapShotType &snapshot_type, const SCN &snapshot_scn, const int64_t schema_version, const char* comment); void reset(); bool is_valid() const; @@ -70,9 +70,9 @@ struct TenantSnapshot { public: uint64_t tenant_id_; - palf::SCN snapshot_scn_; + SCN snapshot_scn_; TenantSnapshot() {} - TenantSnapshot(const uint64_t tenant_id, const palf::SCN &snapshot_scn) + TenantSnapshot(const uint64_t tenant_id, const SCN &snapshot_scn) : tenant_id_(tenant_id), snapshot_scn_(snapshot_scn) {} ~TenantSnapshot() {} void reset(); @@ -96,7 +96,7 @@ public: const share::ObSnapShotType snapshot_type, const uint64_t tenant_id, const int64_t schema_version, - const palf::SCN &snapshot_scn, + const SCN &snapshot_scn, const char *comment, const common::ObIArray &tablet_id_array); @@ -110,7 +110,7 @@ public: share::ObSnapShotType snapshot_type, const uint64_t tenant_id, const int64_t schema_version, - const palf::SCN &snapshot_scn, + const SCN &snapshot_scn, const common::ObIArray &tablet_ids); int get_all_snapshots(common::ObISQLClient &proxy, const uint64_t tenant_id, @@ -127,7 +127,7 @@ public: int get_snapshot(common::ObISQLClient &proxy, const uint64_t tenant_id, const ObSnapShotType snapshot_type, - const palf::SCN &snapshot_scn, + const SCN &snapshot_scn, ObSnapshotInfo &snapshot_info); int get_max_snapshot_info(common::ObISQLClient &proxy, @@ -148,7 +148,7 @@ public: int64_t &count); private: int gen_event_ts(int64_t &event_ts); - int check_snapshot_valid(const palf::SCN &snapshot_gc_scn, + int check_snapshot_valid(const SCN &snapshot_gc_scn, const ObSnapshotInfo &info, bool &is_valid) const; int fill_snapshot_item(const ObSnapshotInfo &info, diff --git a/src/share/ob_table_range.cpp b/src/share/ob_table_range.cpp index ed65a6941..2e9b58803 100644 --- a/src/share/ob_table_range.cpp +++ b/src/share/ob_table_range.cpp @@ -9,12 +9,11 @@ namespace oceanbase { -using palf::SCN; namespace share { -const palf::SCN ObScnRange::MIN_SCN = palf::SCN::min_scn(); -const palf::SCN ObScnRange::MAX_SCN = palf::SCN::max_scn(); +const SCN ObScnRange::MIN_SCN = SCN::min_scn(); +const SCN ObScnRange::MAX_SCN = SCN::max_scn(); ObScnRange::ObScnRange() : start_scn_(MIN_SCN), diff --git a/src/share/ob_table_range.h b/src/share/ob_table_range.h index d5b729129..f0370fbdc 100644 --- a/src/share/ob_table_range.h +++ b/src/share/ob_table_range.h @@ -7,24 +7,18 @@ #ifndef OCEANBASE_SHARE_OB_TABLE_RANGE_H_ #define OCEANBASE_SHARE_OB_TABLE_RANGE_H_ -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { -namespace palf -{ - class SCN; -} - namespace share { - struct ObScnRange { OB_UNIS_VERSION(1); public: - static const palf::SCN MIN_SCN; - static const palf::SCN MAX_SCN; + static const SCN MIN_SCN; + static const SCN MAX_SCN; ObScnRange(); int64_t hash() const; @@ -56,7 +50,7 @@ public: return !this->operator==(range); } - OB_INLINE bool contain(const palf::SCN &scn) const + OB_INLINE bool contain(const SCN &scn) const { return is_valid() && start_scn_ < scn && end_scn_ >= scn; @@ -64,8 +58,8 @@ public: TO_STRING_KV(K_(start_scn), K_(end_scn)); public: - palf::SCN start_scn_; - palf::SCN end_scn_; + SCN start_scn_; + SCN end_scn_; }; diff --git a/src/share/ob_tablet_autoincrement_param.cpp b/src/share/ob_tablet_autoincrement_param.cpp index 362c4dd65..950d53aaa 100644 --- a/src/share/ob_tablet_autoincrement_param.cpp +++ b/src/share/ob_tablet_autoincrement_param.cpp @@ -82,7 +82,14 @@ void ObTabletAutoincSeq::reset() bool ObTabletAutoincSeq::is_valid() const { - return true; + bool valid = true; + + if (intervals_.empty()) { + valid = false; + } + // TODO(shuangcan.yjw): verify elemetns in array + + return valid; } int ObTabletAutoincSeq::get_autoinc_seq_value(uint64_t &autoinc_seq) diff --git a/src/share/ob_tablet_autoincrement_param.h b/src/share/ob_tablet_autoincrement_param.h index 9a67f9b48..d4c589d07 100644 --- a/src/share/ob_tablet_autoincrement_param.h +++ b/src/share/ob_tablet_autoincrement_param.h @@ -76,7 +76,7 @@ struct ObTabletAutoincInterval final public: ObTabletAutoincInterval() : tablet_id_(), start_(0), end_(0) {} - bool is_valid() const { return tablet_id_.is_valid(); } + bool is_valid() const { return tablet_id_.is_valid() && end_ >= start_; } void reset() { tablet_id_.reset(); diff --git a/src/share/ob_tablet_autoincrement_service.cpp b/src/share/ob_tablet_autoincrement_service.cpp index f2204e8aa..471327980 100644 --- a/src/share/ob_tablet_autoincrement_service.cpp +++ b/src/share/ob_tablet_autoincrement_service.cpp @@ -18,7 +18,7 @@ #include "share/location_cache/ob_location_service.h" #include "share/ob_rpc_struct.h" #include "storage/tx_storage/ob_ls_handle.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -497,7 +497,7 @@ int ObTabletAutoincSeqRpcHandler::batch_set_tablet_autoinc_seq( if (OB_TMP_FAIL(ls_handle.get_ls()->get_tablet(autoinc_param.dest_tablet_id_, tablet_handle))) { LOG_WARN("failed to get tablet", K(tmp_ret), K(autoinc_param)); } else if (OB_TMP_FAIL(tablet_handle.get_obj()->update_tablet_autoinc_seq(autoinc_param.autoinc_seq_, - palf::SCN::max_scn()))) { + SCN::max_scn()))) { LOG_WARN("failed to update tablet autoinc seq", K(tmp_ret), K(autoinc_param)); } autoinc_param.ret_code_ = tmp_ret; @@ -512,7 +512,7 @@ int ObTabletAutoincSeqRpcHandler::replay_update_tablet_autoinc_seq( const ObLS *ls, const ObTabletID &tablet_id, const uint64_t autoinc_seq, - const palf::SCN &replay_scn) + const SCN &replay_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(ls == nullptr || !tablet_id.is_valid() || autoinc_seq == 0 || !replay_scn.is_valid_and_not_min())) { diff --git a/src/share/ob_tablet_autoincrement_service.h b/src/share/ob_tablet_autoincrement_service.h index b8dd25592..6debe8538 100644 --- a/src/share/ob_tablet_autoincrement_service.h +++ b/src/share/ob_tablet_autoincrement_service.h @@ -135,7 +135,7 @@ public: const ObLS *ls, const ObTabletID &tablet_id, const uint64_t autoinc_seq, - const palf::SCN &replay_scn); + const SCN &replay_scn); private: ObTabletAutoincSeqRpcHandler(); ~ObTabletAutoincSeqRpcHandler(); diff --git a/src/share/ob_tablet_checksum_iterator.h b/src/share/ob_tablet_checksum_iterator.h index c320a9576..9f1a214e8 100644 --- a/src/share/ob_tablet_checksum_iterator.h +++ b/src/share/ob_tablet_checksum_iterator.h @@ -46,7 +46,7 @@ public: int next(ObTabletChecksumItem &item); - void set_compaction_scn(const palf::SCN &compaction_scn) { compaction_scn_ = compaction_scn; } + void set_compaction_scn(const SCN &compaction_scn) { compaction_scn_ = compaction_scn; } private: int fetch_next_batch(); @@ -56,7 +56,7 @@ private: bool is_inited_; uint64_t tenant_id_; - palf::SCN compaction_scn_; + SCN compaction_scn_; common::ObSEArray checksum_items_; int64_t cur_idx_; common::ObISQLClient *sql_proxy_; diff --git a/src/share/ob_tablet_checksum_operator.cpp b/src/share/ob_tablet_checksum_operator.cpp index b1a8550e5..bef96127f 100644 --- a/src/share/ob_tablet_checksum_operator.cpp +++ b/src/share/ob_tablet_checksum_operator.cpp @@ -136,7 +136,7 @@ int ObTabletChecksumOperator::load_tablet_checksum_items( const ObTabletLSPair &start_pair, const int64_t batch_cnt, const uint64_t tenant_id, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, ObIArray &items) { int ret = OB_SUCCESS; @@ -283,7 +283,7 @@ int ObTabletChecksumOperator::construct_load_sql_str_( const uint64_t tenant_id, const ObTabletLSPair &start_pair, const int64_t batch_cnt, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, ObSqlString &sql) { int ret = OB_SUCCESS; @@ -291,7 +291,7 @@ int ObTabletChecksumOperator::construct_load_sql_str_( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(batch_cnt), K(compaction_scn)); } else { - if (compaction_scn > palf::SCN::min_scn()) { + if (compaction_scn > SCN::min_scn()) { if (OB_FAIL(sql.append_fmt("SELECT * FROM %s WHERE tenant_id = '%lu' and compaction_scn = %lu " "and tablet_id >= '%lu' and ls_id > %ld ", OB_ALL_TABLET_CHECKSUM_TNAME, tenant_id, compaction_scn.get_val_for_inner_table_field(), start_pair.get_tablet_id().id(), @@ -468,7 +468,7 @@ int ObTabletChecksumOperator::insert_or_update_tablet_checksum_items_( int ObTabletChecksumOperator::delete_tablet_checksum_items( ObISQLClient &sql_client, const uint64_t tenant_id, - const palf::SCN &gc_compaction_scn) + const SCN &gc_compaction_scn) { int ret = OB_SUCCESS; ObSqlString sql; @@ -532,7 +532,7 @@ int ObTabletChecksumOperator::delete_tablet_checksum_items( int ObTabletChecksumOperator::load_all_compaction_scn( ObISQLClient &sql_client, const uint64_t tenant_id, - ObIArray &compaction_scn_arr) + ObIArray &compaction_scn_arr) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { @@ -561,7 +561,7 @@ int ObTabletChecksumOperator::load_all_compaction_scn( EXTRACT_UINT_FIELD_MYSQL(*result, "dis_comp_scn", compaction_scn_val, uint64_t); } - palf::SCN tmp_compaction_scn; + SCN tmp_compaction_scn; if (FAILEDx(tmp_compaction_scn.convert_for_inner_table_field(compaction_scn_val))) { LOG_WARN("fail to convert val to SCN", KR(ret), K(compaction_scn_val)); } else if (OB_UNLIKELY(!tmp_compaction_scn.is_valid())) { diff --git a/src/share/ob_tablet_checksum_operator.h b/src/share/ob_tablet_checksum_operator.h index c9bcc8070..608eb5075 100644 --- a/src/share/ob_tablet_checksum_operator.h +++ b/src/share/ob_tablet_checksum_operator.h @@ -53,7 +53,7 @@ public: share::ObLSID ls_id_; int64_t data_checksum_; int64_t row_count_; - palf::SCN compaction_scn_; + SCN compaction_scn_; int replica_type_; ObTabletReplicaReportColumnMeta column_meta_; }; @@ -72,7 +72,7 @@ public: const ObTabletLSPair &start_pair, const int64_t batch_cnt, const uint64_t tenant_id, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, common::ObIArray &items); // multi get tablet checksum static int load_tablet_checksum_items( @@ -101,7 +101,7 @@ public: static int delete_tablet_checksum_items( common::ObISQLClient &sql_client, const uint64_t tenant_id, - const palf::SCN &gc_compaction_scn); + const SCN &gc_compaction_scn); static int delete_tablet_checksum_items( common::ObISQLClient &sql_client, const uint64_t tenant_id, @@ -109,14 +109,14 @@ public: static int load_all_compaction_scn( common::ObISQLClient &sql_client, const uint64_t tenant_id, - common::ObIArray &compaction_scn_arr); + common::ObIArray &compaction_scn_arr); private: static int construct_load_sql_str_( const uint64_t tenant_id, const ObTabletLSPair &start_pair, const int64_t batch_cnt, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, common::ObSqlString &sql); static int construct_load_sql_str_( const uint64_t tenant_id, diff --git a/src/share/ob_tablet_replica_checksum_iterator.h b/src/share/ob_tablet_replica_checksum_iterator.h index 2abe9c4a3..1cf482177 100644 --- a/src/share/ob_tablet_replica_checksum_iterator.h +++ b/src/share/ob_tablet_replica_checksum_iterator.h @@ -37,7 +37,7 @@ public: int next(ObTabletReplicaChecksumItem &item); - void set_compaction_scn(const palf::SCN &compaction_scn) { compaction_scn_ = compaction_scn; } + void set_compaction_scn(const SCN &compaction_scn) { compaction_scn_ = compaction_scn; } protected: int fetch_next_batch(); @@ -47,7 +47,7 @@ private: bool is_inited_; uint64_t tenant_id_; - palf::SCN compaction_scn_; + SCN compaction_scn_; common::ObSEArray checksum_items_; int64_t cur_idx_; common::ObISQLClient *sql_proxy_; diff --git a/src/share/ob_tablet_replica_checksum_operator.cpp b/src/share/ob_tablet_replica_checksum_operator.cpp index ccea55bad..b14843ba2 100644 --- a/src/share/ob_tablet_replica_checksum_operator.cpp +++ b/src/share/ob_tablet_replica_checksum_operator.cpp @@ -20,6 +20,8 @@ #include "lib/mysqlclient/ob_mysql_proxy.h" #include "share/inner_table/ob_inner_table_schema_constants.h" #include "share/schema/ob_column_schema.h" +#include "share/tablet/ob_tablet_info.h" +#include "share/config/ob_server_config.h" namespace oceanbase { @@ -346,14 +348,77 @@ ObTabletReplicaChecksumItem &ObTabletReplicaChecksumItem::operator=(const ObTabl /****************************** ObTabletReplicaChecksumOperator ******************************/ -int ObTabletReplicaChecksumOperator::batch_remove( +int ObTabletReplicaChecksumOperator::batch_remove_with_trans( + ObMySQLTransaction &trans, const uint64_t tenant_id, - const ObIArray &ls_ids, - const ObIArray &tablet_ids, - ObISQLClient &sql_proxy) + const common::ObIArray &tablet_replicas) { - int ret = OB_NOT_IMPLEMENT; - UNUSEDx(tenant_id, ls_ids, tablet_ids, sql_proxy); + int ret = OB_SUCCESS; + const int64_t replicas_count = tablet_replicas.count(); + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || replicas_count <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), "tablet_replica cnt", replicas_count); + } else { + int64_t start_idx = 0; + int64_t end_idx = min(MAX_BATCH_COUNT, replicas_count); + while (OB_SUCC(ret) && (start_idx < end_idx)) { + if (OB_FAIL(inner_batch_remove_by_sql_(tenant_id, tablet_replicas, start_idx, end_idx, trans))) { + LOG_WARN("fail to inner batch remove", KR(ret), K(tenant_id), K(start_idx), K(end_idx)); + } else { + start_idx = end_idx; + end_idx = min(start_idx + MAX_BATCH_COUNT, replicas_count); + } + } + } + return ret; +} + +int ObTabletReplicaChecksumOperator::inner_batch_remove_by_sql_( + const uint64_t tenant_id, + const common::ObIArray &tablet_replicas, + const int64_t start_idx, + const int64_t end_idx, + ObMySQLTransaction &trans) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) + || (tablet_replicas.count() <= 0) + || (start_idx < 0) + || (start_idx > end_idx) + || (end_idx > tablet_replicas.count()))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(start_idx), K(end_idx), + "tablet_replica cnt", tablet_replicas.count()); + } else { + ObSqlString sql; + int64_t affected_rows = 0; + if (OB_FAIL(sql.assign_fmt("DELETE FROM %s WHERE tenant_id = '%lu' AND (tablet_id, svr_ip, svr_port, ls_id) IN(", + OB_ALL_TABLET_REPLICA_CHECKSUM_TNAME, tenant_id))) { + LOG_WARN("fail to assign sql", KR(ret), K(tenant_id)); + } else { + char ip[OB_MAX_SERVER_ADDR_SIZE] = ""; + for (int64_t idx = start_idx; OB_SUCC(ret) && (idx < end_idx); ++idx) { + if (OB_UNLIKELY(!tablet_replicas.at(idx).get_server().ip_to_string(ip, sizeof(ip)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("convert server ip to string failed", KR(ret), "server", tablet_replicas.at(idx).get_server()); + } else if (OB_FAIL(sql.append_fmt("('%lu', '%s', %d, %ld)%s", + tablet_replicas.at(idx).get_tablet_id().id(), + ip, + tablet_replicas.at(idx).get_server().get_port(), + tablet_replicas.at(idx).get_ls_id().id(), + ((idx == end_idx - 1) ? ")" : ", ")))) { + LOG_WARN("fail to assign sql", KR(ret), K(tenant_id), K(idx), K(start_idx), K(end_idx)); + } + } + + const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id); + if (FAILEDx(trans.write(meta_tenant_id, sql.ptr(), affected_rows))) { + LOG_WARN("fail to execute sql", KR(ret), K(meta_tenant_id), K(sql)); + } else { + LOG_INFO("will batch delete tablet replica checksum", K(affected_rows)); + } + } + } return ret; } @@ -361,7 +426,7 @@ int ObTabletReplicaChecksumOperator::batch_get( const uint64_t tenant_id, const ObTabletLSPair &start_pair, const int64_t batch_cnt, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, ObISQLClient &sql_proxy, ObIArray &items) { @@ -538,7 +603,7 @@ int ObTabletReplicaChecksumOperator::construct_batch_get_sql_str_( const uint64_t tenant_id, const ObTabletLSPair &start_pair, const int64_t batch_cnt, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, ObSqlString &sql) { int ret = OB_SUCCESS; @@ -687,58 +752,38 @@ int ObTabletReplicaChecksumOperator::construct_tablet_replica_checksum_item_( return ret; } -int ObTabletReplicaChecksumOperator::batch_update( + +int ObTabletReplicaChecksumOperator::batch_update_with_trans( + common::ObMySQLTransaction &trans, const uint64_t tenant_id, - const ObIArray &items, - ObISQLClient &sql_proxy) + const common::ObIArray &items) { - return batch_insert_or_update_(tenant_id, items, sql_proxy, true); + return batch_insert_or_update_with_trans_(tenant_id, items, trans, true); } -int ObTabletReplicaChecksumOperator::batch_insert( +int ObTabletReplicaChecksumOperator::batch_insert_or_update_with_trans_( const uint64_t tenant_id, const ObIArray &items, - ObISQLClient &sql_proxy) -{ - return batch_insert_or_update_(tenant_id, items, sql_proxy, false); -} - -int ObTabletReplicaChecksumOperator::batch_insert_or_update_( - const uint64_t tenant_id, - const ObIArray &items, - ObISQLClient &sql_proxy, + common::ObMySQLTransaction &trans, const bool is_update) { int ret = OB_SUCCESS; - ObMySQLTransaction trans; if (OB_UNLIKELY((OB_INVALID_TENANT_ID == tenant_id) || (items.count() <= 0))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), "items count", items.count()); } else { - const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id); - if (OB_FAIL(trans.start(&sql_proxy, meta_tenant_id))) { - LOG_WARN("fail to start transaction", KR(ret), K(tenant_id), K(meta_tenant_id)); - } else { - int64_t start_idx = 0; - int64_t end_idx = min(MAX_BATCH_COUNT, items.count()); - while (OB_SUCC(ret) && (start_idx < end_idx)) { - if (OB_FAIL(inner_batch_insert_or_update_by_sql_(tenant_id, items, start_idx, - end_idx, trans, is_update))) { - LOG_WARN("fail to inner batch insert", KR(ret), K(tenant_id), K(start_idx), K(is_update)); - } else { - start_idx = end_idx; - end_idx = min(start_idx + MAX_BATCH_COUNT, items.count()); - } + int64_t start_idx = 0; + int64_t end_idx = min(MAX_BATCH_COUNT, items.count()); + while (OB_SUCC(ret) && (start_idx < end_idx)) { + if (OB_FAIL(inner_batch_insert_or_update_by_sql_(tenant_id, items, start_idx, + end_idx, trans, is_update))) { + LOG_WARN("fail to inner batch insert", KR(ret), K(tenant_id), K(start_idx), K(is_update)); + } else { + start_idx = end_idx; + end_idx = min(start_idx + MAX_BATCH_COUNT, items.count()); } } } - if (trans.is_started()) { - int trans_ret = trans.end(OB_SUCCESS == ret); - if (OB_SUCCESS != trans_ret) { - LOG_WARN("fail to end transaction", KR(trans_ret)); - ret = ((OB_SUCCESS == ret) ? trans_ret : ret); - } - } return ret; } @@ -821,7 +866,7 @@ int ObTabletReplicaChecksumOperator::check_column_checksum( const uint64_t tenant_id, const ObTableSchema &data_table_schema, const ObTableSchema &index_table_schema, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; @@ -853,7 +898,7 @@ int ObTabletReplicaChecksumOperator::check_global_index_column_checksum( const uint64_t tenant_id, const ObTableSchema &data_table_schema, const ObTableSchema &index_table_schema, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; @@ -950,7 +995,7 @@ int ObTabletReplicaChecksumOperator::check_local_index_column_checksum( const uint64_t tenant_id, const ObTableSchema &data_table_schema, const ObTableSchema &index_table_schema, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; @@ -1062,7 +1107,7 @@ int ObTabletReplicaChecksumOperator::check_local_index_column_checksum( int ObTabletReplicaChecksumOperator::get_column_checksum_sum_map_( const ObTableSchema &table_schema, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, hash::ObHashMap &column_ckm_sum_map, const ObIArray &items) { @@ -1124,7 +1169,7 @@ int ObTabletReplicaChecksumOperator::get_column_checksum_sum_map_( int ObTabletReplicaChecksumOperator::get_column_checksum_map_( const ObTableSchema &table_schema, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, hash::ObHashMap &column_ckm_map, const ObTabletReplicaChecksumItem &item) { @@ -1167,7 +1212,7 @@ int ObTabletReplicaChecksumOperator::get_column_checksum_map_( int ObTabletReplicaChecksumOperator::find_checksum_item_by_id_( const ObTabletID &tablet_id, ObIArray &items, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, int64_t &idx) { int ret = OB_SUCCESS; @@ -1284,7 +1329,7 @@ int ObTabletReplicaChecksumOperator::get_table_all_tablet_id_( } int ObTabletReplicaChecksumOperator::need_verify_checksum_( - const palf::SCN &compaction_scn, + const SCN &compaction_scn, bool &need_verify, ObIArray &schema_tablet_ids, ObIArray &items) @@ -1297,10 +1342,10 @@ int ObTabletReplicaChecksumOperator::need_verify_checksum_( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(item_cnt)); } else { - palf::SCN min_compaction_scn = palf::SCN::max_scn(); - palf::SCN max_compaction_scn = palf::SCN::min_scn(); + SCN min_compaction_scn = SCN::max_scn(); + SCN max_compaction_scn = SCN::min_scn(); for (int64_t i = 0; i < item_cnt; ++i) { - const palf::SCN &cur_compaction_scn = items.at(i).compaction_scn_; + const SCN &cur_compaction_scn = items.at(i).compaction_scn_; if (cur_compaction_scn < min_compaction_scn) { min_compaction_scn = cur_compaction_scn; } diff --git a/src/share/ob_tablet_replica_checksum_operator.h b/src/share/ob_tablet_replica_checksum_operator.h index c47691f81..2db94f7f6 100644 --- a/src/share/ob_tablet_replica_checksum_operator.h +++ b/src/share/ob_tablet_replica_checksum_operator.h @@ -22,7 +22,7 @@ #include "share/schema/ob_table_schema.h" #include "share/tablet/ob_tablet_info.h" #include "share/ob_column_checksum_error_operator.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -31,6 +31,7 @@ namespace common class ObISQLClient; class ObAddr; class ObTabletID; +class ObMySQLTransaction; namespace sqlclient { class ObMySQLResult; @@ -38,6 +39,7 @@ class ObMySQLResult; } namespace share { +class ObTabletReplica; struct ObTabletReplicaReportColumnMeta { @@ -92,7 +94,7 @@ public: common::ObTabletID tablet_id_; common::ObAddr server_; int64_t row_count_; - palf::SCN compaction_scn_; + SCN compaction_scn_; int64_t data_checksum_; ObTabletReplicaReportColumnMeta column_meta_; }; @@ -105,7 +107,7 @@ public: const uint64_t tenant_id, const ObTabletLSPair &start_pair, const int64_t batch_cnt, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, common::ObISQLClient &sql_proxy, common::ObIArray &items); static int batch_get( @@ -118,25 +120,20 @@ public: const common::ObSqlString &sql, common::ObISQLClient &sql_proxy, common::ObIArray &items); - static int batch_insert( + static int batch_update_with_trans( + common::ObMySQLTransaction &trans, const uint64_t tenant_id, - const common::ObIArray &items, - common::ObISQLClient &sql_proxy); - static int batch_update( + const common::ObIArray &item); + static int batch_remove_with_trans( + common::ObMySQLTransaction &trans, const uint64_t tenant_id, - const common::ObIArray &items, - common::ObISQLClient &sql_proxy); - static int batch_remove( - const uint64_t tenant_id, - const common::ObIArray &ls_ids, - const common::ObIArray &tablet_ids, - common::ObISQLClient &sql_proxy); + const common::ObIArray &tablet_replicas); static int check_column_checksum( const uint64_t tenant_id, const schema::ObTableSchema &data_table_schema, const schema::ObTableSchema &index_table_schema, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, common::ObMySQLProxy &sql_proxy); static int set_column_meta_with_hex_str( @@ -154,10 +151,10 @@ public: common::ObString &column_meta_hex_str); private: - static int batch_insert_or_update_( + static int batch_insert_or_update_with_trans_( const uint64_t tenant_id, const common::ObIArray &items, - common::ObISQLClient &sql_proxy, + common::ObMySQLTransaction &trans, const bool is_update); static int inner_batch_insert_or_update_by_sql_( @@ -168,6 +165,13 @@ private: common::ObISQLClient &sql_client, const bool is_update); + static int inner_batch_remove_by_sql_( + const uint64_t tenant_id, + const common::ObIArray &tablet_replicas, + const int64_t start_idx, + const int64_t end_idx, + common::ObMySQLTransaction &trans); + static int inner_batch_get_by_sql_( const uint64_t tenant_id, const common::ObSqlString &sql, @@ -178,7 +182,7 @@ private: const uint64_t tenant_id, const ObTabletLSPair &start_pair, const int64_t batch_cnt, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, common::ObSqlString &sql); static int construct_batch_get_sql_str_( @@ -204,21 +208,21 @@ private: const uint64_t tenant_id, const schema::ObTableSchema &data_table_schema, const schema::ObTableSchema &index_table_schema, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, common::ObMySQLProxy &sql_proxy); static int check_local_index_column_checksum( const uint64_t tenant_id, const schema::ObTableSchema &data_table_schema, const schema::ObTableSchema &index_table_schema, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, common::ObMySQLProxy &sql_proxy); // get column checksum_sum from items and store result in map // KV of @column_ckm_sum_map is: static int get_column_checksum_sum_map_( const schema::ObTableSchema &table_schema, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, common::hash::ObHashMap &column_ckm_sum_map, const common::ObIArray &items); @@ -226,7 +230,7 @@ private: // KV of @column_ckm_map is: static int get_column_checksum_map_( const schema::ObTableSchema &table_schema, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, common::hash::ObHashMap &column_ckm_map, const ObTabletReplicaChecksumItem &item); @@ -240,7 +244,7 @@ private: static int find_checksum_item_by_id_( const common::ObTabletID &tablet_id, common::ObIArray &items, - const palf::SCN &compaction_scn, + const SCN &compaction_scn, int64_t &idx); static int get_table_all_tablet_id_( @@ -248,7 +252,7 @@ private: common::ObIArray &schema_tablet_ids); static int need_verify_checksum_( - const palf::SCN &compaction_scn, + const SCN &compaction_scn, bool &need_verify, common::ObIArray &schema_tablet_ids, common::ObIArray &items); diff --git a/src/share/ob_tenant_info_proxy.cpp b/src/share/ob_tenant_info_proxy.cpp index dd00c2721..adac3a68c 100644 --- a/src/share/ob_tenant_info_proxy.cpp +++ b/src/share/ob_tenant_info_proxy.cpp @@ -24,7 +24,6 @@ using namespace oceanbase; using namespace oceanbase::common; -using namespace oceanbase::palf; namespace oceanbase { namespace share @@ -79,8 +78,8 @@ int ObAllTenantInfo::init(const uint64_t tenant_id, const ObTenantRole tenant_ro int ObAllTenantInfo::init( const uint64_t tenant_id, const ObTenantRole &tenant_role, const ObTenantSwitchoverStatus &switchover_status, - int64_t switchover_epoch, const palf::SCN &sync_scn, const palf::SCN &replayable_scn, - const palf::SCN &standby_scn, const palf::SCN &recovery_until_scn) + int64_t switchover_epoch, const SCN &sync_scn, const SCN &replayable_scn, + const SCN &standby_scn, const SCN &recovery_until_scn) { int ret = OB_SUCCESS; reset(); @@ -233,8 +232,8 @@ int ObAllTenantInfoProxy::load_tenant_info(const uint64_t tenant_id, int ObAllTenantInfoProxy::update_tenant_recovery_status( const uint64_t tenant_id, ObMySQLProxy *proxy, - ObTenantSwitchoverStatus status, const palf::SCN &sync_scn, - const palf::SCN &replay_scn, const palf::SCN &readable_scn) + ObTenantSwitchoverStatus status, const SCN &sync_scn, + const SCN &replay_scn, const SCN &readable_scn) { int ret = OB_SUCCESS; const uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); diff --git a/src/share/ob_tenant_info_proxy.h b/src/share/ob_tenant_info_proxy.h index d460c0057..4fe2d30cc 100644 --- a/src/share/ob_tenant_info_proxy.h +++ b/src/share/ob_tenant_info_proxy.h @@ -18,7 +18,7 @@ #include "share/ob_tenant_switchover_status.h"//ObTenantSwitchoverStatus #include "lib/container/ob_array.h"//ObArray #include "lib/container/ob_iarray.h"//ObIArray -#include "logservice/palf/scn.h" +#include "share/scn.h" //#include "share/ls/ob_ls_status_operator.h" @@ -51,13 +51,13 @@ public: */ int init(const uint64_t tenant_id, const ObTenantRole type); int init(const uint64_t tenant_id, const ObTenantRole &tenant_role, const ObTenantSwitchoverStatus &switchover_status, - int64_t switchover_epoch, const palf::SCN &sync_scn, const palf::SCN &replayable_scn, - const palf::SCN &standby_scn, const palf::SCN &recovery_until_scn); + int64_t switchover_epoch, const SCN &sync_scn, const SCN &replayable_scn, + const SCN &standby_scn, const SCN &recovery_until_scn); ObAllTenantInfo &operator=(const ObAllTenantInfo &other); int assign(const ObAllTenantInfo &other); void reset(); bool is_valid() const; - const palf::SCN get_ref_scn() const; + const SCN get_ref_scn() const; // ObTenantRole related function bool is_standby() const { return tenant_role_.is_standby(); } @@ -91,11 +91,11 @@ public:\ Property_declare_var(uint64_t, tenant_id) Property_declare_var(int64_t, switchover_epoch) - Property_declare_var(palf::SCN, sync_scn) - Property_declare_var(palf::SCN, replayable_scn) - Property_declare_var(palf::SCN, standby_scn) + Property_declare_var(share::SCN, sync_scn) + Property_declare_var(share::SCN, replayable_scn) + Property_declare_var(share::SCN, standby_scn) //TODO msy164651 no use now - Property_declare_var(palf::SCN, recovery_until_scn) + Property_declare_var(share::SCN, recovery_until_scn) #undef Property_declare_var private: ObTenantRole tenant_role_; @@ -136,9 +136,9 @@ public: static int update_tenant_recovery_status(const uint64_t tenant_id, ObMySQLProxy *proxy, ObTenantSwitchoverStatus status, - const palf::SCN &sync_scn, - const palf::SCN &replay_scn, - const palf::SCN &reabable_scn); + const SCN &sync_scn, + const SCN &replay_scn, + const SCN &reabable_scn); /** * @description: update tenant switchover status of __all_tenant_info * @param[in] tenant_id : user tenant id diff --git a/src/share/ob_upgrade_utils.cpp b/src/share/ob_upgrade_utils.cpp index 59204b552..9f58da812 100644 --- a/src/share/ob_upgrade_utils.cpp +++ b/src/share/ob_upgrade_utils.cpp @@ -10,7 +10,7 @@ * See the Mulan PubL v2 for more details. */ -#define USING_LOG_PREFIX SHARE +#define USING_LOG_PREFIX RS #include "lib/string/ob_sql_string.h" #include "share/ob_rpc_struct.h" @@ -280,17 +280,17 @@ int ObUpgradeUtils::upgrade_sys_variable( || OB_INVALID_ID == tenant_id) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); - } else if (OB_FAIL(calc_diff_sys_var(sql_client, tenant_id, update_list, add_list))) { + } else if (OB_FAIL(calc_diff_sys_var_(sql_client, tenant_id, update_list, add_list))) { LOG_WARN("fail to calc diff sys var", KR(ret), K(tenant_id)); - } else if (OB_FAIL(update_sys_var(rpc_proxy, tenant_id, update_list))) { + } else if (OB_FAIL(update_sys_var_(rpc_proxy, tenant_id, true, update_list))) { LOG_WARN("fail to update sys var", KR(ret), K(tenant_id)); - } else if (OB_FAIL(add_sys_var(sql_client, tenant_id, add_list))) { + } else if (OB_FAIL(update_sys_var_(rpc_proxy, tenant_id, false, add_list))) { LOG_WARN("fail to add sys var", KR(ret), K(tenant_id)); } return ret; } -int ObUpgradeUtils::calc_diff_sys_var( +int ObUpgradeUtils::calc_diff_sys_var_( common::ObISQLClient &sql_client, const uint64_t tenant_id, common::ObArray &update_list, @@ -361,7 +361,7 @@ int ObUpgradeUtils::calc_diff_sys_var( || 0 != hard_code_min_val.compare(min_val) || 0 != hard_code_max_val.compare(max_val)) { // sys var to modify - LOG_INFO("sys var diff, need modify", K(tenant_id), K(name), + LOG_INFO("[UPGRADE] sys var diff, need modify", K(tenant_id), K(name), K(data_type), K(flags), K(min_val), K(max_val), K(info), K(hard_code_type), K(hard_code_flag), K(hard_code_min_val), K(hard_code_max_val), K(hard_code_info)); @@ -398,7 +398,7 @@ int ObUpgradeUtils::calc_diff_sys_var( } else if (OB_FAIL(add_list.push_back(i))) { LOG_WARN("fail to push back var_store_idx", KR(ret), K(tenant_id), K(name)); } else { - LOG_INFO("sys var miss, need add", K(tenant_id), K(name), K(i)); + LOG_INFO("[UPGRADE] sys var miss, need add", K(tenant_id), K(name), K(i)); } } } @@ -406,15 +406,11 @@ int ObUpgradeUtils::calc_diff_sys_var( return ret; } -/* - * This function is used to restore backup data from cluster with lower cluster version. - * For modified system variable schema in physical restore, we compensate by methods below. - * 1. Modify system variable(except value) by DDL in physical restore. - * 2. Observer should run with system variable schema which is not modified yet. - */ -int ObUpgradeUtils::update_sys_var( +// modify & add sys var according by hard code schema +int ObUpgradeUtils::update_sys_var_( obrpc::ObCommonRpcProxy &rpc_proxy, const uint64_t tenant_id, + const bool is_update, common::ObArray &update_list) { int ret = OB_SUCCESS; @@ -423,7 +419,9 @@ int ObUpgradeUtils::update_sys_var( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); } else { + const int64_t timeout = GCONF.internal_sql_execute_timeout; for (int64_t i = 0; OB_SUCC(ret) && i < update_list.count(); i++) { + int64_t start_ts = ObTimeUtility::current_time(); int64_t var_store_idx = update_list.at(i); const ObString &name = ObSysVariables::get_name(var_store_idx); const ObObjType &type = ObSysVariables::get_type(var_store_idx); @@ -436,9 +434,9 @@ int ObUpgradeUtils::update_sys_var( ObSysParam sys_param; obrpc::ObAddSysVarArg arg; arg.exec_tenant_id_ = tenant_id; - arg.if_not_exist_ = true; + arg.if_not_exist_ = true; // not used arg.sysvar_.set_tenant_id(tenant_id); - arg.update_sys_var_ = true; + arg.update_sys_var_ = is_update; if (OB_FAIL(sys_param.init(tenant_id, zone, name.ptr(), type, value.ptr(), min.ptr(), max.ptr(), info.ptr(), flag))) { LOG_WARN("sys_param init failed", KR(ret), K(tenant_id), K(name), @@ -448,298 +446,16 @@ int ObUpgradeUtils::update_sys_var( LOG_WARN("sys param is invalid", KR(ret), K(tenant_id), K(sys_param)); } else if (OB_FAIL(ObSchemaUtils::convert_sys_param_to_sysvar_schema(sys_param, arg.sysvar_))) { LOG_WARN("convert sys param to sysvar schema failed", KR(ret)); - } else if (OB_FAIL(rpc_proxy.add_system_variable(arg))) { - LOG_WARN("add system variable failed", KR(ret), K(arg)); - /*} else if (OB_FAIL(execute_update_sys_var_sql(sql_client, tenant_id, sys_param))) { - LOG_WARN("fail to execute update sys var sql", KR(ret), K(tenant_id)); - } else if (OB_FAIL(execute_update_sys_var_history_sql(sql_client, tenant_id, sys_param))) { - LOG_WARN("fail to execute update sys var history sql", KR(ret), K(tenant_id));*/ + } else if (OB_FAIL(rpc_proxy.timeout(timeout).add_system_variable(arg))) { + LOG_WARN("add system variable failed", KR(ret), K(timeout), K(arg)); } + LOG_INFO("[UPGRADE] finish upgrade system variable", + KR(ret), K(tenant_id), K(name), "cost", ObTimeUtility::current_time() - start_ts); } } return ret; } -int ObUpgradeUtils::execute_update_sys_var_sql( - common::ObISQLClient &sql_client, - const uint64_t tenant_id, - const ObSysParam &sys_param) -{ - int ret = OB_SUCCESS; - if (OB_INVALID_TENANT_ID == tenant_id - || OB_INVALID_ID == tenant_id) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); - } else { - int64_t affected_rows = 0; - ObDMLSqlSplicer dml; - ObDMLExecHelper exec(sql_client, tenant_id); - if (OB_FAIL(gen_basic_sys_variable_dml(tenant_id, sys_param, dml))) { - LOG_WARN("fail to gen dml", KR(ret), K(tenant_id), K(sys_param)); - } else if (OB_FAIL(exec.exec_update(OB_ALL_SYS_VARIABLE_TNAME, dml, affected_rows))) { - LOG_WARN("execute insert failed", KR(ret)); - } else if (!is_zero_row(affected_rows) && !is_single_row(affected_rows)) { - LOG_WARN("invalid affected_rows", KR(ret), K(tenant_id), K(affected_rows)); - } else { - LOG_INFO("[UPGRADE] modify sys var", KR(ret), K(tenant_id), K(sys_param)); - } - } - return ret; -} - -int ObUpgradeUtils::execute_update_sys_var_history_sql( - common::ObISQLClient &sql_client, - const uint64_t tenant_id, - const ObSysParam &sys_param) -{ - int ret = OB_SUCCESS; - if (OB_INVALID_TENANT_ID == tenant_id - || OB_INVALID_ID == tenant_id) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); - } else { - int64_t schema_version = OB_INVALID_VERSION; - { - SMART_VAR(ObMySQLProxy::MySQLResult, res) { - ObMySQLResult *result = NULL; - ObSqlString sql; - if (OB_FAIL(sql.append_fmt( - "select schema_version from %s where tenant_id = %lu" - " and zone = '' and name = '%s' order by schema_version desc limit 1", - OB_ALL_SYS_VARIABLE_HISTORY_TNAME, - ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id), - sys_param.name_))) { - LOG_WARN("fail to append sql", KR(ret), K(tenant_id), K(sql)); - } else if (OB_FAIL(sql_client.read(res, tenant_id, sql.ptr()))) { - LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(sql)); - } else if (NULL == (result = res.get_result())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("result is not expected to be NULL", KR(ret), K(tenant_id), K(sql)); - } else if (OB_FAIL(result->next())) { - LOG_WARN("fail to get row", KR(ret), K(tenant_id), K(sql)); - } else { - EXTRACT_INT_FIELD_MYSQL(*result, "schema_version", schema_version, int64_t); - } - } - } - if (OB_SUCC(ret)) { - int64_t affected_rows = 0; - ObDMLSqlSplicer dml; - ObDMLExecHelper exec(sql_client, tenant_id); - if (OB_INVALID_VERSION == schema_version) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid schema_version", KR(ret), K(tenant_id), K(sys_param)); - } else if (OB_FAIL(gen_basic_sys_variable_dml(tenant_id, sys_param, dml))) { - LOG_WARN("fail to gen dml", KR(ret), K(tenant_id), K(sys_param)); - } else if (OB_FAIL(dml.add_pk_column("schema_version", schema_version))) { - LOG_WARN("fail to add column", KR(ret), K(tenant_id), K(schema_version)); - } else if (OB_FAIL(exec.exec_update(OB_ALL_SYS_VARIABLE_HISTORY_TNAME, dml, affected_rows))) { - LOG_WARN("execute insert failed", KR(ret)); - } else if (!is_zero_row(affected_rows) && !is_single_row(affected_rows)) { - LOG_WARN("invalid affected_rows", KR(ret), K(tenant_id), K(affected_rows)); - } else { - LOG_INFO("[UPGRADE] modify sys var history", KR(ret), K(tenant_id), K(sys_param)); - } - } - } - return ret; -} - -/* - * This function is used to restore backup data from cluster with lower cluster version. - * For missing system variable schema in physical restore, we compensate by methods below. - * 1. Missing system variable schema will be added according to hardcoded meta schema when refreshing schema. - * 2. (Not necessary) Modify __all_sys_variable/__all_sys_variable_history, so we can construct system variable schema - * from inner table when observer restarts. - */ -int ObUpgradeUtils::add_sys_var( - common::ObISQLClient &sql_client, - const uint64_t tenant_id, - common::ObArray &add_list) -{ - int ret = OB_SUCCESS; - if (OB_INVALID_TENANT_ID == tenant_id - || OB_INVALID_ID == tenant_id) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); - } else { - ObArenaAllocator allocator("AddSysVar"); - for (int64_t i = 0; OB_SUCC(ret) && i < add_list.count(); i++) { - int64_t var_store_idx = add_list.at(i); - const ObString &name = ObSysVariables::get_name(var_store_idx); - const ObObjType &type = ObSysVariables::get_type(var_store_idx); - const ObString &min = ObSysVariables::get_min(var_store_idx); - const ObString &max = ObSysVariables::get_max(var_store_idx); - const ObString &info = ObSysVariables::get_info(var_store_idx); - const int64_t flag = ObSysVariables::get_flags(var_store_idx); - const ObString zone(""); - ObSysParam sys_param; - ObString value; - if (OB_FAIL(convert_sys_variable_value(var_store_idx, allocator, value))) { - LOG_WARN("fail to get sys variable value", KR(ret), K(tenant_id), K(var_store_idx)); - } else if (OB_FAIL(sys_param.init(tenant_id, zone, name.ptr(), type, - value.ptr(), min.ptr(), max.ptr(), info.ptr(), flag))) { - LOG_WARN("sys_param init failed", KR(ret), K(tenant_id), K(name), - K(type), K(value), K(min), K(max), K(info), K(flag)); - } else if (!sys_param.is_valid()) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("sys param is invalid", KR(ret), K(tenant_id), K(sys_param)); - } else if (OB_FAIL(execute_add_sys_var_sql(sql_client, tenant_id, sys_param))) { - LOG_WARN("fail to execute add sys var sql", KR(ret), K(tenant_id)); - } else if (OB_FAIL(execute_add_sys_var_history_sql(sql_client, tenant_id, sys_param))) { - LOG_WARN("fail to execute add sys var history sql", KR(ret), K(tenant_id)); - } - } - } - return ret; -} - -// C++ implement for special_update_sys_vars_for_tenant() in python upgrade script. -int ObUpgradeUtils::convert_sys_variable_value( - const int64_t var_store_idx, - common::ObIAllocator &allocator, - ObString &value) -{ - int ret = OB_SUCCESS; - const ObString &name = ObSysVariables::get_name(var_store_idx); - if (0 == name.compare("nls_date_format")) { - if (OB_FAIL(ob_write_string( - allocator, ObString("YYYY-MM-DD HH24:MI:SS"), value))) { - LOG_WARN("fail to write string", KR(ret), K(name)); - } - } else if (0 == name.compare("nls_timestamp_format")) { - if (OB_FAIL(ob_write_string( - allocator, ObString("YYYY-MM-DD HH24:MI:SS.FF"), value))) { - LOG_WARN("fail to write string", KR(ret), K(name)); - } - } else if (0 == name.compare("nls_timestamp_tz_format")) { - if (OB_FAIL(ob_write_string( - allocator, ObString("YYYY-MM-DD HH24:MI:SS.FF TZR TZD"), value))) { - LOG_WARN("fail to write string", KR(ret), K(name)); - } - } else { - const ObString &ori_value = ObSysVariables::get_value(var_store_idx); - value.assign_ptr(ori_value.ptr(), ori_value.length()); - } - return ret; -} - -int ObUpgradeUtils::execute_add_sys_var_sql( - common::ObISQLClient &sql_client, - const uint64_t tenant_id, - const ObSysParam &sys_param) -{ - int ret = OB_SUCCESS; - if (OB_INVALID_TENANT_ID == tenant_id - || OB_INVALID_ID == tenant_id) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); - } else { - int64_t affected_rows = 0; - ObDMLSqlSplicer dml; - ObDMLExecHelper exec(sql_client, tenant_id); - if (OB_FAIL(gen_basic_sys_variable_dml(tenant_id, sys_param, dml))) { - LOG_WARN("fail to gen dml", KR(ret), K(tenant_id), K(sys_param)); - } else if (OB_FAIL(dml.add_column("value", FORMAT_STR(ObString(sys_param.value_))))) { - LOG_WARN("fail to gen dml", KR(ret), K(tenant_id), K(sys_param)); - } else if (OB_FAIL(exec.exec_replace(OB_ALL_SYS_VARIABLE_TNAME, dml, affected_rows))) { - LOG_WARN("execute insert failed", KR(ret)); - } else if (!is_zero_row(affected_rows) - && !is_single_row(affected_rows) - && !is_double_row(affected_rows)) { - LOG_WARN("invalid affected_rows", KR(ret), K(tenant_id), K(affected_rows)); - } else { - LOG_INFO("[UPGRADE] add sys var", KR(ret), K(tenant_id), K(sys_param)); - } - } - return ret; -} - -int ObUpgradeUtils::execute_add_sys_var_history_sql( - common::ObISQLClient &sql_client, - const uint64_t tenant_id, - const ObSysParam &sys_param) -{ - int ret = OB_SUCCESS; - if (OB_INVALID_TENANT_ID == tenant_id - || OB_INVALID_ID == tenant_id) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); - } else { - int64_t schema_version = OB_INVALID_VERSION; - { - SMART_VAR(ObMySQLProxy::MySQLResult, res) { - ObMySQLResult *result = NULL; - ObSqlString sql; - if (OB_FAIL(sql.append_fmt( - "select schema_version from %s where tenant_id = %lu" - " order by schema_version asc limit 1", - OB_ALL_SYS_VARIABLE_HISTORY_TNAME, - ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id)))) { - LOG_WARN("fail to append sql", KR(ret), K(tenant_id), K(sql)); - } else if (OB_FAIL(sql_client.read(res, tenant_id, sql.ptr()))) { - LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(sql)); - } else if (NULL == (result = res.get_result())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("result is not expected to be NULL", KR(ret), K(tenant_id), K(sql)); - } else if (OB_FAIL(result->next())) { - LOG_WARN("fail to get row", KR(ret), K(tenant_id), K(sql)); - } else { - EXTRACT_INT_FIELD_MYSQL(*result, "schema_version", schema_version, int64_t); - } - } - } - if (OB_SUCC(ret)) { - int64_t affected_rows = 0; - ObDMLSqlSplicer dml; - ObDMLExecHelper exec(sql_client, tenant_id); - if (OB_INVALID_VERSION == schema_version) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid schema_version", KR(ret), K(tenant_id), K(sys_param)); - } else if (OB_FAIL(gen_basic_sys_variable_dml(tenant_id, sys_param, dml))) { - LOG_WARN("fail to gen dml", KR(ret), K(tenant_id), K(sys_param)); - } else if (OB_FAIL(dml.add_pk_column("schema_version", schema_version)) - || OB_FAIL(dml.add_column("value", FORMAT_STR(ObString(sys_param.value_)))) - || OB_FAIL(dml.add_column("is_deleted", 0))) { - LOG_WARN("fail to add column", KR(ret), K(tenant_id), K(schema_version)); - } else if (OB_FAIL(exec.exec_replace(OB_ALL_SYS_VARIABLE_HISTORY_TNAME, dml, affected_rows))) { - LOG_WARN("execute insert failed", KR(ret)); - } else if (!is_zero_row(affected_rows) - && !is_single_row(affected_rows) - && !is_double_row(affected_rows)) { - LOG_WARN("invalid affected_rows", KR(ret), K(tenant_id), K(affected_rows)); - } else { - LOG_INFO("[UPGRADE] add sys var history", KR(ret), K(tenant_id), K(sys_param)); - } - } - } - return ret; -} - -int ObUpgradeUtils::gen_basic_sys_variable_dml( - const uint64_t tenant_id, - const ObSysParam &sys_param, - ObDMLSqlSplicer &dml) -{ - int ret = OB_SUCCESS; - if (OB_INVALID_TENANT_ID == tenant_id - || OB_INVALID_ID == tenant_id) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id)); - } else if ( - OB_FAIL(dml.add_pk_column("tenant_id", - ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id))) - || OB_FAIL(dml.add_pk_column("zone", "")) - || OB_FAIL(dml.add_pk_column("name", FORMAT_STR(ObString(sys_param.name_)))) - || OB_FAIL(dml.add_column("data_type", sys_param.data_type_)) - || OB_FAIL(dml.add_column("min_val", FORMAT_STR(ObString(sys_param.min_val_)))) - || OB_FAIL(dml.add_column("max_val", FORMAT_STR(ObString(sys_param.max_val_)))) - || OB_FAIL(dml.add_column("info", FORMAT_STR(ObString(sys_param.info_)))) - || OB_FAIL(dml.add_column("flags", sys_param.flags_))) { - LOG_WARN("fail to add column", KR(ret), K(tenant_id), K(sys_param)); - } - return ret; -} /* =========== upgrade sys variable end =========== */ /* =========== upgrade sys stat =========== */ diff --git a/src/share/ob_upgrade_utils.h b/src/share/ob_upgrade_utils.h index 9854bf7e5..57f87784d 100644 --- a/src/share/ob_upgrade_utils.h +++ b/src/share/ob_upgrade_utils.h @@ -38,7 +38,6 @@ public: static int can_run_upgrade_job(rootserver::ObRsJobType job_type, bool &can); static int check_upgrade_job_passed(rootserver::ObRsJobType job_type); static int check_schema_sync(bool &is_sync); - /* physical restore related */ // upgrade_sys_variable()/upgrade_sys_stat() can be called when enable_ddl = false. static int upgrade_sys_variable( obrpc::ObCommonRpcProxy &rpc_proxy, @@ -51,42 +50,16 @@ private: static int check_rs_job_success(rootserver::ObRsJobType job_type, bool &success); /* upgrade sys variable */ - static int calc_diff_sys_var( + static int calc_diff_sys_var_( common::ObISQLClient &sql_client, const uint64_t tenant_id, common::ObArray &update_list, common::ObArray &add_list); - static int update_sys_var( + static int update_sys_var_( obrpc::ObCommonRpcProxy &rpc_proxy, const uint64_t tenant_id, + const bool is_update, common::ObArray &update_list); - static int add_sys_var(common::ObISQLClient &sql_client, - const uint64_t tenant_id, - common::ObArray &add_list); - static int execute_update_sys_var_sql( - common::ObISQLClient &sql_client, - const uint64_t tenant_id, - const share::schema::ObSysParam &sys_param); - static int execute_update_sys_var_history_sql( - common::ObISQLClient &sql_client, - const uint64_t tenant_id, - const share::schema::ObSysParam &sys_param); - static int execute_add_sys_var_sql( - common::ObISQLClient &sql_client, - const uint64_t tenant_id, - const share::schema::ObSysParam &sys_param); - static int execute_add_sys_var_history_sql( - common::ObISQLClient &sql_client, - const uint64_t tenant_id, - const share::schema::ObSysParam &sys_param); - static int convert_sys_variable_value( - const int64_t var_store_idx, - common::ObIAllocator &allocator, - common::ObString &value); - static int gen_basic_sys_variable_dml( - const uint64_t tenant_id, - const share::schema::ObSysParam &sys_param, - share::ObDMLSqlSplicer &dml); /* upgrade sys variable end */ static int filter_sys_stat( common::ObISQLClient &sql_client, diff --git a/src/share/ob_zone_merge_info.cpp b/src/share/ob_zone_merge_info.cpp index f5e8d4f54..321c17309 100644 --- a/src/share/ob_zone_merge_info.cpp +++ b/src/share/ob_zone_merge_info.cpp @@ -23,7 +23,6 @@ namespace oceanbase { namespace share { -using namespace oceanbase::palf; template T first_param_percnetage(const T first, const T second) @@ -288,6 +287,11 @@ bool ObGlobalMergeInfo::is_merge_error() const return (is_merge_error_.get_value() > 0); } +bool ObGlobalMergeInfo::is_in_verifying_status() const +{ + return (ObZoneMergeInfo::MERGE_STATUS_VERIFYING == merge_status_.get_value()); +} + bool ObGlobalMergeInfo::is_valid() const { bool is_valid = true; diff --git a/src/share/ob_zone_merge_info.h b/src/share/ob_zone_merge_info.h index 765467c56..7c235fbb1 100644 --- a/src/share/ob_zone_merge_info.h +++ b/src/share/ob_zone_merge_info.h @@ -24,7 +24,7 @@ #include "common/ob_zone_status.h" #include "common/ob_zone_type.h" #include "share/ob_replica_info.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -34,7 +34,7 @@ struct ObMergeInfoItem : public common::ObDLinkBase { public: typedef common::ObDList ItemList; - ObMergeInfoItem(ItemList &list, const char *name, const palf::SCN &scn, const bool need_update); + ObMergeInfoItem(ItemList &list, const char *name, const SCN &scn, const bool need_update); ObMergeInfoItem(ItemList &list, const char *name, const int64_t value, const bool need_update); ObMergeInfoItem(const ObMergeInfoItem &item); @@ -43,10 +43,10 @@ public: void assign_value(const ObMergeInfoItem &item); bool is_valid() const; void set_val(const int64_t value, const bool need_update); - void set_scn(const palf::SCN &scn, const bool need_update); + void set_scn(const SCN &scn, const bool need_update); int set_scn(const uint64_t scn_val); int set_scn(const uint64_t scn_val, const bool need_update); - const palf::SCN &get_scn() const { return scn_; } + const SCN &get_scn() const { return scn_; } uint64_t get_scn_val() const { return scn_.get_val_for_inner_table_field(); } int64_t get_value() const { return value_; } @@ -54,7 +54,7 @@ public: public: const char *name_; bool is_scn_; - palf::SCN scn_; + SCN scn_; int64_t value_; bool need_update_; // used to mark the table field need to be updated or not }; @@ -89,10 +89,10 @@ public: bool is_in_merge() const; bool need_merge(const int64_t broadcast_version) const; - const palf::SCN &broadcast_scn() const { return broadcast_scn_.get_scn(); } - const palf::SCN &last_merged_scn() const { return last_merged_scn_.get_scn(); } - const palf::SCN &all_merged_scn() const { return all_merged_scn_.get_scn(); } - const palf::SCN &frozen_scn() const { return frozen_scn_.get_scn(); } + const SCN &broadcast_scn() const { return broadcast_scn_.get_scn(); } + const SCN &last_merged_scn() const { return last_merged_scn_.get_scn(); } + const SCN &all_merged_scn() const { return all_merged_scn_.get_scn(); } + const SCN &frozen_scn() const { return frozen_scn_.get_scn(); } TO_STRING_KV(K_(tenant_id), K_(zone), K_(is_merging), K_(broadcast_scn), K_(last_merged_scn), K_(last_merged_time), K_(all_merged_scn), K_(merge_start_time), K_(merge_status), K_(frozen_scn), @@ -123,14 +123,15 @@ public: bool is_in_merge() const; bool is_valid() const; bool is_merge_error() const; + bool is_in_verifying_status() const; ObGlobalMergeInfo &operator = (const ObGlobalMergeInfo &other) = delete; int assign(const ObGlobalMergeInfo &other); // differ from assign, only exclude 'need_update_' copy int assign_value(const ObGlobalMergeInfo &other); - const palf::SCN &frozen_scn() const { return frozen_scn_.get_scn(); } - const palf::SCN &global_broadcast_scn() const { return global_broadcast_scn_.get_scn(); } - const palf::SCN &last_merged_scn() const { return last_merged_scn_.get_scn(); } + const SCN &frozen_scn() const { return frozen_scn_.get_scn(); } + const SCN &global_broadcast_scn() const { return global_broadcast_scn_.get_scn(); } + const SCN &last_merged_scn() const { return last_merged_scn_.get_scn(); } TO_STRING_KV(K_(tenant_id), K_(cluster), K_(frozen_scn), K_(global_broadcast_scn), K_(last_merged_scn), K_(is_merge_error), @@ -165,7 +166,7 @@ public: int64_t merged_tablet_cnt_; int64_t merged_data_size_; - palf::SCN smallest_snapshot_scn_; + SCN smallest_snapshot_scn_; int64_t get_merged_tablet_percentage() const; int64_t get_merged_data_percentage() const; diff --git a/src/share/restore/ob_log_archive_source.h b/src/share/restore/ob_log_archive_source.h index fcc7a0076..45af82d91 100644 --- a/src/share/restore/ob_log_archive_source.h +++ b/src/share/restore/ob_log_archive_source.h @@ -18,7 +18,7 @@ #include "lib/utility/ob_macro_utils.h" #include "lib/utility/ob_print_utils.h" #include "lib/worker.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include namespace oceanbase { @@ -60,7 +60,7 @@ struct ObLogArchiveSourceItem int64_t id_; ObLogArchiveSourceType type_; common::ObString value_; - palf::SCN until_scn_; + SCN until_scn_; lib::ObArenaAllocator allocator_; ObLogArchiveSourceItem() : tenant_id_(), @@ -70,7 +70,7 @@ struct ObLogArchiveSourceItem allocator_() {} ObLogArchiveSourceItem(const uint64_t tenant_id, const int64_t id, - const palf::SCN &until_scn) : + const SCN &until_scn) : tenant_id_(tenant_id), id_(id), type_(ObLogArchiveSourceType::INVALID), @@ -80,7 +80,7 @@ struct ObLogArchiveSourceItem const int64_t id, const ObLogArchiveSourceType &type, const ObString &value, - const palf::SCN &until_scn) : + const SCN &until_scn) : tenant_id_(tenant_id), id_(id), type_(type), diff --git a/src/share/restore/ob_log_archive_source_mgr.cpp b/src/share/restore/ob_log_archive_source_mgr.cpp index 35741b98e..653bdfb63 100644 --- a/src/share/restore/ob_log_archive_source_mgr.cpp +++ b/src/share/restore/ob_log_archive_source_mgr.cpp @@ -40,7 +40,7 @@ int ObLogArchiveSourceMgr::init(const uint64_t tenant_id, ObISQLClient *proxy) return ret; } -int ObLogArchiveSourceMgr::update_recovery_until_ts(const palf::SCN &recovery_until_scn) +int ObLogArchiveSourceMgr::update_recovery_until_ts(const SCN &recovery_until_scn) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -76,7 +76,7 @@ int ObLogArchiveSourceMgr::delete_source() return ret; } -int ObLogArchiveSourceMgr::add_service_source(const palf::SCN &recovery_until_scn, +int ObLogArchiveSourceMgr::add_service_source(const SCN &recovery_until_scn, const ObAddr &addr) { int ret = OB_SUCCESS; @@ -92,7 +92,7 @@ int ObLogArchiveSourceMgr::add_service_source(const palf::SCN &recovery_until_sc return ret; } -int ObLogArchiveSourceMgr::add_location_source(const palf::SCN &recovery_until_scn, +int ObLogArchiveSourceMgr::add_location_source(const SCN &recovery_until_scn, const ObString &archive_dest) { int ret = OB_SUCCESS; @@ -124,7 +124,7 @@ int ObLogArchiveSourceMgr::add_location_source(const palf::SCN &recovery_until_s return ret; } -int ObLogArchiveSourceMgr::add_rawpath_source(const palf::SCN &recovery_until_scn, const DirArray &array) +int ObLogArchiveSourceMgr::add_rawpath_source(const SCN &recovery_until_scn, const DirArray &array) { return OB_NOT_SUPPORTED; } diff --git a/src/share/restore/ob_log_archive_source_mgr.h b/src/share/restore/ob_log_archive_source_mgr.h index 7172b4cde..7719277bd 100644 --- a/src/share/restore/ob_log_archive_source_mgr.h +++ b/src/share/restore/ob_log_archive_source_mgr.h @@ -39,19 +39,19 @@ public: int init(const uint64_t tenant_id, ObISQLClient *proxy); public: // add source with net service - int add_service_source(const palf::SCN &recovery_until_scn, const ObAddr &addr); + int add_service_source(const SCN &recovery_until_scn, const ObAddr &addr); // add source with archive dest // 1. nfs example // file:///data/1/ // 2. oss example // oss://backup_dir/?host=xxx.com&access_id=111&access_key=222 // 3. cos example - int add_location_source(const palf::SCN &recovery_until_scn, const ObString &archive_dest); + int add_location_source(const SCN &recovery_until_scn, const ObString &archive_dest); // add source with raw pieces - int add_rawpath_source(const palf::SCN &recovery_until_scn, const DirArray &array); + int add_rawpath_source(const SCN &recovery_until_scn, const DirArray &array); // modify log archive source recovery until ts - int update_recovery_until_ts(const palf::SCN &recovery_until_scn); + int update_recovery_until_ts(const SCN &recovery_until_scn); // delete all log archive source int delete_source(); diff --git a/src/share/restore/ob_physical_restore_info.cpp b/src/share/restore/ob_physical_restore_info.cpp index 622d5d165..f04518007 100644 --- a/src/share/restore/ob_physical_restore_info.cpp +++ b/src/share/restore/ob_physical_restore_info.cpp @@ -329,7 +329,7 @@ void ObPhysicalRestoreJob::reset() status_ = PhysicalRestoreStatus::PHYSICAL_RESTORE_MAX_STATUS; comment_.reset(); restore_start_ts_ = 0; - restore_scn_ = palf::SCN::min_scn(); + restore_scn_ = SCN::min_scn(); post_cluster_version_ = 0; source_cluster_version_ = 0; restore_option_.reset(); diff --git a/src/share/restore/ob_physical_restore_info.h b/src/share/restore/ob_physical_restore_info.h index d2f753f0c..22cf0e400 100644 --- a/src/share/restore/ob_physical_restore_info.h +++ b/src/share/restore/ob_physical_restore_info.h @@ -159,7 +159,7 @@ public: Property_declare_ObString(comment) Property_declare_int(int64_t, restore_start_ts) - Property_declare_int(palf::SCN, restore_scn) + Property_declare_int(share::SCN, restore_scn) Property_declare_int(uint64_t, post_cluster_version) Property_declare_int(uint64_t, source_cluster_version) //from cmd diff --git a/src/share/restore/ob_physical_restore_table_operator.cpp b/src/share/restore/ob_physical_restore_table_operator.cpp index d6cce9058..eb32989e8 100644 --- a/src/share/restore/ob_physical_restore_table_operator.cpp +++ b/src/share/restore/ob_physical_restore_table_operator.cpp @@ -459,8 +459,8 @@ int ObPhysicalRestoreTableOperator::retrieve_restore_option( RETRIEVE_INT_VALUE(restore_start_ts, job); if (OB_SUCC(ret)) { if (name == "restore_scn") { - uint64_t current_value = palf::OB_INVALID_SCN_VAL; - palf::SCN restore_scn; + uint64_t current_value = share::OB_INVALID_SCN_VAL; + SCN restore_scn; if (OB_FAIL(retrieve_uint_value(result, current_value))) { LOG_WARN("fail to retrive int value", K(ret), "column_name", "restore_scn"); } else if (OB_FAIL(restore_scn.convert_for_inner_table_field(current_value))) { diff --git a/src/share/restore/ob_restore_persist_helper.cpp b/src/share/restore/ob_restore_persist_helper.cpp index 99b418cef..2c3d7355a 100644 --- a/src/share/restore/ob_restore_persist_helper.cpp +++ b/src/share/restore/ob_restore_persist_helper.cpp @@ -930,7 +930,7 @@ int ObRestorePersistHelper::inc_finished_restored_block_bytes( int ObRestorePersistHelper::update_log_restore_progress( common::ObISQLClient &proxy, const ObLSRestoreJobPersistKey &ls_key, - const palf::SCN &last_replay_scn) const + const SCN &last_replay_scn) const { int ret = OB_SUCCESS; int64_t affected_rows = 0; diff --git a/src/share/restore/ob_restore_persist_helper.h b/src/share/restore/ob_restore_persist_helper.h index de3dc810e..81c0a9260 100644 --- a/src/share/restore/ob_restore_persist_helper.h +++ b/src/share/restore/ob_restore_persist_helper.h @@ -20,7 +20,7 @@ #include "share/ob_inner_table_operator.h" #include "share/restore/ob_ls_restore_status.h" #include "share/restore/ob_restore_type.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -87,7 +87,7 @@ struct ObInitiatorRestoreJobPersistKey final : public ObIInnerTableKey struct ObRestoreProgressPersistInfo final : public ObIInnerTableRow { ObRestoreJobPersistKey key_; - palf::SCN restore_scn_; + SCN restore_scn_; int64_t ls_count_; // to restore log stream replica number. int64_t finish_ls_count_; int64_t tablet_count_; @@ -96,7 +96,7 @@ struct ObRestoreProgressPersistInfo final : public ObIInnerTableRow int64_t finish_bytes_; ObRestoreProgressPersistInfo() { - restore_scn_ = palf::SCN::min_scn(); + restore_scn_ = share::SCN::min_scn(); ls_count_ = 0; finish_ls_count_ = 0; tablet_count_ = 0; @@ -167,9 +167,9 @@ struct ObLSRestoreJobPersistKey final : public ObIInnerTableKey struct ObLSHisRestorePersistInfo final : public ObIInnerTableRow { ObLSRestoreJobPersistKey key_; - palf::SCN restore_scn_; - palf::SCN start_replay_scn_; - palf::SCN last_replay_scn_; + SCN restore_scn_; + SCN start_replay_scn_; + SCN last_replay_scn_; int64_t tablet_count_; int64_t finish_tablet_count_; int64_t total_bytes_; @@ -179,9 +179,9 @@ struct ObLSHisRestorePersistInfo final : public ObIInnerTableRow common::ObSqlString comment_; ObLSHisRestorePersistInfo() { - restore_scn_ = palf::SCN::min_scn(); - start_replay_scn_ = palf::SCN::min_scn(); - last_replay_scn_ = palf::SCN::min_scn(); + restore_scn_ = share::SCN::min_scn(); + start_replay_scn_ = share::SCN::min_scn(); + last_replay_scn_ = share::SCN::min_scn(); tablet_count_ = 0; finish_tablet_count_ = 0; total_bytes_ = 0; @@ -221,9 +221,9 @@ struct ObLSRestoreProgressPersistInfo final : public ObIInnerTableRow { ObLSRestoreJobPersistKey key_; ObLSRestoreStatus status_; - palf::SCN restore_scn_; - palf::SCN start_replay_scn_; - palf::SCN last_replay_scn_; + SCN restore_scn_; + SCN start_replay_scn_; + SCN last_replay_scn_; int64_t tablet_count_; int64_t finish_tablet_count_; int64_t total_bytes_; @@ -233,9 +233,9 @@ struct ObLSRestoreProgressPersistInfo final : public ObIInnerTableRow int result_; ObLSRestoreProgressPersistInfo() { - restore_scn_ = palf::SCN::min_scn(); - start_replay_scn_ = palf::SCN::min_scn(); - last_replay_scn_ = palf::SCN::min_scn(); + restore_scn_ = share::SCN::min_scn(); + start_replay_scn_ = share::SCN::min_scn(); + last_replay_scn_ = share::SCN::min_scn(); tablet_count_ = 0; finish_tablet_count_ = 0; total_bytes_ = 0; @@ -291,7 +291,7 @@ struct ObHisRestoreJobPersistInfo final : public ObIInnerTableRow uint64_t backup_tenant_id_; ClusterName backup_cluster_name_; LongString backup_dest_; - palf::SCN restore_scn_; + SCN restore_scn_; LongString restore_option_; LongString table_list_; // white table list LongString remap_table_list_; // renamed table list @@ -324,7 +324,7 @@ struct ObHisRestoreJobPersistInfo final : public ObIInnerTableRow restore_tenant_id_ = OB_INVALID_TENANT_ID; backup_tenant_id_ = OB_INVALID_TENANT_ID; - restore_scn_ = palf::SCN::min_scn(); + restore_scn_ = share::SCN::min_scn(); backup_cluster_version_ = 0; ls_count_ = 0; @@ -452,7 +452,7 @@ public: // Update log restore progress will be updated to log stream restore progress table. int update_log_restore_progress( common::ObISQLClient &proxy, const ObLSRestoreJobPersistKey &ls_key, - const palf::SCN &last_replay_scn) const; + const SCN &last_replay_scn) const; int update_ls_restore_status( common::ObISQLClient &proxy, const ObLSRestoreJobPersistKey &ls_key, diff --git a/src/share/scheduler/ob_dag_scheduler.cpp b/src/share/scheduler/ob_dag_scheduler.cpp index 7ae3f88c9..502f79ba5 100644 --- a/src/share/scheduler/ob_dag_scheduler.cpp +++ b/src/share/scheduler/ob_dag_scheduler.cpp @@ -2029,6 +2029,27 @@ int ObTenantDagScheduler::get_all_compaction_dag_info( return ret; } +int ObTenantDagScheduler::check_ls_compaction_dag_exist(const ObLSID &ls_id, bool &exist) +{ + int ret = OB_SUCCESS; + exist = false; + compaction::ObTabletMergeDag *dag = nullptr; + ObThreadCondGuard guard(scheduler_sync_); + for (int64_t i = 0; i < ObIDag::MergeDagPrioCnt; ++i) { + ObIDag *head = dag_list_[READY_DAG_LIST].get_head(ObIDag::MergeDagPrio[i]); + ObIDag *cur = head->get_next(); + while (head != cur) { + dag = static_cast(cur); + if (ls_id == dag->get_ctx().param_.ls_id_) { + exist = true; + break; + } + cur = cur->get_next(); + } + } + return ret; +} + // get max estimated_finish_time to update server_progress int ObTenantDagScheduler::get_max_major_finish_time(const int64_t version, int64_t &estimated_finish_time) { diff --git a/src/share/scheduler/ob_dag_scheduler.h b/src/share/scheduler/ob_dag_scheduler.h index 9e174db01..3d24def46 100644 --- a/src/share/scheduler/ob_dag_scheduler.h +++ b/src/share/scheduler/ob_dag_scheduler.h @@ -790,6 +790,7 @@ public: ObIArray &progress_array); int get_max_major_finish_time(const int64_t version, int64_t &estimated_finish_time); int diagnose_dag(const ObIDag *dag, compaction::ObDiagnoseTabletCompProgress &input_progress); + int check_ls_compaction_dag_exist(const ObLSID &ls_id, bool &exist); int check_dag_net_exist( const ObDagId &dag_id, bool &exist); private: diff --git a/src/share/schema/ob_column_schema.cpp b/src/share/schema/ob_column_schema.cpp index 62fe3b805..af4bd8ef5 100644 --- a/src/share/schema/ob_column_schema.cpp +++ b/src/share/schema/ob_column_schema.cpp @@ -169,6 +169,12 @@ ObColumnSchemaV2 &ObColumnSchemaV2::operator =(const ObColumnSchemaV2 &src_schem return *this; } +int ObColumnSchemaV2::assign(const ObColumnSchemaV2 &other) +{ + *this = other; + return error_ret_; +} + bool ObColumnSchemaV2::operator==(const ObColumnSchemaV2 &r) const { return (tenant_id_ == r.tenant_id_ && table_id_ == r.table_id_ && column_id_ == r.column_id_ diff --git a/src/share/schema/ob_column_schema.h b/src/share/schema/ob_column_schema.h index 641ce9e53..d62d0d456 100644 --- a/src/share/schema/ob_column_schema.h +++ b/src/share/schema/ob_column_schema.h @@ -61,6 +61,8 @@ public: bool operator==(const ObColumnSchemaV2 &r) const; bool operator!=(const ObColumnSchemaV2 &r) const; + int assign(const ObColumnSchemaV2 &other); + //set methods inline void set_tenant_id(const uint64_t id) { tenant_id_ = id; } inline void set_table_id(const uint64_t id) { table_id_ = id; } diff --git a/src/share/schema/ob_schema_service_sql_impl.cpp b/src/share/schema/ob_schema_service_sql_impl.cpp index 216c6a2b0..151411bd1 100644 --- a/src/share/schema/ob_schema_service_sql_impl.cpp +++ b/src/share/schema/ob_schema_service_sql_impl.cpp @@ -1271,7 +1271,7 @@ int ObSchemaServiceSQLImpl::get_sys_variable_schema( } if (OB_SUCC(ret)) { - // mock missed system variable schema by hardcoded schema + // To avoid -5044 error, mock missed system variable schema with default value by hardcoded schema. for (int64_t i = 0; OB_SUCC(ret) && i < ObSysVariables::get_amount(); i++) { ObSysVarClassType sys_var_id = ObSysVariables::get_sys_var_id(i); const ObSysVarSchema *sys_var = NULL; diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index 32086b245..c1bf244c7 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -2252,6 +2252,17 @@ bool ObSysVarSchema::is_equal_except_value(const ObSysVarSchema &other) const } return bret; } + +bool ObSysVarSchema::is_equal_for_add(const ObSysVarSchema &other) const +{ + bool bret = false; + if (is_equal_except_value(other) + && 0 == value_.compare(other.value_) + && zone_ == other.zone_) { + bret = true; + } + return bret; +} /*------------------------------------------------------------------------------------------------- * ------------------------------ObDatabaseSchema------------------------------------------- ----------------------------------------------------------------------------------------------------*/ diff --git a/src/share/schema/ob_schema_struct.h b/src/share/schema/ob_schema_struct.h index 23f6a0506..eacf2c46f 100644 --- a/src/share/schema/ob_schema_struct.h +++ b/src/share/schema/ob_schema_struct.h @@ -1271,6 +1271,7 @@ public: void reset(); int64_t get_convert_size() const; bool is_equal_except_value(const ObSysVarSchema &other) const; + bool is_equal_for_add(const ObSysVarSchema &other) const; uint64_t get_tenant_id() const { return tenant_id_; } void set_tenant_id(uint64_t tenant_id) { tenant_id_ = tenant_id; } const common::ObString &get_name() const { return name_; } diff --git a/src/share/schema/ob_schema_utils.cpp b/src/share/schema/ob_schema_utils.cpp index a5a48643d..b738d982d 100644 --- a/src/share/schema/ob_schema_utils.cpp +++ b/src/share/schema/ob_schema_utils.cpp @@ -339,23 +339,18 @@ int ObSchemaUtils::construct_tenant_space_simple_table( ObSimpleTableSchemaV2 &table) { int ret = OB_SUCCESS; - if (is_sys_tenant(tenant_id)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid tenant id", KR(ret), K(tenant_id)); - } else { - table.set_tenant_id(tenant_id); - // for distributed virtual table in tenant space - int64_t part_num = table.get_partition_num(); - for (int64_t i = 0; OB_SUCC(ret) && i < part_num; i++) { - ObPartition *part = table.get_part_array()[i]; - if (OB_ISNULL(part)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("part is null", KR(ret), K(i)); - } else { - part->set_tenant_id(tenant_id); - } - } // end for - } + table.set_tenant_id(tenant_id); + // for distributed virtual table in tenant space + int64_t part_num = table.get_partition_num(); + for (int64_t i = 0; OB_SUCC(ret) && i < part_num; i++) { + ObPartition *part = table.get_part_array()[i]; + if (OB_ISNULL(part)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("part is null", KR(ret), K(i)); + } else { + part->set_tenant_id(tenant_id); + } + } // end for return ret; } diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index 198a89fc8..bab236b15 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -1852,8 +1852,7 @@ int ObTableSchema::alter_column(ObColumnSchemaV2 &column_schema, ObColumnCheckMo } } if (OB_FAIL(ret)) { - } else if (OB_FALSE_IT(*src_schema = column_schema)) { - } else if (OB_FAIL(src_schema->get_err_ret())) { + } else if (OB_FAIL(src_schema->assign(column_schema))) { LOG_WARN("failed to assign src schema", K(ret), K(column_schema)); } } @@ -3826,11 +3825,6 @@ int ObTableSchema::check_alter_column_type(const ObColumnSchemaV2 &src_column, || (src_meta.is_varbinary() && dst_meta.is_blob()) || (src_meta.is_text() && (dst_meta.is_text() || dst_meta.is_varchar())) || (src_meta.is_blob() && (dst_meta.is_blob() || dst_meta.is_varbinary())))) { - if (src_meta.is_integer_type() && dst_meta.is_integer_type()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("Type increment of integer type is not supported", K(ret), K(src_meta), K(dst_meta)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "Type increment of integer type is"); - } // online, do nothing } else { is_offline = true; @@ -4276,10 +4270,10 @@ int ObTableSchema::check_column_can_be_altered_offline( int ObTableSchema::check_column_can_be_altered_online( const ObColumnSchemaV2 *src_schema, - ObColumnSchemaV2 *dst_schema) + ObColumnSchemaV2 *dst_schema) const { int ret = OB_SUCCESS; - ObColumnSchemaV2 *tmp_column = NULL; + const ObColumnSchemaV2 *tmp_column = NULL; if (OB_ISNULL(src_schema) || NULL == dst_schema) { ret = OB_INVALID_ARGUMENT; LOG_WARN("The column schema is NULL", K(ret)); @@ -4294,8 +4288,23 @@ int ObTableSchema::check_column_can_be_altered_online( LOG_WARN("Only NORMAL table and INDEX table and SYSTEM table are allowed", K(ret)); } else { LOG_DEBUG("check column schema can be altered", KPC(src_schema), KPC(dst_schema)); + // Additional restriction for system table: + // 1. Can't alter column name + // 2. Can't alter column from "NULL" to "NOT NULL" + if (is_system_table(get_table_id())) { + if (0 != src_schema->get_column_name_str().compare(dst_schema->get_column_name_str())) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Alter system table's column name is"); + LOG_WARN("Alter system table's column name is not supported", KR(ret), K(src_schema), K(dst_schema)); + } else if (src_schema->is_nullable() && !dst_schema->is_nullable()) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Alter system table's column from `NULL` to `NOT NULL`"); + LOG_WARN("Alter system table's column from `NULL` to `NOT NULL` is not supported", KR(ret), K(src_schema), K(dst_schema)); + } + } + bool is_oracle_mode = false; - if (OB_FAIL(check_if_oracle_compat_mode(is_oracle_mode))) { + if (FAILEDx(check_if_oracle_compat_mode(is_oracle_mode))) { LOG_WARN("check if oracle compat mode failed", K(ret)); } else if (is_oracle_mode && ob_is_number_tc(src_schema->get_data_type()) diff --git a/src/share/schema/ob_table_schema.h b/src/share/schema/ob_table_schema.h index 30d75feff..b2fe6198f 100644 --- a/src/share/schema/ob_table_schema.h +++ b/src/share/schema/ob_table_schema.h @@ -1257,7 +1257,7 @@ public: virtual int init_column_meta_array( common::ObIArray &meta_array) const override; int check_column_can_be_altered_online(const ObColumnSchemaV2 *src_schema, - ObColumnSchemaV2 *dst_schema); + ObColumnSchemaV2 *dst_schema) const; int check_column_can_be_altered_offline(const ObColumnSchemaV2 *src_schema, ObColumnSchemaV2 *dst_schema) const; int check_alter_column_is_offline(const ObColumnSchemaV2 *src_schema, diff --git a/src/share/schema/ob_table_sql_service.cpp b/src/share/schema/ob_table_sql_service.cpp index d862a2545..368ea3d3b 100644 --- a/src/share/schema/ob_table_sql_service.cpp +++ b/src/share/schema/ob_table_sql_service.cpp @@ -860,9 +860,11 @@ int ObTableSqlService::revise_check_cst_column_info( return ret; } -int ObTableSqlService::insert_single_column(ObISQLClient &sql_client, - const ObTableSchema &new_table_schema, - const ObColumnSchemaV2 &new_column_schema) +int ObTableSqlService::insert_single_column( + ObISQLClient &sql_client, + const ObTableSchema &new_table_schema, + const ObColumnSchemaV2 &new_column_schema, + const bool record_ddl_operation) { int ret = OB_SUCCESS; if (OB_FAIL(check_ddl_allowed(new_table_schema))) { @@ -881,7 +883,7 @@ int ObTableSqlService::insert_single_column(ObISQLClient &sql_client, } } } - if (OB_SUCC(ret)) { + if (OB_SUCC(ret) && record_ddl_operation) { ObSchemaOperation opt; opt.tenant_id_ = new_table_schema.get_tenant_id(); opt.database_id_ = new_table_schema.get_database_id(); @@ -3681,10 +3683,10 @@ int ObTableSqlService::gen_column_dml( orig_default_value.assign_ptr(orig_default_value_buf, static_cast(orig_default_value_len)); cur_default_value.assign_ptr(cur_default_value_buf, static_cast(cur_default_value_len)); } - LOG_DEBUG("begin gen_column_dml", K(ret), K(compat_mode), K(orig_default_value), K(cur_default_value), K(orig_default_value_len), K(cur_default_value_len)); + LOG_TRACE("begin gen_column_dml", K(ret), K(compat_mode), K(orig_default_value), K(cur_default_value), K(orig_default_value_len), K(cur_default_value_len)); } } - LOG_DEBUG("begin gen_column_dml", K(ret), K(orig_default_value), K(cur_default_value), K(column)); + LOG_TRACE("begin gen_column_dml", K(ret), K(orig_default_value), K(cur_default_value), K(column)); if (OB_SUCC(ret)) { ObString cur_default_value_v1; if (column.get_orig_default_value().is_null()) { diff --git a/src/share/schema/ob_table_sql_service.h b/src/share/schema/ob_table_sql_service.h index b18b20eab..c41f83aa1 100644 --- a/src/share/schema/ob_table_sql_service.h +++ b/src/share/schema/ob_table_sql_service.h @@ -98,7 +98,8 @@ public: //alter table add column int insert_single_column(common::ObISQLClient &sql_client, const ObTableSchema &new_table_schema, - const ObColumnSchemaV2 &column_schema); + const ObColumnSchemaV2 &column_schema, + const bool record_ddl_operation); //alter table add constraint int insert_single_constraint(common::ObISQLClient &sql_client, const ObTableSchema &new_table_schema, diff --git a/src/logservice/palf/scn.cpp b/src/share/scn.cpp similarity index 98% rename from src/logservice/palf/scn.cpp rename to src/share/scn.cpp index e2a7e861d..5368c4c5f 100644 --- a/src/logservice/palf/scn.cpp +++ b/src/share/scn.cpp @@ -13,9 +13,10 @@ #include "scn.h" #include "share/ob_table_range.h" #include "lib/utility/ob_macro_utils.h" // OB_UNLIKELY +#include "lib/json/ob_yson.h" namespace oceanbase { -namespace palf +namespace share { void SCN::reset() @@ -279,7 +280,7 @@ int SCN::convert_for_gts(int64_t ts_ns) return ret; } -int SCN::convert_for_lsn_allocator(uint64_t scn_val) +int SCN::convert_for_logservice(uint64_t scn_val) { int ret = OB_SUCCESS; if (OB_UNLIKELY(OB_MAX_SCN_TS_NS < scn_val)) { @@ -337,7 +338,7 @@ uint64_t SCN::get_val_for_gts() const return val_; } -uint64_t SCN::get_val_for_lsn_allocator() const +uint64_t SCN::get_val_for_logservice() const { return val_; } @@ -525,5 +526,5 @@ int64_t SCN::get_serialize_size(void) const return size; } -} // end namespace palf +} // end namespace share } // end namespace oceanbase diff --git a/src/logservice/palf/scn.h b/src/share/scn.h similarity index 93% rename from src/logservice/palf/scn.h rename to src/share/scn.h index eed0469c5..eb1d03874 100644 --- a/src/logservice/palf/scn.h +++ b/src/share/scn.h @@ -10,13 +10,12 @@ * See the Mulan PubL v2 for more details. */ -#ifndef OCEABASE_LOGSERVICE_SCN_ -#define OCEABASE_LOGSERVICE_SCN_ +#ifndef OCEABASE_SHARE_SCN_ +#define OCEABASE_SHARE_SCN_ #include "lib/ob_define.h" // Serialization #include "lib/utility/ob_print_utils.h" // Print* -#include "log_define.h" // OB_INVALID_SCN_VAL namespace oceanbase { -namespace palf { +namespace share { const uint64_t OB_INVALID_SCN_VAL = UINT64_MAX; const uint64_t OB_MIN_SCN_TS_NS = 0; @@ -68,7 +67,7 @@ public: // convert id generated by lsn allocator. only used by logservice // @param[in] id: id generated by lsn allocator - int convert_for_lsn_allocator(uint64_t scn_val); + int convert_for_logservice(uint64_t scn_val); // convert scn_related column value to scn. only used when extracting inner table query result // @param[in] column_value: query result of scn_related column of inner table @@ -91,7 +90,7 @@ public: uint64_t get_val_for_gts() const; //only for log service use - uint64_t get_val_for_lsn_allocator() const; + uint64_t get_val_for_logservice() const; //only for sql use uint64_t get_val_for_sql() const; @@ -130,6 +129,6 @@ private: }; }; -} // end namespace palf +} // end namespace share } // end namespace oceanbase #endif diff --git a/src/share/system_variable/ob_system_variable.cpp b/src/share/system_variable/ob_system_variable.cpp index d3b3b5e88..68fc989d1 100644 --- a/src/share/system_variable/ob_system_variable.cpp +++ b/src/share/system_variable/ob_system_variable.cpp @@ -2083,49 +2083,11 @@ int ObSysVarOnCheckFuncs::check_and_convert_time_zone(ObExecContext &ctx, common::ObObj &out_val) { int ret = OB_SUCCESS; - UNUSED(sys_var); + UNUSED(ctx); UNUSED(set_var); - - int32_t sec_val = 0; - int ret_more = OB_SUCCESS; - bool check_timezone_valid = false; - bool is_oralce_mode = false; - ObSQLSessionInfo *session = ctx.get_my_session(); - if (OB_ISNULL(session)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("fail to get session info", K(ret), K(session)); - } else { - is_oralce_mode = is_oracle_compatible(session->get_sql_mode()); - ObString str = in_val.get_string(); - if (OB_ISNULL(str.ptr()) || OB_UNLIKELY(str.length() <= 0)) { - ret = OB_ERR_UNKNOWN_TIME_ZONE; - LOG_WARN("invalid time zone offset", K(ret), K(str)); - } else { - if (OB_FAIL(ObTimeConverter::str_to_offset(str, sec_val, ret_more, is_oralce_mode, check_timezone_valid))) { - if (ret != OB_ERR_UNKNOWN_TIME_ZONE) { - LOG_WARN("fail to convert time zone", K(sec_val), K(ret)); - } else { - ret = OB_SUCCESS; - } - } else { - int64_t pos = 0; - const int64_t buf_len = 16; - char *tmp_buf = reinterpret_cast(ctx.get_allocator().alloc(buf_len)); - if(OB_ISNULL(tmp_buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate memory", K(ret), K(tmp_buf)); - } else { - int32_t offset_min = static_cast(SEC_TO_MIN(sec_val)); - const char *fmt_str = (offset_min < 0 ? "-%02d:%02d" : "+%02d:%02d"); - if (OB_FAIL(databuff_printf(tmp_buf, buf_len, pos, fmt_str, abs(offset_min) / 60, abs(offset_min) % 60))) { - LOG_ERROR("fail to print offset_min information to tmp_buf", K(ret), K(tmp_buf), K(offset_min)); - } else { - out_val.set_varchar(tmp_buf, pos); - } - } - } - } - } + UNUSED(sys_var); + UNUSED(in_val); + UNUSED(out_val); return ret; } diff --git a/src/share/tablet/ob_tablet_table_operator.cpp b/src/share/tablet/ob_tablet_table_operator.cpp index a212da1a8..e35daf4aa 100644 --- a/src/share/tablet/ob_tablet_table_operator.cpp +++ b/src/share/tablet/ob_tablet_table_operator.cpp @@ -396,35 +396,49 @@ int ObTabletTableOperator::batch_update( const ObIArray &replicas) { int ret = OB_SUCCESS; - common::ObMySQLTransaction trans; + if (OB_UNLIKELY(!inited_) || OB_ISNULL(sql_proxy_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || replicas.count() <= 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), "replicas count", replicas.count()); } else { + common::ObMySQLTransaction trans; const uint64_t sql_tenant_id = gen_meta_tenant_id(tenant_id); if (OB_FAIL(trans.start(sql_proxy_, sql_tenant_id))) { LOG_WARN("start transaction failed", KR(ret), K(sql_tenant_id)); - } else { - int64_t start_idx = 0; - int64_t end_idx = min(MAX_BATCH_COUNT, replicas.count()); - while (OB_SUCC(ret) && (start_idx < end_idx)) { - if (OB_FAIL(inner_batch_update_by_sql_(tenant_id, replicas, start_idx, end_idx, trans))) { - LOG_WARN("fail to inner batch update", KR(ret), K(tenant_id), K(replicas), K(start_idx)); - } else { - start_idx = end_idx; - end_idx = min(start_idx + MAX_BATCH_COUNT, replicas.count()); - } + } else if (OB_FAIL(batch_update(trans, tenant_id, replicas))) { + LOG_WARN("fail to batch update", KR(ret), K(tenant_id)); + } + + if (trans.is_started()) { + int trans_ret = trans.end(OB_SUCCESS == ret); + if (OB_SUCCESS != trans_ret) { + LOG_WARN("end transaction failed", KR(trans_ret)); + ret = OB_SUCCESS == ret ? trans_ret : ret; } } } - if (trans.is_started()) { - int trans_ret = trans.end(OB_SUCCESS == ret); - if (OB_SUCCESS != trans_ret) { - LOG_WARN("end transaction failed", KR(trans_ret)); - ret = OB_SUCCESS == ret ? trans_ret : ret; + return ret; +} + +int ObTabletTableOperator::batch_update( + ObISQLClient &sql_client, + const uint64_t tenant_id, + const ObIArray &replicas) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || replicas.count() <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), "replicas count", replicas.count()); + } else { + int64_t start_idx = 0; + int64_t end_idx = min(MAX_BATCH_COUNT, replicas.count()); + while (OB_SUCC(ret) && (start_idx < end_idx)) { + if (OB_FAIL(inner_batch_update_by_sql_(tenant_id, replicas, start_idx, end_idx, sql_client))) { + LOG_WARN("fail to inner batch update", KR(ret), K(tenant_id), K(replicas), K(start_idx)); + } else { + start_idx = end_idx; + end_idx = min(start_idx + MAX_BATCH_COUNT, replicas.count()); + } } } return ret; @@ -549,40 +563,52 @@ int ObTabletTableOperator::batch_remove( const ObIArray &replicas) { int ret = OB_SUCCESS; - common::ObMySQLTransaction trans; if (OB_UNLIKELY(!inited_) || OB_ISNULL(sql_proxy_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || replicas.count() <= 0)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(tenant_id), "replicas count", replicas.count()); } else { + common::ObMySQLTransaction trans; const uint64_t sql_tenant_id = gen_meta_tenant_id(tenant_id); if (OB_FAIL(trans.start(sql_proxy_, sql_tenant_id))) { LOG_WARN("start transaction failed", KR(ret), K(sql_tenant_id)); - } else { - int64_t start_idx = 0; - int64_t end_idx = min(MAX_BATCH_COUNT, replicas.count()); - while (OB_SUCC(ret) && (start_idx < end_idx)) { - if (OB_FAIL(inner_batch_remove_by_sql_(tenant_id, replicas, start_idx, end_idx, trans))) { - LOG_WARN("fail to inner batch remove", KR(ret), K(tenant_id), K(replicas), K(start_idx)); - } else { - start_idx = end_idx; - end_idx = min(start_idx + MAX_BATCH_COUNT, replicas.count()); - } - } + } else if (OB_FAIL(batch_remove(trans, tenant_id, replicas))) { + LOG_WARN("fail to batch remove", KR(ret)); } - } - if (trans.is_started()) { - int trans_ret = trans.end(OB_SUCCESS == ret); - if (OB_SUCCESS != trans_ret) { - LOG_WARN("end transaction failed", KR(trans_ret)); - ret = OB_SUCCESS == ret ? trans_ret : ret; + + if (trans.is_started()) { + int trans_ret = trans.end(OB_SUCCESS == ret); + if (OB_SUCCESS != trans_ret) { + LOG_WARN("end transaction failed", KR(trans_ret)); + ret = OB_SUCCESS == ret ? trans_ret : ret; + } } } return ret; } +int ObTabletTableOperator::batch_remove( + ObISQLClient &sql_client, + const uint64_t tenant_id, + const ObIArray &replicas) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || replicas.count() <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), "replicas count", replicas.count()); + } else { + int64_t start_idx = 0; + int64_t end_idx = min(MAX_BATCH_COUNT, replicas.count()); + while (OB_SUCC(ret) && (start_idx < end_idx)) { + if (OB_FAIL(inner_batch_remove_by_sql_(tenant_id, replicas, start_idx, end_idx, sql_client))) { + LOG_WARN("fail to inner batch remove", KR(ret), K(tenant_id), K(replicas), K(start_idx)); + } else { + start_idx = end_idx; + end_idx = min(start_idx + MAX_BATCH_COUNT, replicas.count()); + } + } + } + return ret; +} int ObTabletTableOperator::inner_batch_remove_by_sql_( const uint64_t tenant_id, diff --git a/src/share/tablet/ob_tablet_table_operator.h b/src/share/tablet/ob_tablet_table_operator.h index 120c907b1..4e5304760 100644 --- a/src/share/tablet/ob_tablet_table_operator.h +++ b/src/share/tablet/ob_tablet_table_operator.h @@ -102,6 +102,12 @@ public: int batch_update( const uint64_t tenant_id, const ObIArray &replicas); + // batch update replicas into __all_tablet_meta_table + // differ from above batch_update(), it will use @sql_client to commit, not inner sql_proxy_. + int batch_update( + common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const ObIArray &replicas); // batch remove replicas from __all_tablet_meta_table // // @param [in] tenant_id, target tenant_id @@ -110,6 +116,12 @@ public: int batch_remove( const uint64_t tenant_id, const ObIArray &replicas); + // batch remove replicas from __all_tablet_meta_table + // differ from above batch_remove(), it will use @sql_client to commit, not inner sql_proxy_. + int batch_remove( + common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const ObIArray &replicas); // remove residual tablet in __all_tablet_meta_table for ObServerMetaTableChecker // // @param [in] tenant_id, tenant for query diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index f7a241245..ba3831b8a 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -1917,7 +1917,6 @@ int ObStaticEngineCG::generate_insert_with_das(ObLogInsert &op, ObTableInsertSpe spec.use_dist_das_ = op.is_multi_part_dml(); spec.gi_above_ = op.is_gi_above() && !spec.use_dist_das_; spec.is_returning_ = op.is_returning(); - spec.has_instead_of_trigger_ = op.has_instead_of_trigger(); } } for (int64_t i = 0; OB_SUCC(ret) && i < index_dml_infos.count(); ++i) { @@ -1928,7 +1927,11 @@ int ObStaticEngineCG::generate_insert_with_das(ObLogInsert &op, ObTableInsertSpe LOG_WARN("index dml info is null", K(ret)); } else if (OB_FAIL(dml_cg_service_.generate_insert_ctdef(op, *index_dml_info, ins_ctdef))) { LOG_WARN("generate insert ctdef failed", K(ret)); + } else if (OB_ISNULL(ins_ctdef)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ins_ctdef is null", K(ret)); } else { + ins_ctdef->has_instead_of_trigger_ = op.has_instead_of_trigger(); spec.ins_ctdefs_.at(0).at(i) = ins_ctdef; } } // for index_dml_infos end @@ -1994,7 +1997,6 @@ int ObStaticEngineCG::generate_delete_with_das(ObLogDelete &op, ObTableDeleteSpe spec.use_dist_das_ = op.is_multi_part_dml(); spec.gi_above_ = op.is_gi_above() && !spec.use_dist_das_; spec.is_returning_ = op.is_returning(); - spec.has_instead_of_trigger_ = op.has_instead_of_trigger(); if (OB_FAIL(spec.del_ctdefs_.allocate_array(phy_plan_->get_allocator(), delete_table_list.count()))) { LOG_WARN("allocate delete ctdef array failed", K(ret)); @@ -2025,7 +2027,11 @@ int ObStaticEngineCG::generate_delete_with_das(ObLogDelete &op, ObTableDeleteSpe LOG_WARN("index dml info is null", K(ret)); } else if (OB_FAIL(dml_cg_service_.generate_delete_ctdef(op, *index_dml_info, del_ctdef))) { LOG_WARN("generate delete ctdef failed", K(ret)); + } else if (OB_ISNULL(del_ctdef)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("del_ctdef is null", K(ret)); } else { + del_ctdef->has_instead_of_trigger_ = op.has_instead_of_trigger(); ctdefs.at(j) = del_ctdef; } } // for index_dml_infos end @@ -2117,7 +2123,6 @@ int ObStaticEngineCG::generate_update_with_das(ObLogUpdate &op, ObTableUpdateSpe spec.use_dist_das_ = op.is_multi_part_dml(); spec.gi_above_ = op.is_gi_above() && !spec.use_dist_das_; spec.is_returning_ = op.is_returning(); - spec.has_instead_of_trigger_ = op.has_instead_of_trigger(); if (OB_FAIL(spec.upd_ctdefs_.allocate_array(phy_plan_->get_allocator(), table_list.count()))) { LOG_WARN("allocate update ctdef array failed", K(ret), K(table_list)); @@ -2145,7 +2150,11 @@ int ObStaticEngineCG::generate_update_with_das(ObLogUpdate &op, ObTableUpdateSpe LOG_WARN("index dml info is null", K(ret)); } else if (OB_FAIL(dml_cg_service_.generate_update_ctdef(op, *index_dml_info, upd_ctdef))) { LOG_WARN("generate update ctdef failed", K(ret)); + } else if (OB_ISNULL(upd_ctdef)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("upd_ctdef is null", K(ret)); } else { + upd_ctdef->has_instead_of_trigger_ = op.has_instead_of_trigger(); ctdefs.at(j) = upd_ctdef; } } // for index_dml_infos end @@ -4969,8 +4978,12 @@ int ObStaticEngineCG::fill_aggr_infos(ObLogGroupBy &op, } //4.add aggr columns + spec.support_fast_single_row_agg_ = true; for (int64_t i = 0; OB_SUCC(ret) && i < all_aggr_exprs.count(); ++i) { ObAggrInfo &aggr_info = spec.aggr_infos_.at(i); + if (!is_simple_aggr_expr(aggr_exprs.at(i)->get_expr_type())) { + spec.support_fast_single_row_agg_ = false; + } if (OB_FAIL(fill_aggr_info(*static_cast(aggr_exprs.at(i)), *all_aggr_exprs.at(i), aggr_info, diff --git a/src/sql/code_generator/ob_static_engine_cg.h b/src/sql/code_generator/ob_static_engine_cg.h index 30fa194ca..d7bb434d7 100644 --- a/src/sql/code_generator/ob_static_engine_cg.h +++ b/src/sql/code_generator/ob_static_engine_cg.h @@ -457,6 +457,10 @@ private: int add_output_datum_check_flag(ObOpSpec &spec); int generate_calc_part_id_expr(const ObRawExpr &src, const ObDASTableLocMeta *loc_meta, ObExpr *&dst); int check_only_one_unique_key(const ObLogPlan &log_plan, const ObTableSchema* table_schema, bool& only_one_unique_key); + bool is_simple_aggr_expr(const ObItemType &expr_type) { return T_FUN_COUNT == expr_type + || T_FUN_SUM == expr_type + || T_FUN_MAX == expr_type + || T_FUN_MIN == expr_type; } private: ObPhysicalPlan *phy_plan_; ObOptimizerContext *opt_ctx_; diff --git a/src/sql/das/ob_das_context.cpp b/src/sql/das/ob_das_context.cpp index f7951251e..dc789beba 100644 --- a/src/sql/das/ob_das_context.cpp +++ b/src/sql/das/ob_das_context.cpp @@ -135,10 +135,8 @@ int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc, ObDASTabletLoc *&tablet_loc) { int ret = OB_SUCCESS; - FOREACH(tmp_node, table_loc.tablet_locs_) { - if ((*tmp_node)->tablet_id_ == tablet_id) { - tablet_loc = *tmp_node; - } + if (OB_FAIL(table_loc.get_tablet_loc_by_id(tablet_id, tablet_loc))) { + LOG_WARN("get tablet loc failed", KR(ret)); } if (OB_SUCC(ret) && tablet_loc == nullptr) { LOG_DEBUG("tablet location is not exists, begin to construct it", K(table_loc), K(tablet_id)); @@ -152,7 +150,7 @@ int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc, tablet_id, *tablet_loc))) { LOG_WARN("nonblock get tablet location failed", K(ret), KPC(table_loc.loc_meta_), K(tablet_id)); - } else if (OB_FAIL(table_loc.tablet_locs_.push_back(tablet_loc))) { + } else if (OB_FAIL(table_loc.add_tablet_loc(tablet_loc))) { LOG_WARN("store tablet location info failed", K(ret)); } else { tablet_loc->loc_meta_ = table_loc.loc_meta_; @@ -178,7 +176,9 @@ int ObDASCtx::check_same_server(const ObDASTabletLoc *tablet_loc) ObDASTabletLoc *first_tablet = NULL; FOREACH_X(table_node, table_locs_, NULL == first_tablet) { ObDASTableLoc *cur_table_loc = *table_node; - FOREACH_X(tablet_node, cur_table_loc->tablet_locs_, NULL == first_tablet) { + for (DASTabletLocListIter tablet_node = cur_table_loc->tablet_locs_begin(); + NULL == first_tablet && tablet_node != cur_table_loc->tablet_locs_end(); + ++tablet_node) { first_tablet = *tablet_node; } } @@ -198,10 +198,8 @@ int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc, { int ret = OB_SUCCESS; const ObOptTabletLoc &opt_tablet_loc = candi_tablet_loc.get_partition_location(); - FOREACH(tmp_node, table_loc.tablet_locs_) { - if ((*tmp_node)->tablet_id_ == opt_tablet_loc.get_tablet_id()) { - tablet_loc = *tmp_node; - } + if (OB_FAIL(table_loc.get_tablet_loc_by_id(opt_tablet_loc.get_tablet_id(), tablet_loc))) { + LOG_WARN("get tablet loc failed", KR(ret), K(opt_tablet_loc.get_tablet_id())); } if (OB_SUCC(ret) && tablet_loc == nullptr) { ObLSReplicaLocation replica_loc; @@ -217,7 +215,7 @@ int ObDASCtx::extended_tablet_loc(ObDASTableLoc &table_loc, tablet_loc->tablet_id_ = opt_tablet_loc.get_tablet_id(); tablet_loc->ls_id_ = opt_tablet_loc.get_ls_id(); tablet_loc->loc_meta_ = table_loc.loc_meta_; - if (OB_FAIL(table_loc.tablet_locs_.push_back(tablet_loc))) { + if (OB_FAIL(table_loc.add_tablet_loc(tablet_loc))) { LOG_WARN("store tablet loc failed", K(ret), K(tablet_loc)); } } @@ -260,7 +258,7 @@ OB_INLINE int ObDASCtx::build_related_tablet_loc(ObDASTabletLoc &tablet_loc) related_tablet_loc->loc_meta_ = related_table_loc->loc_meta_; related_tablet_loc->next_ = tablet_loc.next_; tablet_loc.next_ = related_tablet_loc; - if (OB_FAIL(related_table_loc->tablet_locs_.push_back(related_tablet_loc))) { + if (OB_FAIL(related_table_loc->add_tablet_loc(related_tablet_loc))) { LOG_WARN("add related tablet location failed", K(ret)); } } @@ -274,7 +272,8 @@ OB_INLINE int ObDASCtx::build_related_table_loc(ObDASTableLoc &table_loc) { int ret = OB_SUCCESS; if (!table_loc.loc_meta_->related_table_ids_.empty()) { - FOREACH_X(node, table_loc.tablet_locs_, OB_SUCC(ret)) { + for (DASTabletLocListIter node = table_loc.tablet_locs_begin(); + OB_SUCC(ret) && node != table_loc.tablet_locs_end(); ++node) { ObDASTabletLoc *tablet_loc = *node; if (OB_FAIL(build_related_tablet_loc(*tablet_loc))) { LOG_WARN("build related tablet loc failed", K(ret)); @@ -353,7 +352,7 @@ int ObDASCtx::add_candi_table_loc(const ObDASTableLocMeta &loc_meta, LOG_WARN("extended tablet loc failed", K(ret)); } } - LOG_TRACE("das table loc assign finish", K(candi_table_loc), K(loc_meta), K(table_loc->tablet_locs_)); + LOG_TRACE("das table loc assign finish", K(candi_table_loc), K(loc_meta), K(table_loc->get_tablet_locs())); return ret; } @@ -363,7 +362,8 @@ bool ObDASCtx::has_same_lsid(ObLSID *lsid) ObLSID first_lsid; FOREACH_X(table_node, table_locs_, bret) { ObDASTableLoc *table_loc = *table_node; - FOREACH_X(tablet_node, table_loc->tablet_locs_, bret) { + for (DASTabletLocListIter tablet_node = table_loc->tablet_locs_begin(); + bret && tablet_node != table_loc->tablet_locs_end(); ++tablet_node) { ObDASTabletLoc *tablet_loc = *tablet_node; if (!first_lsid.is_valid()) { first_lsid = tablet_loc->ls_id_; @@ -386,7 +386,7 @@ int64_t ObDASCtx::get_related_tablet_cnt() const int64_t total_cnt = 0; FOREACH(table_node, table_locs_) { ObDASTableLoc *table_loc = *table_node; - total_cnt += table_loc->tablet_locs_.size(); + total_cnt += table_loc->get_tablet_locs().size(); } return total_cnt; @@ -408,14 +408,14 @@ int ObDASCtx::rebuild_tablet_loc_reference() ObTableID related_table_id = table_loc->loc_meta_->related_table_ids_.at(i); ObDASTableLoc *related_table_loc = get_table_loc_by_id(table_loc_id, related_table_id); related_table_loc->rebuild_reference_ = 1; - if (table_loc->tablet_locs_.size() != related_table_loc->tablet_locs_.size()) { + if (table_loc->get_tablet_locs().size() != related_table_loc->get_tablet_locs().size()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet location count not matched", K(ret), KPC(table_loc), KPC(related_table_loc)); } - DASTabletLocList::iterator tablet_iter = table_loc->tablet_locs_.begin(); - DASTabletLocList::iterator related_tablet_iter = related_table_loc->tablet_locs_.begin(); - for (; OB_SUCC(ret) && tablet_iter != table_loc->tablet_locs_.end(); + DASTabletLocList::iterator tablet_iter = table_loc->tablet_locs_begin(); + DASTabletLocList::iterator related_tablet_iter = related_table_loc->tablet_locs_begin(); + for (; OB_SUCC(ret) && tablet_iter != table_loc->tablet_locs_end(); ++tablet_iter, ++related_tablet_iter) { ObDASTabletLoc *tablet_loc = *tablet_iter; ObDASTabletLoc *related_tablet_loc = *related_tablet_iter; @@ -460,8 +460,8 @@ bool ObDASCtx::is_partition_hit() { bool bret = true; if (same_server_) { - if (!table_locs_.empty() && !table_locs_.get_first()->tablet_locs_.empty()) { - if (MYADDR == table_locs_.get_first()->tablet_locs_.get_first()->server_) { + if (!table_locs_.empty() && !table_locs_.get_first()->get_tablet_locs().empty()) { + if (MYADDR == table_locs_.get_first()->get_first_tablet_loc()->server_) { // all local partitions bret = true; } else { @@ -476,7 +476,7 @@ bool ObDASCtx::is_partition_hit() // For background, please see comments for ObDASCtx::is_partition_hit(). void ObDASCtx::unmark_need_check_server() { - if (!table_locs_.empty() && !table_locs_.get_first()->tablet_locs_.empty()) { + if (!table_locs_.empty() && !table_locs_.get_first()->get_tablet_locs().empty()) { need_check_server_ = false; } } diff --git a/src/sql/das/ob_das_define.cpp b/src/sql/das/ob_das_define.cpp index 898a240b7..54568948b 100644 --- a/src/sql/das/ob_das_define.cpp +++ b/src/sql/das/ob_das_define.cpp @@ -29,6 +29,7 @@ using namespace share; using namespace share::schema; namespace sql { + //not assign array member void ObDASTableLocMeta::light_assign(const ObDASTableLocMeta &other) { @@ -164,5 +165,176 @@ int ObDASTableLoc::assign(const ObCandiTableLoc &candi_table_loc) LOG_DEBUG("das table loc assign", K(candi_table_loc), KPC_(loc_meta), K(tablet_locs_)); return ret; } + +int ObDASTableLoc::get_tablet_loc_by_id(const ObTabletID &tablet_id, + ObDASTabletLoc *&tablet_loc) +{ + int ret = OB_SUCCESS; + tablet_loc = NULL; + lookup_cnt_++; + if (tablet_locs_map_.created()) { + if (OB_FAIL(tablet_locs_map_.get(tablet_id, tablet_loc))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("look up from hash map failed", KR(ret), K(tablet_id)); + } + } + } + if (OB_SUCC(ret) && NULL != tablet_loc) { + // found in hash map + } else if (OB_HASH_NOT_EXIST == ret) { + // key not found + ret = OB_SUCCESS; + } else { + FOREACH(tmp_node, tablet_locs_) { + ObDASTabletLoc *tmp_tablet_loc = *tmp_node; + if (tmp_tablet_loc->tablet_id_ == tablet_id) { + tablet_loc = tmp_tablet_loc; + break; + } + } + } + if (OB_FAIL(ret) || tablet_locs_map_.created()) { + // do nothing + } else if (lookup_cnt_ > DAS_TABLET_LOC_LOOKUP_THRESHOLD + && tablet_locs_.size() > DAS_TABLET_LOC_SIZE_THRESHOLD + && OB_FAIL(create_tablet_locs_map())) { + LOG_WARN("create tablet locs hash map failed", KR(ret)); + } + return ret; +} + +int ObDASTableLoc::add_tablet_loc(ObDASTabletLoc *tablet_loc) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(tablet_loc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet loc is null", KR(ret), KP(tablet_loc)); + } else if (OB_FAIL(tablet_locs_.push_back(tablet_loc))) { + LOG_WARN("push into tablet locs list failed", KR(ret), KPC(tablet_loc)); + } else if (tablet_locs_map_.created()) { + if (OB_FAIL(tablet_locs_map_.set(tablet_loc->tablet_id_, tablet_loc))) { + LOG_WARN("insert into tablet locs map failed", KR(ret), KPC(tablet_loc)); + } + } + return ret; +} + +int TabletHashMap::create(int64_t bucket_num) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(0 >= bucket_num)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid bucket number", KR(ret), K(bucket_num)); + } else if (created()) { + ret = OB_INIT_TWICE; + LOG_WARN("hash map was already created", KR(ret)); + } else if (FALSE_IT(bucket_num = hash::cal_next_prime(bucket_num))) { + } else if (OB_ISNULL(buckets_ = static_cast( + allocator_.alloc(bucket_num * sizeof(TabletHashNode *))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc buckets failed", KR(ret)); + } else { + MEMSET(buckets_, 0, bucket_num * sizeof(TabletHashNode *)); + bucket_num_ = bucket_num; + is_inited_ = true; + } + return ret; +} + +int TabletHashMap::find_node(const ObTabletID key, + TabletHashNode *head, + TabletHashNode *&node) const +{ + int ret = OB_SUCCESS; + node = NULL; + if (!created()) { + ret = OB_NOT_INIT; + LOG_WARN("hash map was not created", KR(ret)); + } else if (NULL == head) { + // do nothing + } else { + TabletHashNode *cur = head; + while (NULL != cur && NULL == node) { + if (cur->key_ == key) { + node = cur; + } else { + cur = cur->next_; + } + } + } + return ret; +} + +int TabletHashMap::set(const ObTabletID key, ObDASTabletLoc *value) +{ + int ret = OB_SUCCESS; + if (!created()) { + ret = OB_NOT_INIT; + LOG_WARN("hash map was not created", KR(ret)); + } else { + TabletHashNode *&bucket = buckets_[key.hash() % bucket_num_]; + TabletHashNode *dst_node = NULL; + if (OB_FAIL(find_node(key, bucket, dst_node))) { + LOG_WARN("find node failed", KR(ret)); + } else if (NULL != dst_node) { + ret = OB_HASH_EXIST; + LOG_WARN("key already exists", KR(ret), K(key), KP(value)); + } else { + TabletHashNode *new_node = static_cast( + allocator_.alloc(sizeof(TabletHashNode))); + if (OB_ISNULL(new_node)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc new node failed", KR(ret)); + } else { + new_node->key_ = key; + new_node->value_ = value; + new_node->next_ = bucket; + bucket = new_node; + } + } + } + return ret; +} + +int TabletHashMap::get(const ObTabletID key, ObDASTabletLoc *&value) +{ + int ret = OB_SUCCESS; + if (!created()) { + ret = OB_NOT_INIT; + LOG_WARN("hash map was not created", KR(ret)); + } else { + TabletHashNode *&bucket = buckets_[key.hash() % bucket_num_]; + TabletHashNode *dst_node = NULL; + if (OB_FAIL(find_node(key, bucket, dst_node))) { + LOG_WARN("find node failed", KR(ret)); + } else if (NULL == dst_node) { + ret = OB_HASH_NOT_EXIST; + LOG_WARN("key dost not exist", KR(ret), K(key)); + } else { + value = dst_node->value_; + } + } + return ret; +} + +int ObDASTableLoc::create_tablet_locs_map() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(tablet_locs_map_.created())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet locs map was already created", KR(ret), K(tablet_locs_map_.created())); + } else if (OB_FAIL(tablet_locs_map_.create(DAS_TABLET_LOC_MAP_BUCKET_SIZE))) { + LOG_WARN("create tablet locs map failed", KR(ret)); + } else { + for (DASTabletLocListIter tablet_node = tablet_locs_begin(); + OB_SUCC(ret) && tablet_node != tablet_locs_end(); ++tablet_node) { + ObDASTabletLoc *tablet_loc = *tablet_node; + if (OB_FAIL(tablet_locs_map_.set(tablet_loc->tablet_id_, tablet_loc))) { + LOG_WARN("insert into tablet locs map failed", KR(ret), KPC(tablet_loc)); + } + } + } + return ret; +} } // namespace sql } // namespace oceanbase diff --git a/src/sql/das/ob_das_define.h b/src/sql/das/ob_das_define.h index a49075a11..d4ec844d6 100644 --- a/src/sql/das/ob_das_define.h +++ b/src/sql/das/ob_das_define.h @@ -27,6 +27,14 @@ (::oceanbase::sql::DAS_OP_TABLE_BATCH_SCAN != (_task_op)->get_type() ? \ nullptr : static_cast<::oceanbase::sql::ObDASGroupScanOp*>(_task_op)) +#define IS_DAS_DML_OP(_task_op) \ + ({ \ + DAS_OP_TABLE_INSERT == (_task_op).get_type() || \ + DAS_OP_TABLE_UPDATE == (_task_op).get_type() || \ + DAS_OP_TABLE_LOCK == (_task_op).get_type() || \ + DAS_OP_TABLE_DELETE == (_task_op).get_type(); \ + }) + namespace oceanbase { namespace sql @@ -172,10 +180,42 @@ private: int assign(const ObDASTabletLoc &other); }; +static const int64_t DAS_TABLET_LOC_LOOKUP_THRESHOLD = 1000; +static const int64_t DAS_TABLET_LOC_SIZE_THRESHOLD = 10; +static const int64_t DAS_TABLET_LOC_MAP_BUCKET_SIZE = 5000; + typedef common::ObList DASTabletLocList; +typedef common::ObList::iterator DASTabletLocListIter; typedef common::ObIArray DASTabletLocIArray; typedef common::ObSEArray DASTabletLocSEArray; typedef common::ObArray DASTabletLocArray; + +class TabletHashMap +{ + struct TabletHashNode + { + ObTabletID key_; + ObDASTabletLoc *value_; + struct TabletHashNode *next_; + }; +public: + TabletHashMap(common::ObIAllocator &allocator) + : allocator_(allocator), bucket_num_(0), buckets_(NULL), is_inited_(false) + {} + virtual ~TabletHashMap() {} // does not free memory + int create(int64_t bucket_num); + bool created() const { return is_inited_; } + int set(const ObTabletID key, ObDASTabletLoc *value); + int get(const ObTabletID key, ObDASTabletLoc *&value); +private: + int find_node(const ObTabletID key, TabletHashNode *head, TabletHashNode *&node) const; + + common::ObIAllocator &allocator_; + int64_t bucket_num_; + TabletHashNode **buckets_; + bool is_inited_; +}; + /** * store the location information of which tables are accessed in this plan * generate this table location when this plan is chosen in the plan cache or generated this plan by CG @@ -188,8 +228,10 @@ public: ObDASTableLoc(common::ObIAllocator &allocator) : allocator_(allocator), loc_meta_(nullptr), + flags_(0), tablet_locs_(allocator), - flags_(0) + tablet_locs_map_(allocator), + lookup_cnt_(0) { } ~ObDASTableLoc() = default; @@ -197,6 +239,12 @@ public: int64_t get_table_location_key() const { return loc_meta_->table_loc_id_; } int64_t get_ref_table_id() const { return loc_meta_->ref_table_id_; } bool empty() const { return tablet_locs_.size() == 0; } + const DASTabletLocList &get_tablet_locs() const { return tablet_locs_; } + ObDASTabletLoc *get_first_tablet_loc() { return tablet_locs_.get_first(); } + DASTabletLocListIter tablet_locs_begin() { return tablet_locs_.begin(); } + DASTabletLocListIter tablet_locs_end() { return tablet_locs_.end(); } + int get_tablet_loc_by_id(const ObTabletID &tablet_id, ObDASTabletLoc *&tablet_loc); + int add_tablet_loc(ObDASTabletLoc *table_loc); TO_STRING_KV(KPC_(loc_meta), K_(tablet_locs), @@ -213,6 +261,26 @@ public: */ common::ObIAllocator &allocator_; const ObDASTableLocMeta *loc_meta_; + union { + /** + * used to mark some status related to table access, + * and reserve some expansion bits for subsequent needs + */ + uint64_t flags_; + struct { + uint64_t is_writing_ : 1; //mark this table is writing + uint64_t is_reading_ : 1; //mark this table is reading + uint64_t rebuild_reference_ : 1; //mark whether rebuild the related reference + uint64_t need_refresh_ : 1; + uint64_t reserved_ : 60; + }; + }; + +private: + DISALLOW_COPY_AND_ASSIGN(ObDASTableLoc); + int assign(const ObDASTableLoc &other); + int create_tablet_locs_map(); + /** * The reason for using ObList to store ObTabletLoc objects is that * during the execution process, @@ -228,23 +296,8 @@ public: * ObDList has more restrictions and is not suitable as a container for ObTabletLoc **/ DASTabletLocList tablet_locs_; - union { - /** - * used to mark some status related to table access, - * and reserve some expansion bits for subsequent needs - */ - uint64_t flags_; - struct { - uint64_t is_writing_ : 1; //mark this table is writing - uint64_t is_reading_ : 1; //mark this table is reading - uint64_t rebuild_reference_ : 1; //mark whether rebuild the related reference - uint64_t need_refresh_ : 1; - uint64_t reserved_ : 60; - }; - }; -private: - DISALLOW_COPY_AND_ASSIGN(ObDASTableLoc); - int assign(const ObDASTableLoc &other); + TabletHashMap tablet_locs_map_; + int64_t lookup_cnt_; }; typedef common::ObList DASTableLocList; typedef common::ObFixedArray UIntFixedArray; diff --git a/src/sql/das/ob_das_ref.cpp b/src/sql/das/ob_das_ref.cpp index 8f719d75b..a362baed1 100644 --- a/src/sql/das/ob_das_ref.cpp +++ b/src/sql/das/ob_das_ref.cpp @@ -24,6 +24,19 @@ namespace oceanbase using namespace common; namespace sql { +bool DasRefKey::operator==(const DasRefKey &other) const +{ + return (tablet_loc_ == other.tablet_loc_ && op_type_ == other.op_type_); +} + +uint64_t DasRefKey::hash() const +{ + uint64_t hash = 0; + hash = murmurhash(&tablet_loc_, sizeof(tablet_loc_), hash); + hash = murmurhash(&op_type_, sizeof(op_type_), hash); + return hash; +} + ObDASRef::ObDASRef(ObEvalCtx &eval_ctx, ObExecContext &exec_ctx) : das_alloc_(exec_ctx.get_allocator()), reuse_alloc_(nullptr), @@ -34,6 +47,9 @@ ObDASRef::ObDASRef(ObEvalCtx &eval_ctx, ObExecContext &exec_ctx) frozen_op_node_(nullptr), expr_frame_info_(nullptr), wild_datum_info_(eval_ctx), + lookup_cnt_(0), + task_cnt_(0), + task_map_(), flags_(0) { } @@ -45,22 +61,91 @@ DASOpResultIter ObDASRef::begin_result_iter() ObIDASTaskOp* ObDASRef::find_das_task(const ObDASTabletLoc *tablet_loc, ObDASOpType op_type) { + int ret = OB_SUCCESS; ObIDASTaskOp *das_task = nullptr; if (nullptr == frozen_op_node_) { frozen_op_node_ = batched_tasks_.get_header_node(); } - DASTaskIter task_iter(frozen_op_node_->get_next(), batched_tasks_.get_header_node()); - for (; nullptr == das_task && !task_iter.is_end(); ++task_iter) { - ObIDASTaskOp *tmp_task = *task_iter; - if (tmp_task != nullptr && - tmp_task->get_tablet_loc() == tablet_loc && - tmp_task->get_type() == op_type) { - das_task = tmp_task; + lookup_cnt_++; + if (task_map_.created()) { + DasRefKey key(tablet_loc, op_type); + if (OB_FAIL(task_map_.get_refactored(key, das_task))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("look up from hash map failed", KR(ret), KP(tablet_loc), K(op_type)); + } } } + if (OB_SUCC(ret) && NULL != das_task) { + // found in hash map + } else if (OB_HASH_NOT_EXIST == ret) { + // key not found + } else { + DASTaskIter task_iter(frozen_op_node_->get_next(), batched_tasks_.get_header_node()); + for (; nullptr == das_task && !task_iter.is_end(); ++task_iter) { + ObIDASTaskOp *tmp_task = *task_iter; + if (tmp_task != nullptr && + tmp_task->get_tablet_loc() == tablet_loc && + tmp_task->get_type() == op_type) { + das_task = tmp_task; + } + } + } + if (OB_FAIL(ret) || task_map_.created()) { + // do nothing + } else if (lookup_cnt_ > DAS_REF_TASK_LOOKUP_THRESHOLD + && task_cnt_ > DAS_REF_TASK_SIZE_THRESHOLD + && OB_FAIL(create_task_map())) { + LOG_WARN("create task hash map failed", KR(ret)); + } return das_task; } +int ObDASRef::create_task_map() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(task_map_.created())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("task map was already created", KR(ret), K(task_map_.created())); + } else if (OB_FAIL(task_map_.create(DAS_REF_MAP_BUCKET_SIZE, ObModIds::OB_HASH_BUCKET))) { + LOG_WARN("create task map failed", KR(ret)); + } else { + DASTaskIter task_iter(frozen_op_node_->get_next(), batched_tasks_.get_header_node()); + for (; OB_SUCC(ret) && !task_iter.is_end(); ++task_iter) { + ObIDASTaskOp *task = *task_iter; + if (OB_ISNULL(task)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("task is null", KR(ret), KP(task)); + } else { + DasRefKey key(task->get_tablet_loc(), task->get_type()); + if (OB_FAIL(task_map_.set_refactored(key, task))) { + LOG_WARN("insert into task map failed", KR(ret), K(key), KP(task)); + } + } + } + if (OB_FAIL(ret)) { + task_map_.destroy(); + } + } + return ret; +} + +int ObDASRef::add_batched_task(ObIDASTaskOp *das_task) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(batched_tasks_.store_obj(das_task))) { + LOG_WARN("store das task failed", KR(ret)); + } else if (task_map_.created()) { + DasRefKey key(das_task->get_tablet_loc(), das_task->get_type()); + if (OB_FAIL(task_map_.set_refactored(key, das_task))) { + LOG_WARN("insert into task map failed", KR(ret), K(key), KP(das_task)); + } + } + if (OB_SUCC(ret)) { + task_cnt_++; + } + return ret; +} + void ObDASRef::print_all_das_task() { DASTaskIter task_iter(batched_tasks_.get_header_node()->get_next(), batched_tasks_.get_header_node()); @@ -148,6 +233,11 @@ int ObDASRef::execute_all_task() void ObDASRef::set_frozen_node() { frozen_op_node_ = batched_tasks_.get_last_node(); + lookup_cnt_ = 0; + task_cnt_ = 0; + if (task_map_.created()) { + task_map_.clear(); + } } int ObDASRef::close_all_task() @@ -187,6 +277,9 @@ int ObDASRef::close_all_task() session->get_trans_result().set_incomplete(); } batched_tasks_.destroy(); + if (task_map_.created()) { + task_map_.destroy(); + } } return ret; } @@ -203,8 +296,6 @@ int ObDASRef::create_das_task(const ObDASTabletLoc *tablet_loc, LOG_WARN("get das task id failed", KR(ret)); } else if (OB_FAIL(das_factory.create_das_task_op(op_type, task_op))) { LOG_WARN("create das task op failed", K(ret), KPC(task_op)); - } else if (OB_FAIL(add_batched_task(task_op))) { - LOG_WARN("add batched task failed", K(ret), KPC(task_op)); } else { task_op->set_trans_desc(session->get_tx_desc()); task_op->set_snapshot(&get_exec_ctx().get_das_ctx().get_snapshot()); @@ -218,6 +309,9 @@ int ObDASRef::create_das_task(const ObDASTabletLoc *tablet_loc, LOG_WARN("init task info failed", K(ret)); } } + if (OB_SUCC(ret) && OB_FAIL(add_batched_task(task_op))) { + LOG_WARN("add batched task failed", KR(ret), KPC(task_op)); + } return ret; } @@ -225,6 +319,11 @@ void ObDASRef::reset() { das_factory_.cleanup(); batched_tasks_.destroy(); + lookup_cnt_ = 0; + task_cnt_ = 0; + if (task_map_.created()) { + task_map_.destroy(); + } flags_ = false; frozen_op_node_ = nullptr; expr_frame_info_ = nullptr; @@ -238,6 +337,11 @@ void ObDASRef::reuse() { das_factory_.cleanup(); batched_tasks_.destroy(); + lookup_cnt_ = 0; + task_cnt_ = 0; + if (task_map_.created()) { + task_map_.destroy(); + } frozen_op_node_ = nullptr; if (reuse_alloc_ != nullptr) { reuse_alloc_->reset_remain_one_page(); diff --git a/src/sql/das/ob_das_ref.h b/src/sql/das/ob_das_ref.h index 6e450a01c..04a953447 100644 --- a/src/sql/das/ob_das_ref.h +++ b/src/sql/das/ob_das_ref.h @@ -23,6 +23,32 @@ namespace sql { class ObDASScanOp; class ObDASInsertOp; +struct DasRefKey +{ +public: + DasRefKey() + : tablet_loc_(NULL), + op_type_(ObDASOpType::DAS_OP_INVALID) + {} + DasRefKey(const ObDASTabletLoc *tablet_loc, ObDASOpType op_type) + : tablet_loc_(tablet_loc), + op_type_(op_type) + {} + ~DasRefKey() {} + bool operator==(const DasRefKey &other) const; + uint64_t hash() const; + TO_STRING_KV(KP_(tablet_loc), K_(op_type)); + +public: + const ObDASTabletLoc *tablet_loc_; + ObDASOpType op_type_; +}; + +static const int64_t DAS_REF_TASK_LOOKUP_THRESHOLD = 1000; +static const int64_t DAS_REF_TASK_SIZE_THRESHOLD = 1000; +static const int64_t DAS_REF_MAP_BUCKET_SIZE = 5000; +typedef common::hash::ObHashMap ObDASRefMap; + class ObDASRef { public: @@ -37,7 +63,7 @@ public: template bool has_das_op(const ObDASTabletLoc *tablet_loc, DASOp *&das_op); ObIDASTaskOp* find_das_task(const ObDASTabletLoc *tablet_loc, ObDASOpType op_type); - int add_batched_task(ObIDASTaskOp *das_task) { return batched_tasks_.store_obj(das_task); } + int add_batched_task(ObIDASTaskOp *das_task); //创建一个DAS Task，并由das_ref持有 template int prepare_das_task(const ObDASTabletLoc *tablet_loc, DASOp *&task_op); @@ -67,6 +93,7 @@ public: void set_lookup_iter(DASOpResultIter *lookup_iter) { wild_datum_info_.lookup_iter_ = lookup_iter; } private: DISABLE_COPY_ASSIGN(ObDASRef); + int create_task_map(); private: typedef common::ObObjNode DasOpNode; //declare das allocator @@ -84,6 +111,9 @@ private: DasOpNode *frozen_op_node_; // 初始为链表的head节点，冻结一次之后为链表的最后一个节点 const ObExprFrameInfo *expr_frame_info_; DASOpResultIter::WildDatumPtrInfo wild_datum_info_; + int64_t lookup_cnt_; + int64_t task_cnt_; + ObDASRefMap task_map_; public: //all flags union { diff --git a/src/sql/das/ob_das_scan_op.h b/src/sql/das/ob_das_scan_op.h index 8814569af..3b669f638 100644 --- a/src/sql/das/ob_das_scan_op.h +++ b/src/sql/das/ob_das_scan_op.h @@ -123,7 +123,7 @@ public: bool force_refresh_lc_; bool need_check_output_datum_; int64_t frozen_version_; - palf::SCN fb_snapshot_; + share::SCN fb_snapshot_; int64_t timeout_ts_; int64_t tx_lock_timeout_; ObSQLMode sql_mode_; diff --git a/src/sql/das/ob_das_task.h b/src/sql/das/ob_das_task.h index 5a9b71029..7945be256 100644 --- a/src/sql/das/ob_das_task.h +++ b/src/sql/das/ob_das_task.h @@ -153,7 +153,7 @@ public: void set_tenant_id(uint64_t tenant_id) { tenant_id_ = tenant_id; } uint64_t get_tenant_id() const { return tenant_id_; } void set_type(ObDASOpType op_type) { op_type_ = op_type; } - ObDASOpType get_type() { return op_type_; } + ObDASOpType get_type() const { return op_type_; } void set_trans_desc(transaction::ObTxDesc *trans_desc) { trans_desc_ = trans_desc; } transaction::ObTxDesc *get_trans_desc() { return trans_desc_; } void set_snapshot(transaction::ObTxReadSnapshot *snapshot) { snapshot_ = snapshot; } diff --git a/src/sql/das/ob_das_utils.cpp b/src/sql/das/ob_das_utils.cpp index 92031750c..177efdc58 100644 --- a/src/sql/das/ob_das_utils.cpp +++ b/src/sql/das/ob_das_utils.cpp @@ -59,22 +59,6 @@ int ObDASUtils::store_warning_msg(const ObWarningBuffer &wb, obrpc::ObRpcResultC return ret; } -int ObDASUtils::get_tablet_loc_by_id(const ObTabletID &tablet_id, - ObDASTableLoc &table_loc, - ObDASTabletLoc *&tablet_loc) -{ - int ret = OB_SUCCESS; - tablet_loc = nullptr; - FOREACH(tmp_node, table_loc.tablet_locs_) { - ObDASTabletLoc *tmp_tablet_loc = *tmp_node; - if (tmp_tablet_loc->tablet_id_ == tablet_id) { - tablet_loc = tmp_tablet_loc; - break; - } - } - return ret; -} - int ObDASUtils::check_nested_sql_mutating(ObTableID ref_table_id, ObExecContext &exec_ctx) { int ret = OB_SUCCESS; diff --git a/src/sql/das/ob_data_access_service.cpp b/src/sql/das/ob_data_access_service.cpp index c9623fadf..2f0e9b277 100644 --- a/src/sql/das/ob_data_access_service.cpp +++ b/src/sql/das/ob_data_access_service.cpp @@ -179,21 +179,53 @@ int ObDataAccessService::retry_das_task(ObDASRef &das_ref, ObIDASTaskOp &task_op while (is_master_changed_error(ret) || is_partition_change_error(ret) || OB_REPLICA_NOT_READABLE == ret) { - task_op.in_part_retry_ = true; - das_ref.get_exec_ctx().get_my_session()->set_session_in_retry(true, ret); - if (OB_FAIL(clear_task_exec_env(das_ref, task_op))) { - LOG_WARN("clear task execution environment", K(ret)); - } else if (OB_FAIL(das_ref.get_exec_ctx().check_status())) { - LOG_WARN("query is timeout, terminate retry", K(ret)); - } else if (OB_FAIL(refresh_partition_location(das_ref, task_op))) { - LOG_WARN("refresh partition location failed", K(ret)); - } else if (OB_FAIL(execute_dist_das_task(das_ref, task_op))) { - LOG_WARN("execute dist das task failed", K(ret)); + if (!can_fast_fail(task_op)) { + task_op.in_part_retry_ = true; + das_ref.get_exec_ctx().get_my_session()->set_session_in_retry(true, ret); + if (OB_FAIL(clear_task_exec_env(das_ref, task_op))) { + LOG_WARN("clear task execution environment", K(ret)); + } else if (OB_FAIL(das_ref.get_exec_ctx().check_status())) { + LOG_WARN("query is timeout, terminate retry", K(ret)); + } else if (OB_FAIL(refresh_partition_location(das_ref, task_op))) { + LOG_WARN("refresh partition location failed", K(ret)); + } else if (OB_FAIL(execute_dist_das_task(das_ref, task_op))) { + LOG_WARN("execute dist das task failed", K(ret)); + } } } return ret; } + +bool ObDataAccessService::can_fast_fail(const ObIDASTaskOp &task_op) const +{ + bool bret = false; + int ret = OB_SUCCESS; // no need to pass ret outside. + const common::ObTableID &table_id = IS_DAS_DML_OP(task_op) + ? static_cast(task_op.get_ctdef())->table_id_ + : static_cast(task_op.get_ctdef())->ref_table_id_; + int64_t schema_version = IS_DAS_DML_OP(task_op) + ? static_cast(task_op.get_ctdef())->schema_version_ + : static_cast(task_op.get_ctdef())->schema_version_; + schema::ObSchemaGetterGuard schema_guard; + const schema::ObTableSchema *table_schema = nullptr; + if (OB_ISNULL(GCTX.schema_service_)) { + LOG_ERROR("invalid schema service", KR(ret)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(MTL_ID(), schema_guard))) { + LOG_WARN("get tenant schema guard fail", KR(ret), K(MTL_ID())); + } else if (OB_FAIL(schema_guard.get_table_schema(MTL_ID(), table_id, table_schema))) { + LOG_WARN("failed to get table schema", KR(ret)); + } else if (OB_ISNULL(table_schema)) { + bret = true; + LOG_WARN("table not exist, fast fail das task"); + } else if (table_schema->get_schema_version() != schema_version) { + bret = true; + LOG_WARN("schema version changed, fast fail das task", "current schema version", + table_schema->get_schema_version(), "query schema version", schema_version); + } + return bret; +} + int ObDataAccessService::end_das_task(ObDASRef &das_ref, ObIDASTaskOp &task_op) { int ret = OB_SUCCESS; diff --git a/src/sql/das/ob_data_access_service.h b/src/sql/das/ob_data_access_service.h index 10774853a..001ef1685 100644 --- a/src/sql/das/ob_data_access_service.h +++ b/src/sql/das/ob_data_access_service.h @@ -58,6 +58,7 @@ private: ObIDASTaskOp *task_op, ObDASExtraData *&extra_result); int collect_das_task_info(ObDASTaskArg &task_arg, ObDASRemoteInfo &remote_info); + bool can_fast_fail(const ObIDASTaskOp &task_op) const; private: obrpc::ObDASRpcProxy das_rpc_proxy_; common::ObAddr ctrl_addr_; diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.cpp b/src/sql/engine/aggregate/ob_aggregate_processor.cpp index 9a53c2f83..3e1559a58 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.cpp +++ b/src/sql/engine/aggregate/ob_aggregate_processor.cpp @@ -552,7 +552,8 @@ ObAggregateProcessor::ObAggregateProcessor(ObEvalCtx &eval_ctx, dir_id_(-1), tmp_store_row_(nullptr), io_event_observer_(nullptr), - removal_info_() + removal_info_(), + support_fast_single_row_agg_(false) { } @@ -5793,5 +5794,218 @@ int ObAggregateProcessor::get_json_objectagg_result(const ObAggrInfo &aggr_info, } return ret; } + +int ObAggregateProcessor::single_row_agg(GroupRow &group_row, ObEvalCtx &eval_ctx) +{ + int ret = OB_SUCCESS; + if (!support_fast_single_row_agg_) { + group_row.reuse(); + if (OB_FAIL(prepare(group_row))) { + LOG_WARN("failed to prepare group row", K(ret)); + } else if (OB_FAIL(collect_group_row(&group_row))) { + LOG_WARN("failed to collect group by row", K(ret)); + } + } else if (OB_FAIL(fast_single_row_agg(eval_ctx))) { + LOG_WARN("failed to fill result", K(ret)); + } + return ret; +} + +int ObAggregateProcessor::single_row_agg_batch(GroupRow **group_row, ObEvalCtx &eval_ctx, const int64_t batch_size, const ObBitVector *skip) +{ + int ret = OB_SUCCESS; + CK (OB_NOT_NULL(group_row) && OB_NOT_NULL(skip)); + ObEvalCtx::BatchInfoScopeGuard batch_info_guard(eval_ctx); + batch_info_guard.set_batch_size(batch_size); + if (OB_FAIL(ret)) { + } else if (!support_fast_single_row_agg_) { + for (int64_t i = 0; OB_SUCC(ret) && i < batch_size; ++i) { + if (skip->at(i)) { + continue; + } + if (OB_ISNULL(group_row[i])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("group row is not init", K(ret), K(i)); + } else { + group_row[i]->reuse(); + batch_info_guard.set_batch_idx(i); + if (OB_FAIL(prepare(*group_row[i]))) { + LOG_WARN("failed to prepare group row", K(ret)); + } else if (OB_FAIL(collect_group_row(group_row[i]))) { + LOG_WARN("failed to collect group by row", K(ret)); + } + } + } + } else if (OB_FAIL(fast_single_row_agg_batch(eval_ctx, batch_size, skip))) { + LOG_WARN("failed to fill result", K(ret)); + } + return ret; +} + +int ObAggregateProcessor::fast_single_row_agg(ObEvalCtx &eval_ctx) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < aggr_infos_.count(); ++i) { + ObAggrInfo &aggr_info = aggr_infos_.at(i); + if (aggr_info.is_implicit_first_aggr()) { + continue; + } + const ObItemType aggr_fun = aggr_info.get_expr_type(); + switch (aggr_fun) { + case T_FUN_COUNT: { + bool has_null = false; + ObDatum &result = aggr_info.expr_->locate_datum_for_write(eval_ctx); + for (int64_t j = 0; !has_null && j < aggr_info.param_exprs_.count(); ++j) { + has_null = aggr_info.param_exprs_.at(j)->locate_expr_datum(eval_ctx).is_null(); + } + if (lib::is_mysql_mode()) { + result.set_int(has_null ? 0 : 1); + } else { + result.set_number(has_null ? ObNumber::get_zero() : ObNumber::get_positive_one()); + } + break; + } + case T_FUN_SUM: { + ObDatum &result = aggr_info.expr_->locate_datum_for_write(eval_ctx); + const ObObjTypeClass tc = ob_obj_type_class(aggr_info.get_first_child_type()); + if ((ObIntTC == tc || ObUIntTC == tc) && !aggr_info.param_exprs_.at(0)->locate_expr_datum(eval_ctx).is_null()) { + ObNumStackAllocator<2> tmp_alloc; + ObNumber result_nmb; + if (ObIntTC == tc) { + if (OB_FAIL(result_nmb.from(aggr_info.param_exprs_.at(0)->locate_expr_datum(eval_ctx).get_int(), tmp_alloc))) { + LOG_WARN("create number from int failed", K(ret), K(result_nmb), K(tc)); + } + } else { + if (OB_FAIL(result_nmb.from(aggr_info.param_exprs_.at(0)->locate_expr_datum(eval_ctx).get_uint(), tmp_alloc))) { + LOG_WARN("create number from int failed", K(ret), K(result_nmb), K(tc)); + } + } + OX (result.set_number(result_nmb)); + } else { + result.set_datum(aggr_info.param_exprs_.at(0)->locate_expr_datum(eval_ctx)); + } + break; + } + case T_FUN_MAX: + case T_FUN_MIN: { + ObDatum &result = aggr_info.expr_->locate_expr_datum(eval_ctx); + result.set_datum(aggr_info.param_exprs_.at(0)->locate_expr_datum(eval_ctx)); + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unknown aggr function type", K(ret), K(aggr_fun), K(*aggr_info.expr_)); + } + } + } + return ret; +} + +int ObAggregateProcessor::fast_single_row_agg_batch(ObEvalCtx &eval_ctx, const int64_t batch_size, const ObBitVector *skip) +{ + int ret = OB_SUCCESS; + CK (OB_NOT_NULL(skip)); + for (int64_t i = 0; OB_SUCC(ret) && i < aggr_infos_.count(); ++i) { + ObAggrInfo &aggr_info = aggr_infos_.at(i); + if (aggr_info.is_implicit_first_aggr()) { + continue; + } + const ObItemType aggr_fun = aggr_info.get_expr_type(); + switch (aggr_fun) { + case T_FUN_COUNT: { + bool has_null[batch_size]; + MEMSET(has_null, false, sizeof(bool) * batch_size); + ObDatum *result = aggr_info.expr_->locate_datums_for_update(eval_ctx, batch_size); + for (int64_t j = 0; j < aggr_info.param_exprs_.count(); ++j) { + ObDatumVector param_vec = aggr_info.param_exprs_.at(j)->locate_expr_datumvector(eval_ctx); + for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) { + if (skip->at(batch_idx) || has_null[batch_idx]) { + continue; + } + has_null[batch_idx] = param_vec.at(batch_idx)->is_null(); + } + } + if (lib::is_mysql_mode()) { + for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) { + if (skip->at(batch_idx)) { + continue; + } + result[batch_idx].set_int(has_null[batch_idx] ? 0 : 1); + } + } else { + for (int64_t batch_idx = 0; OB_SUCC(ret) && batch_idx < batch_size; ++batch_idx) { + if (skip->at(batch_idx)) { + continue; + } + result[batch_idx].set_number(has_null[batch_idx] ? ObNumber::get_zero() : ObNumber::get_positive_one()); + } + } + break; + } + case T_FUN_SUM: { + const ObObjTypeClass tc = ob_obj_type_class(aggr_info.get_first_child_type()); + ObDatum *result = aggr_info.expr_->locate_datums_for_update(eval_ctx, batch_size); + ObDatumVector param_vec = aggr_info.param_exprs_.at(0)->locate_expr_datumvector(eval_ctx); + if (ObIntTC == tc) { + for (int64_t batch_idx = 0; OB_SUCC(ret) && batch_idx < batch_size; ++batch_idx) { + if (skip->at(batch_idx)) { + continue; + } + ObNumStackAllocator<2> tmp_alloc; + ObNumber result_nmb; + if (param_vec.at(batch_idx)->is_null()) { + result[batch_idx].set_null(); + } else if (OB_FAIL(result_nmb.from(param_vec.at(batch_idx)->get_int(), tmp_alloc))) { + LOG_WARN("create number from int failed", K(ret), K(result_nmb), K(tc)); + } else { + result[batch_idx].set_number(result_nmb); + } + } + } else if (ObUIntTC == tc) { + for (int64_t batch_idx = 0; OB_SUCC(ret) && batch_idx < batch_size; ++batch_idx) { + if (skip->at(batch_idx)) { + continue; + } + ObNumStackAllocator<2> tmp_alloc; + ObNumber result_nmb; + if (param_vec.at(batch_idx)->is_null()) { + result[batch_idx].set_null(); + } else if (OB_FAIL(result_nmb.from(param_vec.at(batch_idx)->get_uint64(), tmp_alloc))) { + LOG_WARN("create number from int failed", K(ret), K(result_nmb), K(tc)); + } else { + result[batch_idx].set_number(result_nmb); + } + } + } else { + for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) { + if (skip->at(batch_idx)) { + continue; + } + result[batch_idx].set_datum(*param_vec.at(batch_idx)); + } + } + break; + } + case T_FUN_MAX: + case T_FUN_MIN: { + ObDatum *result = aggr_info.expr_->locate_batch_datums(eval_ctx); + ObDatumVector param_vec = aggr_info.param_exprs_.at(0)->locate_expr_datumvector(eval_ctx); + for (int64_t batch_idx = 0; batch_idx < batch_size; ++batch_idx) { + if (skip->at(batch_idx)) { + continue; + } + result[batch_idx].set_datum(*param_vec.at(batch_idx)); + } + break; + } + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unknown aggr function type", K(ret), K(aggr_fun), K(*aggr_info.expr_)); + } + } + } + return ret; +} + } //namespace sql } //namespace oceanbase diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.h b/src/sql/engine/aggregate/ob_aggregate_processor.h index 1e481ee9c..9adb4f927 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.h +++ b/src/sql/engine/aggregate/ob_aggregate_processor.h @@ -654,6 +654,12 @@ public: { io_event_observer_ = observer; } + inline ObIArray &get_aggr_infos() { return aggr_infos_; } + int single_row_agg(GroupRow &group_row, ObEvalCtx &eval_ctx); + int single_row_agg_batch(GroupRow **group_rows, ObEvalCtx &eval_ctx, const int64_t batch_size, const ObBitVector *skip); + int fast_single_row_agg(ObEvalCtx &eval_ctx); + int fast_single_row_agg_batch(ObEvalCtx &eval_ctx, const int64_t batch_size, const ObBitVector *skip); + inline void set_support_fast_single_row_agg(const bool flag) { support_fast_single_row_agg_ = flag; } private: template int inner_process_batch(GroupRow &group_rows, T &selector, int64_t start_idx, int64_t end_idx); @@ -911,6 +917,7 @@ private: ObChunkDatumStore::ShadowStoredRow *tmp_store_row_; ObIOEventObserver *io_event_observer_; RemovalInfo removal_info_; + bool support_fast_single_row_agg_; }; struct ObAggregateCalcFunc diff --git a/src/sql/engine/aggregate/ob_groupby_op.cpp b/src/sql/engine/aggregate/ob_groupby_op.cpp index 23d8d9b9f..0a10b68bf 100644 --- a/src/sql/engine/aggregate/ob_groupby_op.cpp +++ b/src/sql/engine/aggregate/ob_groupby_op.cpp @@ -32,7 +32,8 @@ OB_SERIALIZE_MEMBER((ObGroupBySpec, ObOpSpec), dist_aggr_group_idxes_, aggr_code_idx_, aggr_code_expr_, - by_pass_enabled_); + by_pass_enabled_, + support_fast_single_row_agg_); DEF_TO_STRING(ObGroupBySpec) { diff --git a/src/sql/engine/aggregate/ob_groupby_op.h b/src/sql/engine/aggregate/ob_groupby_op.h index 0c09735d3..b142166e9 100644 --- a/src/sql/engine/aggregate/ob_groupby_op.h +++ b/src/sql/engine/aggregate/ob_groupby_op.h @@ -34,7 +34,8 @@ public: dist_aggr_group_idxes_(alloc), aggr_code_idx_(OB_INVALID_INDEX_INT64), aggr_code_expr_(nullptr), - by_pass_enabled_(false) + by_pass_enabled_(false), + support_fast_single_row_agg_(false) { } DECLARE_VIRTUAL_TO_STRING; @@ -55,6 +56,8 @@ public: int64_t aggr_code_idx_; ObExpr *aggr_code_expr_; bool by_pass_enabled_; + // COUNT/SUM/MIN/MAX can use fast single row agg + bool support_fast_single_row_agg_; }; //modifiable diff --git a/src/sql/engine/aggregate/ob_hash_groupby_op.cpp b/src/sql/engine/aggregate/ob_hash_groupby_op.cpp index aa6e3982e..4e63f7447 100644 --- a/src/sql/engine/aggregate/ob_hash_groupby_op.cpp +++ b/src/sql/engine/aggregate/ob_hash_groupby_op.cpp @@ -17,6 +17,7 @@ #include "sql/engine/px/ob_px_util.h" #include "observer/omt/ob_tenant_config_mgr.h" #include "lib/charset/ob_charset.h" +#include "src/sql/engine/expr/ob_expr_util.h" namespace oceanbase { @@ -169,7 +170,6 @@ int ObHashGroupByOp::inner_open() int64_t est_hash_mem_size = 0; int64_t estimate_mem_size = 0; int64_t init_size = 0; - int64_t init_bkt_size = INIT_BKT_SIZE_FOR_ADAPTIVE_GBY; ObMemAttr attr(ctx_.get_my_session()->get_effective_tenant_id(), ObModIds::OB_HASH_NODE_GROUP_ROWS, ObCtxIds::WORK_AREA); @@ -196,7 +196,7 @@ int ObHashGroupByOp::inner_open() } else if (FALSE_IT(init_size = std::max((int64_t)MIN_GROUP_HT_INIT_SIZE, init_size))) { } else if (FALSE_IT(init_size = std::min((int64_t)MAX_GROUP_HT_INIT_SIZE, init_size))) { } else if (FALSE_IT(init_size = MY_SPEC.by_pass_enabled_ ? - std::min(init_size, init_bkt_size) : init_size)) { + std::min(init_size, (int64_t)INIT_BKT_SIZE_FOR_ADAPTIVE_GBY) : init_size)) { } else if (OB_FAIL(append(dup_groupby_exprs_, MY_SPEC.group_exprs_))) { LOG_WARN("failed to append groupby exprs", K(ret)); } else if (OB_FAIL(append(all_groupby_exprs_, dup_groupby_exprs_))) { @@ -529,11 +529,6 @@ int ObHashGroupByOp::inner_get_next_row() } else if (bypass_ctrl_.by_passing()) { if (OB_FAIL(load_one_row())) { LOG_WARN("failed to load one row", K(ret)); - } else if (OB_ISNULL(by_pass_group_row_) || OB_ISNULL(by_pass_group_row_->group_row_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get group row", K(ret)); - } else if (OB_FAIL(aggr_processor_.collect_group_row(by_pass_group_row_->group_row_))) { - LOG_WARN("failed to collect group by row", K(ret)); } } else if (OB_FAIL(restore_groupby_datum())) { LOG_WARN("failed to restore_groupby_datum", K(ret)); @@ -2431,32 +2426,22 @@ int ObHashGroupByOp::init_by_pass_group_row_item() int ObHashGroupByOp::init_by_pass_group_batch_item() { int ret = OB_SUCCESS; - char *group_item_buf = nullptr; const int64_t group_id = 0; if (by_pass_batch_size_ <= 0) { if (OB_ISNULL(by_pass_group_batch_ = - static_cast (mem_context_->get_arena_allocator() - .alloc(sizeof(ObGroupRowItem *) * MY_SPEC.max_batch_size_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory", K(ret)); - } else if (OB_ISNULL(group_item_buf = - static_cast (mem_context_->get_arena_allocator() - .alloc(sizeof(ObGroupRowItem) * MY_SPEC.max_batch_size_)))) { + static_cast (mem_context_->get_arena_allocator() + .alloc(sizeof(ObAggregateProcessor::GroupRow *) * MY_SPEC.max_batch_size_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc memory", K(ret)); } else { - for (int64_t i = 0; i < MY_SPEC.max_batch_size_; ++i) { - by_pass_group_batch_[i] = new (group_item_buf) ObGroupRowItem(); - group_item_buf += sizeof(ObGroupRowItem); - } by_pass_batch_size_ = MY_SPEC.max_batch_size_; } } CK (by_pass_batch_size_ > 0); for (int64_t i = 0; OB_SUCC(ret) && i < by_pass_batch_size_; ++i) { - if (OB_FAIL(aggr_processor_.generate_group_row(by_pass_group_batch_[i]->group_row_, group_id))) { + if (OB_FAIL(aggr_processor_.generate_group_row(by_pass_group_batch_[i], group_id))) { LOG_WARN("generate group row failed", K(ret)); - } else if (OB_ISNULL(by_pass_group_batch_[i]->group_row_)) { + } else if (OB_ISNULL(by_pass_group_batch_[i])) { ret = OB_ERR_UNEXPECTED; LOG_WARN("generate wrong group row", K(ret), K(i)); } @@ -2506,11 +2491,10 @@ int ObHashGroupByOp::load_one_row() ret = OB_ERR_UNEXPECTED; LOG_WARN("by pass group row is not init", K(ret)); } else { - by_pass_group_row_->group_row_->reuse(); ++agged_row_cnt_; ++agged_group_cnt_; - if (OB_FAIL(aggr_processor_.prepare(*by_pass_group_row_->group_row_))) { - LOG_WARN("failed to prepare group row", K(ret)); + if (OB_FAIL(aggr_processor_.single_row_agg(*by_pass_group_row_->group_row_, eval_ctx_))) { + LOG_WARN("failed to do single row agg", K(ret)); } } return ret; @@ -2552,30 +2536,16 @@ int ObHashGroupByOp::by_pass_prepare_one_batch(const int64_t batch_size) || by_pass_batch_size_ <= 0) { ret = OB_ERR_UNEXPECTED; LOG_WARN("by pass group row is not init", K(ret), K(by_pass_batch_size_)); + } else if (OB_FAIL(aggr_processor_.eval_aggr_param_batch(*by_pass_child_brs_))) { + LOG_WARN("fail to eval aggr param batch", K(ret), K(*by_pass_child_brs_)); } else { - ObEvalCtx::BatchInfoScopeGuard batch_info_guard(eval_ctx_); - batch_info_guard.set_batch_size(by_pass_child_brs_->size_); - for (int64_t i = 0; OB_SUCC(ret) && i < by_pass_child_brs_->size_; ++i) { - if (by_pass_child_brs_->skip_->at(i)) { - continue; - } - if (OB_ISNULL(by_pass_group_batch_[i]->group_row_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("group row is not init", K(ret), K(i)); - } else { - by_pass_group_batch_[i]->group_row_->reuse(); - ++agged_row_cnt_; - ++agged_group_cnt_; - batch_info_guard.set_batch_idx(i); - if (OB_FAIL(aggr_processor_.prepare(*by_pass_group_batch_[i]->group_row_))) { - LOG_WARN("failed to prepare group row", K(ret)); - } else if (OB_ISNULL(by_pass_group_batch_[i]) || OB_ISNULL(by_pass_group_batch_[i]->group_row_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to get group row", K(ret)); - } else if (OB_FAIL(aggr_processor_.collect_group_row(by_pass_group_batch_[i]->group_row_))) { - LOG_WARN("failed to collect group by row", K(ret)); - } - } + if (OB_FAIL(aggr_processor_.single_row_agg_batch(by_pass_group_batch_, eval_ctx_, + by_pass_child_brs_->size_, by_pass_child_brs_->skip_))) { + LOG_WARN("failed to single row agg", K(ret)); + } else { + int64_t aggr_cnt = by_pass_child_brs_->size_ - by_pass_child_brs_->skip_->accumulate_bit_cnt(by_pass_child_brs_->size_); + agged_row_cnt_ += aggr_cnt; + agged_group_cnt_ += aggr_cnt; } } if (OB_SUCC(ret)) { @@ -2650,6 +2620,7 @@ int ObHashGroupByOp::init_by_pass_op() int ret = OB_SUCCESS; int err_sim = 0; void *store_row_buf = nullptr; + aggr_processor_.set_support_fast_single_row_agg(MY_SPEC.support_fast_single_row_agg_); bypass_ctrl_.open_by_pass_ctrl(); uint64_t cut_ratio = 0; uint64_t default_cut_ratio = ObAdaptiveByPassCtrl::INIT_CUT_RATIO; diff --git a/src/sql/engine/aggregate/ob_hash_groupby_op.h b/src/sql/engine/aggregate/ob_hash_groupby_op.h index b0d80c438..5aeded11a 100644 --- a/src/sql/engine/aggregate/ob_hash_groupby_op.h +++ b/src/sql/engine/aggregate/ob_hash_groupby_op.h @@ -586,7 +586,7 @@ private: ObSqlMemMgrProcessor distinct_sql_mem_processor_; ObAdaptiveByPassCtrl bypass_ctrl_; ObGroupRowItem *by_pass_group_row_; - ObGroupRowItem **by_pass_group_batch_; + ObAggregateProcessor::GroupRow **by_pass_group_batch_; int64_t by_pass_batch_size_; int64_t by_pass_nth_group_; ObChunkDatumStore::LastStoredRow *last_child_row_; diff --git a/src/sql/engine/basic/ob_chunk_datum_store.cpp b/src/sql/engine/basic/ob_chunk_datum_store.cpp index 91b57db36..640a493f8 100644 --- a/src/sql/engine/basic/ob_chunk_datum_store.cpp +++ b/src/sql/engine/basic/ob_chunk_datum_store.cpp @@ -75,7 +75,7 @@ int ObChunkDatumStore::StoredRow::to_expr_skip_const(const ObIArray &ex } else { for (uint32_t i = 0; i < cnt_; ++i) { const ObExpr *expr = exprs.at(i); - if (expr->is_dynamic_const_ || expr->is_static_const_) { // T_QUESTIONMARK is included in dynamic_const + if (expr->is_const_expr()) { // T_QUESTIONMARK is included in dynamic_const continue; } else { expr->locate_expr_datum(ctx) = cells()[i]; diff --git a/src/sql/engine/basic/ob_chunk_datum_store.h b/src/sql/engine/basic/ob_chunk_datum_store.h index 10786376a..88cb8ee3b 100644 --- a/src/sql/engine/basic/ob_chunk_datum_store.h +++ b/src/sql/engine/basic/ob_chunk_datum_store.h @@ -1203,7 +1203,7 @@ int ObChunkDatumStore::Iterator::get_next_row_skip_const(ObEvalCtx &ctx, // logical operator, we can not add expr in CG. for (int64_t i = 0; i < sr->cnt_; i++) { const ObExpr *expr = exprs.at(i); - if (expr->is_dynamic_const_) { // T_QUESTIONMARK is included in is_dynamic_const + if (expr->is_const_expr()) { // T_QUESTIONMARK is included in is_dynamic_const continue; } else { const ObDatum &src = sr->cells()[i]; @@ -1270,7 +1270,7 @@ void ObChunkDatumStore::Iterator::attach_rows( if (NULL != srows) { for (int64_t col_idx = 0; col_idx < exprs.count(); col_idx++) { ObExpr *e = exprs.at(col_idx); - if (skip_const && e->is_dynamic_const_) { + if (skip_const && e->is_const_expr()) { continue; } if (OB_LIKELY(!fill_invariable_res_buf || e->is_variable_res_buf())) { diff --git a/src/sql/engine/basic/ob_ra_datum_store.cpp b/src/sql/engine/basic/ob_ra_datum_store.cpp index 1c19dcbe7..8ee5985cc 100644 --- a/src/sql/engine/basic/ob_ra_datum_store.cpp +++ b/src/sql/engine/basic/ob_ra_datum_store.cpp @@ -131,7 +131,7 @@ int ObRADatumStore::StoredRow::to_expr(const common::ObIArray &exprs, LOG_WARN("datum count mismatch", K(ret), K(cnt_), K(exprs.count())); } else { for (uint32_t i = 0; i < cnt_; ++i) { - if (exprs.at(i)->is_static_const_) { + if (exprs.at(i)->is_const_expr()) { continue; } else { exprs.at(i)->locate_expr_datum(ctx) = cells()[i]; diff --git a/src/sql/engine/cmd/ob_variable_set_executor.cpp b/src/sql/engine/cmd/ob_variable_set_executor.cpp index 4adba2423..a82a9b5f3 100644 --- a/src/sql/engine/cmd/ob_variable_set_executor.cpp +++ b/src/sql/engine/cmd/ob_variable_set_executor.cpp @@ -273,7 +273,12 @@ int ObVariableSetExecutor::execute(ObExecContext &ctx, ObVariableSetStmt &stmt) } if (OB_SUCC(ret) && set_var.set_scope_ == ObSetVar::SET_SCOPE_GLOBAL) { - if (OB_FAIL(update_global_variables(ctx, stmt, set_var, value_obj))) { + if(set_var.var_name_ == OB_SV_TIME_ZONE) { + if(OB_FAIL(global_variable_timezone_formalize(ctx, value_obj))) { + LOG_WARN("failed to formalize global variables", K(ret)); + } + } + if (OB_SUCC(ret) && OB_FAIL(update_global_variables(ctx, stmt, set_var, value_obj))) { LOG_WARN("failed to update global variables", K(ret)); } else { } } @@ -670,6 +675,49 @@ int ObVariableSetExecutor::update_global_variables(ObExecContext &ctx, return ret; } +// formalize : '+8:00' ---> '+08:00' +int ObVariableSetExecutor::global_variable_timezone_formalize(ObExecContext &ctx, ObObj &in_val) { + int ret = OB_SUCCESS; + + int32_t sec_val = 0; + int ret_more = OB_SUCCESS; + bool check_timezone_valid = false; + bool is_oralce_mode = false; + ObSQLSessionInfo *session = ctx.get_my_session(); + if (OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get session info", K(ret), K(session)); + } else { + is_oralce_mode = is_oracle_compatible(session->get_sql_mode()); + ObString str = in_val.get_string(); + if (OB_FAIL(ObTimeConverter::str_to_offset(str, sec_val, ret_more, is_oralce_mode, check_timezone_valid))) { + if (ret != OB_ERR_UNKNOWN_TIME_ZONE) { + LOG_WARN("fail to convert time zone", K(sec_val), K(ret)); + } else { + ret = OB_SUCCESS; + } + } else { + int64_t pos = 0; + const int64_t buf_len = 16; + char *tmp_buf = reinterpret_cast(ctx.get_allocator().alloc(buf_len)); + if(OB_ISNULL(tmp_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate memory", K(ret), K(tmp_buf)); + } else { + int32_t offset_min = static_cast(SEC_TO_MIN(sec_val)); + const char *fmt_str = (offset_min < 0 ? "-%02d:%02d" : "+%02d:%02d"); + if (OB_FAIL(databuff_printf(tmp_buf, buf_len, pos, fmt_str, abs(offset_min) / 60, abs(offset_min) % 60))) { + LOG_ERROR("fail to print offset_min information to tmp_buf", K(ret), K(tmp_buf), K(offset_min)); + } else { + in_val.set_varchar(tmp_buf, pos); + } + } + } + } + + return ret; +} + int ObVariableSetExecutor::check_and_convert_sys_var(ObExecContext &ctx, const ObSetVar &set_var, ObBasicSysVar &sys_var, diff --git a/src/sql/engine/cmd/ob_variable_set_executor.h b/src/sql/engine/cmd/ob_variable_set_executor.h index ba3d61ff2..9b12fd91f 100644 --- a/src/sql/engine/cmd/ob_variable_set_executor.h +++ b/src/sql/engine/cmd/ob_variable_set_executor.h @@ -90,6 +90,7 @@ private: ObDDLStmt &stmt, const share::ObSetVar &set_var, const common::ObObj &value_obj); + int global_variable_timezone_formalize(ObExecContext &ctx, ObObj &val); private: DISALLOW_COPY_AND_ASSIGN(ObVariableSetExecutor); }; diff --git a/src/sql/engine/dml/ob_dml_ctx_define.cpp b/src/sql/engine/dml/ob_dml_ctx_define.cpp index b3025ff09..a67fd343c 100644 --- a/src/sql/engine/dml/ob_dml_ctx_define.cpp +++ b/src/sql/engine/dml/ob_dml_ctx_define.cpp @@ -203,7 +203,8 @@ OB_SERIALIZE_MEMBER(ObDMLBaseCtDef, error_logging_ctdef_, view_check_exprs_, is_primary_index_, - is_heap_table_); + is_heap_table_, + has_instead_of_trigger_); OB_SERIALIZE_MEMBER(ObMultiInsCtDef, calc_part_id_expr_, diff --git a/src/sql/engine/dml/ob_dml_ctx_define.h b/src/sql/engine/dml/ob_dml_ctx_define.h index beec9c2fb..7c148d3db 100644 --- a/src/sql/engine/dml/ob_dml_ctx_define.h +++ b/src/sql/engine/dml/ob_dml_ctx_define.h @@ -372,7 +372,8 @@ public: K_(new_row), K_(view_check_exprs), K_(is_primary_index), - K_(is_heap_table)); + K_(is_heap_table), + K_(has_instead_of_trigger)); ObDMLOpType dml_type_; ExprFixedArray check_cst_exprs_; @@ -393,6 +394,7 @@ public: ExprFixedArray view_check_exprs_; bool is_primary_index_; bool is_heap_table_; + bool has_instead_of_trigger_; protected: ObDMLBaseCtDef(common::ObIAllocator &alloc, ObDASDMLBaseCtDef &das_base_ctdef, @@ -408,7 +410,8 @@ protected: error_logging_ctdef_(alloc), view_check_exprs_(alloc), is_primary_index_(false), - is_heap_table_(false) + is_heap_table_(false), + has_instead_of_trigger_(false) { } }; diff --git a/src/sql/engine/dml/ob_dml_service.cpp b/src/sql/engine/dml/ob_dml_service.cpp index c998dc947..2dfcba6e8 100644 --- a/src/sql/engine/dml/ob_dml_service.cpp +++ b/src/sql/engine/dml/ob_dml_service.cpp @@ -385,58 +385,10 @@ int ObDMLService::process_after_stmt_trigger(const ObDMLBaseCtDef &dml_ctdef, return ret; } -int ObDMLService::process_instead_of_trigger_delete( - const ObDelCtDef &del_ctdef, - ObDelRtDef &del_rtdef, - ObTableModifyOp &dml_op) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(TriggerHandle::init_param_old_row( - dml_op.get_eval_ctx(), del_ctdef.trig_ctdef_, del_rtdef.trig_rtdef_))) { - LOG_WARN("failed to handle before trigger", K(ret)); - } else if (OB_FAIL(TriggerHandle::do_handle_before_row( - dml_op, del_ctdef.das_base_ctdef_, del_ctdef.trig_ctdef_, del_rtdef.trig_rtdef_))) { - LOG_WARN("failed to handle before trigger", K(ret)); - } - return ret; -} - -int ObDMLService::process_instead_of_trigger_update( - const ObUpdCtDef &upd_ctdef, - ObUpdRtDef &upd_rtdef, - ObTableModifyOp &dml_op) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(TriggerHandle::init_param_rows( - dml_op.get_eval_ctx(), upd_ctdef.trig_ctdef_, upd_rtdef.trig_rtdef_))) { - LOG_WARN("failed to handle before trigger", K(ret)); - } else if (OB_FAIL(TriggerHandle::do_handle_before_row( - dml_op, upd_ctdef.das_base_ctdef_, upd_ctdef.trig_ctdef_, upd_rtdef.trig_rtdef_))) { - LOG_WARN("failed to handle before trigger", K(ret)); - } - return ret; -} - -int ObDMLService::process_instead_of_trigger_insert( - const ObInsCtDef &ins_ctdef, - ObInsRtDef &ins_rtdef, - ObTableModifyOp &dml_op) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(TriggerHandle::init_param_new_row( - dml_op.get_eval_ctx(), ins_ctdef.trig_ctdef_, ins_rtdef.trig_rtdef_))) { - LOG_WARN("failed to handle before trigger", K(ret)); - } else if (OB_FAIL(TriggerHandle::do_handle_before_row( - dml_op, ins_ctdef.das_base_ctdef_, ins_ctdef.trig_ctdef_, ins_rtdef.trig_rtdef_))) { - LOG_WARN("failed to handle before trigger", K(ret)); - } - return ret; -} - int ObDMLService::init_heap_table_pk_for_ins(const ObInsCtDef &ins_ctdef, ObEvalCtx &eval_ctx) { int ret = OB_SUCCESS; - if (ins_ctdef.is_primary_index_ && ins_ctdef.is_heap_table_) { + if (ins_ctdef.is_primary_index_ && ins_ctdef.is_heap_table_ && !ins_ctdef.has_instead_of_trigger_) { ObExpr *auto_inc_expr = ins_ctdef.new_row_.at(0); if (OB_ISNULL(auto_inc_expr)) { ret = OB_ERR_UNEXPECTED; @@ -465,7 +417,7 @@ int ObDMLService::process_insert_row(const ObInsCtDef &ins_ctdef, ObEvalCtx &eval_ctx = dml_op.get_eval_ctx(); uint64_t ref_table_id = ins_ctdef.das_base_ctdef_.index_tid_; ObSQLSessionInfo *my_session = NULL; - + bool has_instead_of_trg = ins_ctdef.has_instead_of_trigger_; //first, check insert value whether matched column type if (OB_FAIL(check_column_type(ins_ctdef.new_row_, ins_rtdef.cur_row_num_, @@ -483,6 +435,10 @@ int ObDMLService::process_insert_row(const ObInsCtDef &ins_ctdef, } else if (OB_FAIL(TriggerHandle::do_handle_before_row( dml_op, ins_ctdef.das_base_ctdef_, ins_ctdef.trig_ctdef_, ins_rtdef.trig_rtdef_))) { LOG_WARN("failed to handle before trigger", K(ret)); + } + if (OB_FAIL(ret)) { + } else if (has_instead_of_trg) { + is_skipped = true; } else if (OB_FAIL(check_row_null(ins_ctdef.new_row_, dml_op.get_eval_ctx(), ins_rtdef.cur_row_num_, @@ -514,7 +470,7 @@ int ObDMLService::process_insert_row(const ObInsCtDef &ins_ctdef, } } - if (OB_FAIL(ret) && dml_op.is_error_logging_ && should_catch_err(ret)) { + if (OB_FAIL(ret) && dml_op.is_error_logging_ && should_catch_err(ret) && !has_instead_of_trg) { dml_op.err_log_rt_def_.first_err_ret_ = ret; // cover the err_ret by design ret = OB_SUCCESS; @@ -568,13 +524,14 @@ int ObDMLService::process_delete_row(const ObDelCtDef &del_ctdef, if (del_ctdef.is_primary_index_) { uint64_t ref_table_id = del_ctdef.das_base_ctdef_.index_tid_; ObSQLSessionInfo *my_session = NULL; + bool has_instead_of_trg = del_ctdef.has_instead_of_trigger_; if (OB_ISNULL(my_session = dml_op.get_exec_ctx().get_my_session())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("session is NULL", K(ret)); } else if (OB_FAIL(check_nested_sql_legality(dml_op.get_exec_ctx(), del_ctdef.das_ctdef_.index_tid_))) { LOG_WARN("failed to check stmt table", K(ret), K(ref_table_id)); } - if (OB_SUCC(ret) && del_ctdef.need_check_filter_null_) { + if (OB_SUCC(ret) && del_ctdef.need_check_filter_null_ && !has_instead_of_trg) { bool is_null = false; if (OB_FAIL(check_rowkey_is_null(del_ctdef.old_row_, del_ctdef.das_ctdef_.rowkey_cnt_, @@ -585,7 +542,7 @@ int ObDMLService::process_delete_row(const ObDelCtDef &del_ctdef, is_skipped = true; } } - if (OB_SUCC(ret) && !is_skipped) { + if (OB_SUCC(ret) && !is_skipped && !has_instead_of_trg) { bool is_distinct = false; if (OB_FAIL(check_rowkey_whether_distinct(del_ctdef.distinct_key_, del_ctdef.distinct_key_.count(), @@ -601,7 +558,7 @@ int ObDMLService::process_delete_row(const ObDelCtDef &del_ctdef, } } if (OB_SUCC(ret) && !is_skipped) { - if (OB_FAIL(ForeignKeyHandle::do_handle(dml_op, del_ctdef, del_rtdef))) { + if (!has_instead_of_trg && OB_FAIL(ForeignKeyHandle::do_handle(dml_op, del_ctdef, del_rtdef))) { LOG_WARN("do handle old row for delete op failed", K(ret), K(del_ctdef), K(del_rtdef)); } else if (OB_FAIL(TriggerHandle::init_param_old_row( dml_op.get_eval_ctx(), del_ctdef.trig_ctdef_, del_rtdef.trig_rtdef_))) { @@ -612,10 +569,12 @@ int ObDMLService::process_delete_row(const ObDelCtDef &del_ctdef, } else if (OB_SUCC(ret) && OB_FAIL(TriggerHandle::do_handle_after_row( dml_op, del_ctdef.trig_ctdef_, del_rtdef.trig_rtdef_, ObTriggerEvents::get_delete_event()))) { LOG_WARN("failed to handle before trigger", K(ret)); + } else if (has_instead_of_trg) { + is_skipped = true; } } // here only catch foreign key execption - if (OB_FAIL(ret) && dml_op.is_error_logging_ && should_catch_err(ret)) { + if (OB_FAIL(ret) && dml_op.is_error_logging_ && should_catch_err(ret) && !has_instead_of_trg) { dml_op.err_log_rt_def_.first_err_ret_ = ret; } @@ -633,6 +592,7 @@ int ObDMLService::process_update_row(const ObUpdCtDef &upd_ctdef, { int ret = OB_SUCCESS; is_skipped = false; + bool has_instead_of_trg = upd_ctdef.has_instead_of_trigger_; if (upd_ctdef.is_primary_index_) { uint64_t ref_table_id = upd_ctdef.das_base_ctdef_.index_tid_; ObSQLSessionInfo *my_session = NULL; @@ -641,7 +601,7 @@ int ObDMLService::process_update_row(const ObUpdCtDef &upd_ctdef, LOG_WARN("fail to copy heap table hidden pk", K(ret), K(upd_ctdef)); } - if (OB_SUCC(ret) && upd_ctdef.need_check_filter_null_) { + if (OB_SUCC(ret) && upd_ctdef.need_check_filter_null_ && !has_instead_of_trg) { bool is_null = false; if (OB_FAIL(check_rowkey_is_null(upd_ctdef.old_row_, upd_ctdef.dupd_ctdef_.rowkey_cnt_, @@ -652,7 +612,7 @@ int ObDMLService::process_update_row(const ObUpdCtDef &upd_ctdef, is_skipped = true; } } - if (OB_SUCC(ret) && !is_skipped) { + if (OB_SUCC(ret) && !is_skipped && !has_instead_of_trg) { bool is_distinct = false; if (OB_FAIL(check_rowkey_whether_distinct(upd_ctdef.distinct_key_, upd_ctdef.distinct_key_.count(), @@ -686,12 +646,16 @@ int ObDMLService::process_update_row(const ObUpdCtDef &upd_ctdef, } else if (OB_FAIL(TriggerHandle::do_handle_before_row( dml_op, upd_ctdef.das_base_ctdef_, upd_ctdef.trig_ctdef_, upd_rtdef.trig_rtdef_))) { LOG_WARN("failed to handle before trigger", K(ret)); + } + if (OB_FAIL(ret)) { + } else if (has_instead_of_trg) { + is_skipped = true; } else if (OB_FAIL(check_row_null(upd_ctdef.new_row_, - dml_op.get_eval_ctx(), - upd_rtdef.cur_row_num_, - upd_ctdef.assign_columns_, - upd_ctdef.dupd_ctdef_.is_ignore_, - dml_op))) { + dml_op.get_eval_ctx(), + upd_rtdef.cur_row_num_, + upd_ctdef.assign_columns_, + upd_ctdef.dupd_ctdef_.is_ignore_, + dml_op))) { LOG_WARN("check row null failed", K(ret), K(upd_ctdef), K(upd_rtdef)); } else if (OB_FAIL(check_row_whether_changed(upd_ctdef, upd_rtdef, dml_op.get_eval_ctx()))) { LOG_WARN("check row whether changed failed", K(ret), K(upd_ctdef), K(upd_rtdef)); @@ -724,7 +688,7 @@ int ObDMLService::process_update_row(const ObUpdCtDef &upd_ctdef, } } - if (OB_FAIL(ret) && dml_op.is_error_logging_ && should_catch_err(ret)) { + if (OB_FAIL(ret) && dml_op.is_error_logging_ && should_catch_err(ret) && !has_instead_of_trg) { dml_op.err_log_rt_def_.first_err_ret_ = ret; // cover the err_ret by design ret = OB_SUCCESS; diff --git a/src/sql/engine/dml/ob_dml_service.h b/src/sql/engine/dml/ob_dml_service.h index 04c8443ef..0269177aa 100644 --- a/src/sql/engine/dml/ob_dml_service.h +++ b/src/sql/engine/dml/ob_dml_service.h @@ -65,15 +65,6 @@ public: ObDMLBaseRtDef &dml_rtdef, ObDMLRtCtx &dml_rtctx, const ObDmlEventType &dml_event); - static int process_instead_of_trigger_insert(const ObInsCtDef &ins_ctdef, - ObInsRtDef &ins_rtdef, - ObTableModifyOp &dml_op); - static int process_instead_of_trigger_delete(const ObDelCtDef &del_ctdef, - ObDelRtDef &del_rtdef, - ObTableModifyOp &dml_op); - static int process_instead_of_trigger_update(const ObUpdCtDef &upd_ctdef, - ObUpdRtDef &upd_rtdef, - ObTableModifyOp &dml_op); static int process_delete_row(const ObDelCtDef &del_ctdef, ObDelRtDef &del_rtdef, bool &is_skipped, diff --git a/src/sql/engine/dml/ob_table_delete_op.cpp b/src/sql/engine/dml/ob_table_delete_op.cpp index 2c417ea4c..da1751586 100644 --- a/src/sql/engine/dml/ob_table_delete_op.cpp +++ b/src/sql/engine/dml/ob_table_delete_op.cpp @@ -125,32 +125,6 @@ int ObTableDeleteOp::inner_open() return ret; } - -int ObTableDeleteOp::do_instead_of_trigger_delete() -{ - int ret = OB_SUCCESS; - ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); - for (int64_t i = 0; OB_SUCC(ret) && i < MY_SPEC.del_ctdefs_.count(); ++i) { - const ObTableDeleteSpec::DelCtDefArray &ctdefs = MY_SPEC.del_ctdefs_.at(i); - if (0 < del_rtdefs_.count()) { - // pdml del_rtdefs_ maybe empty - DelRtDefArray &rtdefs = del_rtdefs_.at(i); - for (int64_t j = 0; OB_SUCC(ret) && j < ctdefs.count(); ++j) { - const ObDelCtDef &del_ctdef = *ctdefs.at(j); - ObDelRtDef &del_rtdef = rtdefs.at(j); - if (del_ctdef.is_primary_index_) { - if (OB_FAIL(ObDMLService::process_instead_of_trigger_delete(del_ctdef, del_rtdef, *this))) { - LOG_WARN("failed to process instead of trigger", K(ret)); - } else { - plan_ctx->add_affected_rows(1); - } - } - } - } - } - return ret; -} - int ObTableDeleteOp::inner_get_next_row() { int ret = OB_SUCCESS; @@ -167,10 +141,6 @@ int ObTableDeleteOp::inner_get_next_row() } else { iter_end_ = true; } - } else if (MY_SPEC.has_instead_of_trigger_) { - if (OB_FAIL(do_instead_of_trigger_delete())) { - LOG_WARN("failed to do instead of trigger", K(ret)); - } } else if (OB_FAIL(delete_row_to_das())) { LOG_WARN("delete row to das failed", K(ret)); } else if (is_error_logging_ && err_log_rt_def_.first_err_ret_ != OB_SUCCESS) { @@ -181,7 +151,7 @@ int ObTableDeleteOp::inner_get_next_row() } } if (OB_ITER_END == ret) { - if (!MY_SPEC.has_instead_of_trigger_ && OB_FAIL(del_rows_post_proc())) { + if (!MY_SPEC.del_ctdefs_.at(0).at(0)->has_instead_of_trigger_ && OB_FAIL(del_rows_post_proc())) { LOG_WARN("do delete rows post process failed", K(ret)); } else { //can not overwrite the original error code diff --git a/src/sql/engine/dml/ob_table_delete_op.h b/src/sql/engine/dml/ob_table_delete_op.h index bfe670933..e574b76a5 100644 --- a/src/sql/engine/dml/ob_table_delete_op.h +++ b/src/sql/engine/dml/ob_table_delete_op.h @@ -93,7 +93,6 @@ protected: int open_table_for_each(); int close_table_for_each(); int get_next_row_from_child(); - int do_instead_of_trigger_delete(); int check_delete_affected_row(); protected: DelRtDef2DArray del_rtdefs_; //see the comment of DelCtDef2DArray diff --git a/src/sql/engine/dml/ob_table_insert_op.cpp b/src/sql/engine/dml/ob_table_insert_op.cpp index 54e6e750b..8a6d28c8b 100644 --- a/src/sql/engine/dml/ob_table_insert_op.cpp +++ b/src/sql/engine/dml/ob_table_insert_op.cpp @@ -406,29 +406,6 @@ int ObTableInsertOp::get_next_row_from_child() return ret; } -int ObTableInsertOp::do_instead_of_trigger_insert() -{ - int ret = OB_SUCCESS; - ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); - for (int64_t i = 0; OB_SUCC(ret) && i < MY_SPEC.ins_ctdefs_.count(); ++i) { - const ObTableInsertSpec::InsCtDefArray &ctdefs = MY_SPEC.ins_ctdefs_.at(i); - InsRtDefArray &rtdefs = ins_rtdefs_.at(i); - // insert each table with fetched row - for (int64_t j = 0; OB_SUCC(ret) && j < ctdefs.count(); ++j) { - const ObInsCtDef &ins_ctdef = *(ctdefs.at(j)); - ObInsRtDef &ins_rtdef = rtdefs.at(j); - if (ins_ctdef.is_primary_index_) { - if (OB_FAIL(ObDMLService::process_instead_of_trigger_insert(ins_ctdef, ins_rtdef, *this))) { - LOG_WARN("failed to process instead of trigger", K(ret)); - } else { - plan_ctx->add_affected_rows(1); - } - } - } - } - return ret; -} - int ObTableInsertOp::inner_get_next_row() { int ret = OB_SUCCESS; @@ -445,10 +422,6 @@ int ObTableInsertOp::inner_get_next_row() } else { iter_end_ = true; } - } else if (MY_SPEC.has_instead_of_trigger_) { - if (OB_FAIL(do_instead_of_trigger_insert())) { - LOG_WARN("failed to do instead of trigger", K(ret)); - } } else if (OB_FAIL(insert_row_to_das())) { LOG_WARN("insert row to das failed", K(ret)); } else if (is_error_logging_ && err_log_rt_def_.first_err_ret_ != OB_SUCCESS) { @@ -462,7 +435,7 @@ int ObTableInsertOp::inner_get_next_row() } if (OB_ITER_END == ret) { - if (!MY_SPEC.has_instead_of_trigger_ && OB_FAIL(ins_rows_post_proc())) { + if (!MY_SPEC.ins_ctdefs_.at(0).at(0)->has_instead_of_trigger_ && OB_FAIL(ins_rows_post_proc())) { LOG_WARN("do insert rows post process failed", K(ret)); } else { ret = OB_ITER_END; diff --git a/src/sql/engine/dml/ob_table_insert_op.h b/src/sql/engine/dml/ob_table_insert_op.h index 51ca8b5d1..c9372828d 100644 --- a/src/sql/engine/dml/ob_table_insert_op.h +++ b/src/sql/engine/dml/ob_table_insert_op.h @@ -89,7 +89,6 @@ protected: ObDASTabletLoc *&tablet_loc); int open_table_for_each(); int get_next_row_from_child(); - int do_instead_of_trigger_insert(); int close_table_for_each(); int check_insert_affected_row(); diff --git a/src/sql/engine/dml/ob_table_lock_op.cpp b/src/sql/engine/dml/ob_table_lock_op.cpp index 802346845..b6fbe855f 100644 --- a/src/sql/engine/dml/ob_table_lock_op.cpp +++ b/src/sql/engine/dml/ob_table_lock_op.cpp @@ -325,7 +325,7 @@ int ObTableLockOp::calc_tablet_loc(const ObLockCtDef &lock_ctdef, } else { //direct lock to storage tablet_loc = (MY_INPUT.get_tablet_loc() != nullptr ? - MY_INPUT.get_tablet_loc() : MY_INPUT.get_table_loc()->tablet_locs_.get_first()); + MY_INPUT.get_tablet_loc() : MY_INPUT.get_table_loc()->get_first_tablet_loc()); } return ret; } diff --git a/src/sql/engine/dml/ob_table_modify_op.cpp b/src/sql/engine/dml/ob_table_modify_op.cpp index 9cab9c794..005f99b93 100644 --- a/src/sql/engine/dml/ob_table_modify_op.cpp +++ b/src/sql/engine/dml/ob_table_modify_op.cpp @@ -775,7 +775,7 @@ int ObTableModifyOp::calc_single_table_loc() K(table_loc_id), K(ref_table_id), K(das_ctx.get_table_loc_list())); } else { get_input()->table_loc_ = table_loc; - get_input()->tablet_loc_ = table_loc->tablet_locs_.get_first(); + get_input()->tablet_loc_ = table_loc->get_first_tablet_loc(); } } return ret; diff --git a/src/sql/engine/dml/ob_table_modify_op.h b/src/sql/engine/dml/ob_table_modify_op.h index 69e01ea8c..7986b8fda 100644 --- a/src/sql/engine/dml/ob_table_modify_op.h +++ b/src/sql/engine/dml/ob_table_modify_op.h @@ -93,7 +93,6 @@ public: void set_table_location_uncertain(bool v) { table_location_uncertain_ = v; } bool is_table_location_uncertain() const { return table_location_uncertain_; } inline bool use_dist_das() const { return use_dist_das_; } - inline bool has_instead_of_trigger() const { return has_instead_of_trigger_; } public: // Expr frame info for partial expr serialization. (serialize is not need for it self) ObExprFrameInfo *expr_frame_info_; @@ -107,7 +106,7 @@ public: uint64_t is_pdml_index_maintain_ : 1; // 表示当前dml算子是否是pdml中用于维护索引操作的算子（index maintain） uint64_t table_location_uncertain_ : 1; // 目标访问分区位置不确定，需要全表访问 uint64_t use_dist_das_ : 1; - uint64_t has_instead_of_trigger_ : 1; + uint64_t has_instead_of_trigger_ : 1; // abandoned, don't use again uint64_t is_pdml_update_split_ : 1; // 标记delete, insert op是否由update拆分而来 uint64_t reserved_ : 56; }; diff --git a/src/sql/engine/dml/ob_table_update_op.cpp b/src/sql/engine/dml/ob_table_update_op.cpp index 125c15188..272684449 100644 --- a/src/sql/engine/dml/ob_table_update_op.cpp +++ b/src/sql/engine/dml/ob_table_update_op.cpp @@ -170,32 +170,6 @@ int ObTableUpdateOp::inner_switch_iterator() return ret; } -int ObTableUpdateOp::do_instead_of_trigger_update() -{ - int ret = OB_SUCCESS; - ObPhysicalPlanCtx *plan_ctx = GET_PHY_PLAN_CTX(ctx_); - for (int64_t i = 0; OB_SUCC(ret) && i < MY_SPEC.upd_ctdefs_.count(); ++i) { - const ObTableUpdateSpec::UpdCtDefArray &ctdefs = MY_SPEC.upd_ctdefs_.at(i); - if (0 < upd_rtdefs_.count()) { - // pdml upd_rtdefs_ maybe empty - UpdRtDefArray &rtdefs = upd_rtdefs_.at(i); - for (int64_t j = 0; OB_SUCC(ret) && j < ctdefs.count(); ++j) { - const ObUpdCtDef &upd_ctdef = *ctdefs.at(j); - ObUpdRtDef &upd_rtdef = rtdefs.at(j); - if (upd_ctdef.is_primary_index_) { - if (OB_FAIL(ObDMLService::process_instead_of_trigger_update(upd_ctdef, upd_rtdef, *this))) { - LOG_WARN("failed to process instead of trigger", K(ret)); - } else { - plan_ctx->add_affected_rows(1); - plan_ctx->add_row_matched_count(1); - } - } - } - } - } - return ret; -} - int ObTableUpdateOp::inner_get_next_row() { int ret = OB_SUCCESS; @@ -212,10 +186,6 @@ int ObTableUpdateOp::inner_get_next_row() } else { iter_end_ = true; } - } else if (MY_SPEC.has_instead_of_trigger_) { - if (OB_FAIL(do_instead_of_trigger_update())) { - LOG_WARN("failed to do instead of trigger", K(ret)); - } } else if (OB_FAIL(update_row_to_das())) { LOG_WARN("update row to das failed", K(ret)); } else if (is_error_logging_ && err_log_rt_def_.first_err_ret_ != OB_SUCCESS) { @@ -228,7 +198,7 @@ int ObTableUpdateOp::inner_get_next_row() } } if (OB_ITER_END == ret) { - if (!MY_SPEC.has_instead_of_trigger_ && OB_FAIL(upd_rows_post_proc())) { + if (!MY_SPEC.upd_ctdefs_.at(0).at(0)->has_instead_of_trigger_ && OB_FAIL(upd_rows_post_proc())) { LOG_WARN("do update rows post process failed", K(ret)); } else { //can not overwrite the original error code @@ -380,7 +350,6 @@ OB_INLINE int ObTableUpdateOp::calc_tablet_loc(const ObUpdCtDef &upd_ctdef, OB_INLINE int ObTableUpdateOp::update_row_to_das() { int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < MY_SPEC.upd_ctdefs_.count(); ++i) { const ObTableUpdateSpec::UpdCtDefArray &ctdefs = MY_SPEC.upd_ctdefs_.at(i); UpdRtDefArray &rtdefs = upd_rtdefs_.at(i); diff --git a/src/sql/engine/dml/ob_table_update_op.h b/src/sql/engine/dml/ob_table_update_op.h index 319cf322e..c829757b4 100644 --- a/src/sql/engine/dml/ob_table_update_op.h +++ b/src/sql/engine/dml/ob_table_update_op.h @@ -107,7 +107,6 @@ protected: int open_table_for_each(); int close_table_for_each(); int get_next_row_from_child(); - int do_instead_of_trigger_update(); int check_update_affected_row(); protected: UpdRtDef2DArray upd_rtdefs_; //see the comment of UpdCtDef2DArray diff --git a/src/sql/engine/dml/ob_trigger_handler.cpp b/src/sql/engine/dml/ob_trigger_handler.cpp index 4a0accd2c..c6b268b7d 100644 --- a/src/sql/engine/dml/ob_trigger_handler.cpp +++ b/src/sql/engine/dml/ob_trigger_handler.cpp @@ -532,12 +532,12 @@ int TriggerHandle::do_handle_before_row( ret = OB_NOT_INIT; LOG_WARN("trigger row point params is not init", K(ret)); } else { + const ObTableModifySpec &modify_spec = static_cast(dml_op.get_spec()); if (OB_FAIL(calc_before_row(dml_op, trig_rtdef, tg_arg.get_trigger_id()))) { LOG_WARN("failed to calc before row", K(ret)); } else if ((ObTriggerEvents::is_update_event(tg_event) || ObTriggerEvents::is_insert_event(tg_event))) { - const ObTableModifySpec &modify_spec = static_cast(dml_op.get_spec()); - if (!modify_spec.has_instead_of_trigger_ && + if (!trig_ctdef.all_tm_points_.has_instead_row() && OB_FAIL(check_and_update_new_row(&dml_op, trig_ctdef.trig_col_info_, dml_op.get_eval_ctx(), @@ -547,6 +547,13 @@ int TriggerHandle::do_handle_before_row( LOG_WARN("failed to check updated new row", K(ret)); } } + if (OB_SUCC(ret) && trig_ctdef.all_tm_points_.has_instead_row()) { + GET_PHY_PLAN_CTX(dml_op.get_exec_ctx())->add_affected_rows(1); + if (ObTriggerEvents::is_update_event(tg_event)) { + GET_PHY_PLAN_CTX(dml_op.get_exec_ctx())->add_row_matched_count(1); + GET_PHY_PLAN_CTX(dml_op.get_exec_ctx())->add_row_duplicated_count(1); + } + } LOG_DEBUG("TRIGGER calc before row", K(need_fire), K(i)); } } diff --git a/src/sql/engine/expr/ob_expr.h b/src/sql/engine/expr/ob_expr.h index b51ff2b05..cf0458d35 100644 --- a/src/sql/engine/expr/ob_expr.h +++ b/src/sql/engine/expr/ob_expr.h @@ -459,6 +459,8 @@ public: return OBJ_DATUM_STRING == obj_datum_map_; }; + inline bool is_const_expr() const { return is_static_const_ || is_dynamic_const_; } + // Evaluate all parameters, assign the first sizeof...(args) parameters to %args. // // e.g.: diff --git a/src/sql/engine/ob_physical_plan.cpp b/src/sql/engine/ob_physical_plan.cpp index dc2bd3eb5..e567c3bf2 100644 --- a/src/sql/engine/ob_physical_plan.cpp +++ b/src/sql/engine/ob_physical_plan.cpp @@ -742,7 +742,8 @@ OB_SERIALIZE_MEMBER(ObPhysicalPlan, has_instead_of_trigger_, is_plain_insert_, ddl_execution_id_, - ddl_task_id_); + ddl_task_id_, + stat_.plan_id_); int ObPhysicalPlan::set_table_locations(const ObTablePartitionInfoArray &infos, ObSchemaGetterGuard &schema_guard) diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.cpp b/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.cpp index 48d24c855..23a46c83a 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.cpp @@ -170,11 +170,11 @@ int ObPxMultiPartDeleteOp::read_row(ObExecContext &ctx, if (NO_PARTITION_ID_FLAG == part_id_idx) { // 如果row中没有partition id expr对应的cell，默认partition id为0 ObDASTableLoc *table_loc = del_rtdef_.das_rtdef_.table_loc_; - if (OB_ISNULL(table_loc) || table_loc->tablet_locs_.size() != 1) { + if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) { ret = OB_ERR_UNEXPECTED; LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc)); } else { - tablet_id = table_loc->tablet_locs_.get_first()->tablet_id_; + tablet_id = table_loc->get_first_tablet_loc()->tablet_id_; } } else if (child_->get_spec().output_.count() > part_id_idx) { ObExpr *expr = child_->get_spec().output_.at(part_id_idx); diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp index 8ec7a5a16..3f1cec18c 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp @@ -139,11 +139,11 @@ int ObPxMultiPartInsertOp::read_row(ObExecContext &ctx, row = &child_->get_spec().output_; if (NO_PARTITION_ID_FLAG == part_id_idx) { ObDASTableLoc *table_loc = ins_rtdef_.das_rtdef_.table_loc_; - if (OB_ISNULL(table_loc) || table_loc->tablet_locs_.size() != 1) { + if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) { ret = OB_ERR_UNEXPECTED; LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc)); } else { - tablet_id = table_loc->tablet_locs_.get_first()->tablet_id_; + tablet_id = table_loc->get_first_tablet_loc()->tablet_id_; } } else if (child_->get_spec().output_.count() > part_id_idx) { ObExpr *expr = child_->get_spec().output_.at(part_id_idx); diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp b/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp index f48759ad6..a19a47079 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp @@ -141,11 +141,11 @@ int ObPxMultiPartUpdateOp::read_row(ObExecContext &ctx, row = &child_->get_spec().output_; if (NO_PARTITION_ID_FLAG == part_id_idx) { ObDASTableLoc *table_loc = upd_rtdef_.dupd_rtdef_.table_loc_; - if (OB_ISNULL(table_loc) || table_loc->tablet_locs_.size() != 1) { + if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) { ret = OB_ERR_UNEXPECTED; LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc)); } else { - tablet_id = table_loc->tablet_locs_.get_first()->tablet_id_; + tablet_id = table_loc->get_first_tablet_loc()->tablet_id_; } } else if (child_->get_spec().output_.count() > part_id_idx) { ObExpr *expr = child_->get_spec().output_.at(part_id_idx); diff --git a/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp b/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp index e2901cb22..220cc5855 100644 --- a/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_sstable_insert_op.cpp @@ -54,11 +54,11 @@ int ObPxMultiPartSSTableInsertOp::get_tablet_id_from_row(const ObExprPtrIArray & tablet_id.reset(); if (NO_PARTITION_ID_FLAG == part_id_idx) { ObDASTableLoc *table_loc = ins_rtdef_.das_rtdef_.table_loc_; - if (OB_ISNULL(table_loc) || table_loc->tablet_locs_.size() != 1) { + if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) { ret = OB_ERR_UNEXPECTED; LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc)); } else { - tablet_id = table_loc->tablet_locs_.get_first()->tablet_id_; + tablet_id = table_loc->get_first_tablet_loc()->tablet_id_; } } else if (part_id_idx < 0) { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/engine/px/ob_px_sub_coord.cpp b/src/sql/engine/px/ob_px_sub_coord.cpp index 9e242009b..984a6d14e 100644 --- a/src/sql/engine/px/ob_px_sub_coord.cpp +++ b/src/sql/engine/px/ob_px_sub_coord.cpp @@ -885,7 +885,8 @@ int ObPxSubCoord::rebuild_sqc_access_table_locations() ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected table loc", K(ret)); } else { - FOREACH(tmp_node, table_loc->tablet_locs_) { + for (DASTabletLocListIter tmp_node = table_loc->tablet_locs_begin(); + tmp_node != table_loc->tablet_locs_end(); ++tmp_node) { ObDASTabletLoc *tablet_loc = *tmp_node; if (tablet_loc->tablet_id_ == location_keys.at(i).tablet_id_) { if (OB_FAIL(access_locations.push_back(tablet_loc))) { diff --git a/src/sql/engine/px/ob_px_util.cpp b/src/sql/engine/px/ob_px_util.cpp index 2d15cb790..41703a743 100644 --- a/src/sql/engine/px/ob_px_util.cpp +++ b/src/sql/engine/px/ob_px_util.cpp @@ -189,7 +189,7 @@ int ObPXServerAddrUtil::alloc_by_data_distribution_inner( ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get phy table location", K(ret)); } else { - const DASTabletLocList &locations = table_loc->tablet_locs_; + const DASTabletLocList &locations = table_loc->get_tablet_locs(); if (locations.size() <= 0) { ret = OB_ERR_UNEXPECTED; LOG_WARN("the location array is empty", K(locations.size()), K(ret)); @@ -511,7 +511,8 @@ int ObPXServerAddrUtil::alloc_by_random_distribution(ObExecContext &exec_ctx, DASTabletLocArray locations; FOREACH_X(tmp_node, table_locs, OB_SUCC(ret)) { ObDASTableLoc *table_loc = *tmp_node; - FOREACH_X(tablet_node, table_loc->tablet_locs_, OB_SUCC(ret)) { + for (DASTabletLocListIter tablet_node = table_loc->tablet_locs_begin(); + OB_SUCC(ret) && tablet_node != table_loc->tablet_locs_end(); ++tablet_node) { OZ(locations.push_back(*tablet_node)); } } @@ -792,7 +793,7 @@ int ObPXServerAddrUtil::set_sqcs_accessed_location(ObExecContext &ctx, int ret = OB_SUCCESS; common::ObArray sqcs; int n_locations = 0; - const DASTabletLocList &locations = table_loc->tablet_locs_; + const DASTabletLocList &locations = table_loc->get_tablet_locs(); DASTabletLocSEArray temp_locations; if (OB_ISNULL(table_loc) || OB_ISNULL(phy_op)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index 2f6f296a3..2af2c5d4a 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -685,7 +685,8 @@ int ObTableScanOp::prepare_das_task() } } else if (OB_LIKELY(nullptr == MY_CTDEF.das_dppr_tbl_)) { ObDASTableLoc *table_loc = tsc_rtdef_.scan_rtdef_.table_loc_; - FOREACH_X(node, table_loc->tablet_locs_, OB_SUCC(ret)) { + for (DASTabletLocListIter node = table_loc->tablet_locs_begin(); + OB_SUCC(ret) && node != table_loc->tablet_locs_end(); ++node) { ObDASTabletLoc *tablet_loc = *node; if (OB_FAIL(create_one_das_task(tablet_loc))) { LOG_WARN("create one das task failed", K(ret)); @@ -802,7 +803,7 @@ int ObTableScanOp::init_table_scan_rtdef() if (OB_FAIL(init_das_scan_rtdef(scan_ctdef, scan_rtdef, loc_meta))) { LOG_WARN("init das scan rtdef failed", K(ret)); } else if (!MY_SPEC.use_dist_das_ && !MY_SPEC.gi_above_ && !scan_rtdef.table_loc_->empty()) { - MY_INPUT.tablet_loc_ = scan_rtdef.table_loc_->tablet_locs_.get_first(); + MY_INPUT.tablet_loc_ = scan_rtdef.table_loc_->get_first_tablet_loc(); } } if (OB_SUCC(ret) && MY_CTDEF.lookup_ctdef_ != nullptr) { @@ -2057,9 +2058,8 @@ int ObTableScanOp::get_access_tablet_loc(ObGranuleTaskInfo &info) iter_end_ = true; ret = OB_SUCCESS; } - } else if (OB_FAIL(ObDASUtils::get_tablet_loc_by_id(info.tablet_loc_->tablet_id_, - *tsc_rtdef_.scan_rtdef_.table_loc_, - MY_INPUT.tablet_loc_))) { + } else if (OB_FAIL(tsc_rtdef_.scan_rtdef_.table_loc_->get_tablet_loc_by_id(info.tablet_loc_->tablet_id_, + MY_INPUT.tablet_loc_))) { //need use `get_tablet_loc_by_id` to find my px work thread's tablet_loc, //because the tablet_loc in SQC maybe shared with other px work thread, //the tablet loc maybe modify in das partition retry diff --git a/src/sql/executor/ob_remote_identity_task_spliter.cpp b/src/sql/executor/ob_remote_identity_task_spliter.cpp index ab8f0ad3a..61af7773f 100644 --- a/src/sql/executor/ob_remote_identity_task_spliter.cpp +++ b/src/sql/executor/ob_remote_identity_task_spliter.cpp @@ -62,7 +62,7 @@ int ObRemoteIdentityTaskSpliter::get_next_task(ObTaskInfo *&task) // t1 union t1这种情况， t1(p0) union t2(p0)这种情况，等等， // 都是remote模式，但table_loc_list的count可能大于1 // 优化器必须保证：remote模式下，所有表的location都是一致的，并且都是单分区。 - ObDASTabletLoc *first_tablet_loc = first_table_loc->tablet_locs_.get_first(); + ObDASTabletLoc *first_tablet_loc = first_table_loc->get_first_tablet_loc(); if (OB_ISNULL(ptr = allocator_->alloc(sizeof(ObTaskInfo)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_ERROR("fail to alloc ObTaskInfo", K(ret)); diff --git a/src/sql/executor/ob_remote_scheduler.cpp b/src/sql/executor/ob_remote_scheduler.cpp index 136932ade..8cca7b44e 100644 --- a/src/sql/executor/ob_remote_scheduler.cpp +++ b/src/sql/executor/ob_remote_scheduler.cpp @@ -161,7 +161,7 @@ int ObRemoteScheduler::build_remote_task(ObExecContext &ctx, remote_task.set_query_schema_version(task_exec_ctx.get_query_tenant_begin_schema_version(), task_exec_ctx.get_query_sys_begin_schema_version()); remote_task.set_remote_sql_info(&plan_ctx->get_remote_sql_info()); - ObDASTabletLoc *first_tablet_loc = DAS_CTX(ctx).get_table_loc_list().get_first()->tablet_locs_.get_first(); + ObDASTabletLoc *first_tablet_loc = DAS_CTX(ctx).get_table_loc_list().get_first()->get_first_tablet_loc(); if (OB_ISNULL(session = ctx.get_my_session())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("session is null", K(ret)); diff --git a/src/sql/executor/ob_task_executor_ctx.cpp b/src/sql/executor/ob_task_executor_ctx.cpp index 8ae5ab8d6..7e15ebf8a 100644 --- a/src/sql/executor/ob_task_executor_ctx.cpp +++ b/src/sql/executor/ob_task_executor_ctx.cpp @@ -326,7 +326,8 @@ int ObTaskExecutorCtxUtil::refresh_location_cache(ObTaskExecutorCtx &task_exec_c DASTableLocList &table_locs = DAS_CTX(*task_exec_ctx.get_exec_context()).get_table_loc_list(); FOREACH_X(tmp_node, table_locs, OB_SUCC(ret)) { ObDASTableLoc *table_loc = *tmp_node; - FOREACH_X(tablet_node, table_loc->tablet_locs_, OB_SUCC(ret)) { + for (DASTabletLocListIter tablet_node = table_loc->tablet_locs_begin(); + OB_SUCC(ret) && tablet_node != table_loc->tablet_locs_end(); ++tablet_node) { const ObDASTabletLoc *tablet_loc = *tablet_node; if (is_nonblock) { const int64_t expire_renew_time = 0; //表示在刷location cache之前不清空现有的location cache diff --git a/src/sql/monitor/ob_exec_stat.h b/src/sql/monitor/ob_exec_stat.h index bfd3f6c5b..bac5ab248 100644 --- a/src/sql/monitor/ob_exec_stat.h +++ b/src/sql/monitor/ob_exec_stat.h @@ -357,7 +357,7 @@ struct ObAuditRecordData { return sql_len_ + tenant_name_len_ + user_name_len_ + db_name_len_; } - palf::SCN get_snapshot_version() const + share::SCN get_snapshot_version() const { return snapshot_.version_; } @@ -423,7 +423,7 @@ struct ObAuditRecordData { int64_t params_value_len_; char *params_value_; struct StmtSnapshot { - palf::SCN version_; // snapshot version + share::SCN version_; // snapshot version int64_t tx_id_; // snapshot inner which txn int64_t scn_; // snapshot's position in the txn char const* source_; // snapshot's acquire source diff --git a/src/sql/ob_sql.cpp b/src/sql/ob_sql.cpp index bc5355b99..a6980716f 100644 --- a/src/sql/ob_sql.cpp +++ b/src/sql/ob_sql.cpp @@ -4218,7 +4218,7 @@ int ObSql::check_need_reroute(ObPlanCacheCtx &pc_ctx, ObPhysicalPlan *plan, bool } else { const ObTableSchema *table_schema = NULL; ObDASTableLoc *first_table_loc = DAS_CTX(pc_ctx.exec_ctx_).get_table_loc_list().get_first(); - ObDASTabletLoc *first_tablet_loc = first_table_loc->tablet_locs_.get_first(); + ObDASTabletLoc *first_tablet_loc = first_table_loc->get_first_tablet_loc(); ObLSReplicaLocation ls_replica_loc; ObDASLocationRouter &loc_router = DAS_CTX(pc_ctx.exec_ctx_).get_location_router(); if (OB_FAIL(pc_ctx.sql_ctx_.schema_guard_->get_table_schema( diff --git a/src/sql/ob_sql_trans_control.cpp b/src/sql/ob_sql_trans_control.cpp index b05eb9298..0d45efa0a 100644 --- a/src/sql/ob_sql_trans_control.cpp +++ b/src/sql/ob_sql_trans_control.cpp @@ -575,7 +575,7 @@ int ObSqlTransControl::stmt_setup_snapshot_(ObSQLSessionInfo *session, auto cl = plan_ctx->get_consistency_level(); auto &snapshot = das_ctx.get_snapshot(); if (cl == ObConsistencyLevel::WEAK || cl == ObConsistencyLevel::FROZEN) { - palf::SCN snapshot_version = palf::SCN::min_scn(); + SCN snapshot_version = SCN::min_scn(); if (OB_FAIL(txs->get_weak_read_snapshot_version(snapshot_version))) { TRANS_LOG(WARN, "get weak read snapshot fail", KPC(txs)); } else { diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 10e5cc2f7..14e90f20e 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -5701,16 +5701,19 @@ int ObJoinOrder::generate_base_table_paths(PathHelper &helper) ObSEArray tbl_part_infos; uint64_t table_id = table_id_; uint64_t ref_table_id = table_meta_info_.ref_table_id_; - if (OB_FAIL(create_access_paths(table_id, ref_table_id, helper, access_paths))) { + if (!helper.is_inner_path_ && + OB_FAIL(compute_base_table_property(table_id, ref_table_id))) { + LOG_WARN("failed to compute base path property", K(ret)); + } else if (OB_FAIL(create_access_paths(table_id, ref_table_id, helper, access_paths))) { LOG_WARN("failed to add table to join order(single)", K(ret)); } else if (OB_FAIL(compute_table_location_for_paths(access_paths, tbl_part_infos))) { LOG_WARN("failed to calc table location", K(ret)); - } else if (OB_FAIL(compute_base_table_property(table_id, - ref_table_id, - helper, - access_paths))) { - LOG_WARN("failed to compute base path property", K(ret)); + } else if (OB_FAIL(estimate_size_and_width_for_base_table(helper, access_paths))) { + LOG_WARN("failed to estimate_size", K(ret)); + } else if (!helper.is_inner_path_ && !is_virtual_table(ref_table_id) && + OB_FAIL(compute_one_row_info_for_table_scan(access_paths))) { + LOG_WARN("failed to compute one row info", K(ret)); } else if (!access_paths.empty() && // when generate inner path, access_paths may be empty OB_FAIL(pruning_unstable_access_path(helper.table_opt_info_, access_paths))) { LOG_WARN("failed to pruning unstable access path", K(ret)); @@ -5729,16 +5732,10 @@ int ObJoinOrder::generate_base_table_paths(PathHelper &helper) } int ObJoinOrder::compute_base_table_property(uint64_t table_id, - uint64_t ref_table_id, - PathHelper &helper, - ObIArray &access_paths) + uint64_t ref_table_id) { int ret = OB_SUCCESS; - if (OB_FAIL(estimate_size_and_width_for_base_table(helper, access_paths))) { - LOG_WARN("failed to estimate_size", K(ret)); - } else if (helper.is_inner_path_) { - /*do nothing*/ - } else if (OB_FAIL(ObOptimizerUtil::compute_const_exprs(restrict_info_set_, output_const_exprs_))) { + if (OB_FAIL(ObOptimizerUtil::compute_const_exprs(restrict_info_set_, output_const_exprs_))) { LOG_WARN("failed to compute const exprs", K(ret)); } else if (OB_FAIL(ObEqualAnalysis::compute_equal_set(allocator_, restrict_info_set_, @@ -5748,13 +5745,10 @@ int ObJoinOrder::compute_base_table_property(uint64_t table_id, ref_table_id, get_restrict_infos()))) { LOG_WARN("failed to extract fd item set", K(ret)); - } else if (!is_virtual_table(ref_table_id) && - OB_FAIL(compute_one_row_info_for_table_scan(access_paths))) { - LOG_WARN("failed to compute one row info", K(ret)); } else { LOG_TRACE("succeed to compute base table property", K(restrict_info_set_), K(output_const_exprs_), - K(output_equal_sets_), K(is_at_most_one_row_)); + K(output_equal_sets_)); } return ret; } diff --git a/src/sql/optimizer/ob_join_order.h b/src/sql/optimizer/ob_join_order.h index 11931021c..bc9352016 100644 --- a/src/sql/optimizer/ob_join_order.h +++ b/src/sql/optimizer/ob_join_order.h @@ -1456,9 +1456,7 @@ struct NullAwareAntiJoinInfo { int generate_base_table_paths(PathHelper &helper); int compute_base_table_property(uint64_t table_id, - uint64_t ref_table_id, - PathHelper &helper, - ObIArray &access_paths); + uint64_t ref_table_id); /** * @brief generate_subquery_path diff --git a/src/sql/optimizer/ob_table_location.cpp b/src/sql/optimizer/ob_table_location.cpp index 51db3498e..c3814269e 100644 --- a/src/sql/optimizer/ob_table_location.cpp +++ b/src/sql/optimizer/ob_table_location.cpp @@ -5166,7 +5166,7 @@ int ObTableLocation::get_full_leader_table_loc(ObIAllocator &allocator, OX(tablet_loc = new(tablet_buf) ObDASTabletLoc()); OX(tablet_loc->loc_meta_ = loc_meta); OZ(ObDASLocationRouter::get_leader(tenant_id, tablet_ids.at(i), *tablet_loc, expire_renew_time)); - OZ(table_loc->tablet_locs_.push_back(tablet_loc)); + OZ(table_loc->add_tablet_loc(tablet_loc)); } } diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index 75be43cce..e1da9e0ae 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -1031,7 +1031,6 @@ static const NonReservedKeyword Mysql_sql_keywords_in_pl[] = {"function", FUNCTION}, {"get", GET}, {"grant", GRANT}, - {"hosts", HOSTS}, {"index", INDEX}, {"insert", INSERT}, {"install", INSTALL}, diff --git a/src/sql/parser/ob_fast_parser.cpp b/src/sql/parser/ob_fast_parser.cpp index 2eb66cce4..4b453073d 100644 --- a/src/sql/parser/ob_fast_parser.cpp +++ b/src/sql/parser/ob_fast_parser.cpp @@ -82,7 +82,7 @@ ObFastParserBase::ObFastParserBase( is_batched_multi_stmt_split_on_(enable_batched_multi_stmt), is_mysql_compatible_comment_(false), cur_token_begin_pos_(0), copy_begin_pos_(0), copy_end_pos_(0), - tmp_buf_(nullptr), tmp_buf_len_(0), last_well_formed_len_(0), + tmp_buf_(nullptr), tmp_buf_len_(0), last_escape_check_pos_(0), param_node_list_(nullptr), tail_param_node_(nullptr), cur_token_type_(INVALID_TOKEN), allocator_(allocator), parse_next_token_func_(nullptr), process_idf_func_(nullptr) @@ -1382,22 +1382,21 @@ inline void ObFastParserBase::check_real_escape(bool &is_real_escape) { if (OB_NOT_NULL(charset_info_) && charset_info_->escape_with_backslash_is_dangerous) { char *cur_pos = tmp_buf_ + tmp_buf_len_; - char *last_check_pos = tmp_buf_ + last_well_formed_len_; + char *last_check_pos = tmp_buf_ + last_escape_check_pos_; int error = 0; int expected_well_formed_len = cur_pos - last_check_pos; - int real_well_formed_len = charset_info_->cset->well_formed_len( - charset_info_, last_check_pos, cur_pos, UINT64_MAX, &error); - if (error != 0) { + + while (last_check_pos < cur_pos) { + size_t real_well_formed_len = charset_info_->cset->well_formed_len( + charset_info_, last_check_pos, cur_pos, UINT64_MAX, &error); + last_check_pos += (real_well_formed_len + ((error != 0) ? 1 : 0)); + } + + if (error != 0) { //the final well-formed result *cur_pos = '\\'; - if (real_well_formed_len == expected_well_formed_len - 1 - && charset_info_->cset->ismbchar(charset_info_, cur_pos - 1, cur_pos + 1)) { + if (charset_info_->cset->ismbchar(charset_info_, cur_pos - 1, cur_pos + 1)) { is_real_escape = false; - last_well_formed_len_ = tmp_buf_len_ + 1; - } else { - last_well_formed_len_ = tmp_buf_len_; } - } else { - last_well_formed_len_ = tmp_buf_len_; } } } @@ -1826,7 +1825,7 @@ int ObFastParserMysql::process_string(const char quote) ParseNode **child_node = NULL; char ch = INVALID_CHAR; tmp_buf_len_ = 0; - last_well_formed_len_ = 0; + last_escape_check_pos_ = 0; if (nullptr == tmp_buf_ && OB_ISNULL(tmp_buf_ = static_cast(allocator_.alloc(raw_sql_.raw_sql_len_ + 1)))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -1855,6 +1854,7 @@ int ObFastParserMysql::process_string(const char quote) } else { process_escape_string(tmp_buf_, tmp_buf_len_); } + last_escape_check_pos_ = tmp_buf_len_; } else if (quote == ch) { if (quote == raw_sql_.peek()) { // double quote ch = raw_sql_.scan(); diff --git a/src/sql/parser/ob_fast_parser.h b/src/sql/parser/ob_fast_parser.h index 4b6db13f1..0922ad8e0 100644 --- a/src/sql/parser/ob_fast_parser.h +++ b/src/sql/parser/ob_fast_parser.h @@ -495,7 +495,7 @@ protected: int64_t copy_end_pos_; char *tmp_buf_; int64_t tmp_buf_len_; - int64_t last_well_formed_len_; + int64_t last_escape_check_pos_; ParamList *param_node_list_; ParamList *tail_param_node_; TokenType cur_token_type_; diff --git a/src/sql/parser/parse_malloc.cpp b/src/sql/parser/parse_malloc.cpp index 7a9585c00..4b12035b9 100644 --- a/src/sql/parser/parse_malloc.cpp +++ b/src/sql/parser/parse_malloc.cpp @@ -267,27 +267,24 @@ char *parse_strdup_with_replace_multi_byte_char(const char *str, int *connection } bool check_real_escape(const ObCharsetInfo *cs, char *str, int64_t str_len, - int64_t *last_well_formed_len) + int64_t last_escape_check_pos) { bool is_real_escape = true; - if (NULL != cs && NULL != last_well_formed_len && cs->escape_with_backslash_is_dangerous) { + if (NULL != cs && cs->escape_with_backslash_is_dangerous) { char *cur_pos = str + str_len; - char *last_check_pos = str + *last_well_formed_len; + char *last_check_pos = str + last_escape_check_pos; int error = 0; size_t expected_well_formed_len = cur_pos - last_check_pos; - size_t real_well_formed_len = cs->cset->well_formed_len( - cs, last_check_pos, cur_pos, UINT64_MAX, &error); - if (error != 0) { + while (last_check_pos < cur_pos) { + size_t real_well_formed_len = cs->cset->well_formed_len( + cs, last_check_pos, cur_pos, UINT64_MAX, &error); + last_check_pos += (real_well_formed_len + ((error != 0) ? 1 : 0)); + } + if (error != 0) { //the final well-formed result *cur_pos = '\\'; - if (real_well_formed_len == expected_well_formed_len - 1 - && cs->cset->ismbchar(cs, cur_pos - 1, cur_pos + 1)) { + if (cs->cset->ismbchar(cs, cur_pos - 1, cur_pos + 1)) { is_real_escape = false; - *last_well_formed_len = str_len + 1; - } else { - *last_well_formed_len = str_len; } - } else { - *last_well_formed_len = str_len; } } return is_real_escape; diff --git a/src/sql/parser/parse_malloc.h b/src/sql/parser/parse_malloc.h index 193347be8..feeee14c5 100644 --- a/src/sql/parser/parse_malloc.h +++ b/src/sql/parser/parse_malloc.h @@ -35,7 +35,7 @@ extern char *cp_str_value(const char *src, const size_t nbyte, void *malloc_pool extern char *parse_strdup_with_replace_multi_byte_char(const char *str, int *connection_collation_, void *malloc_pool, int64_t *out_len); extern bool check_real_escape(const struct ObCharsetInfo *cs, char *str, int64_t str_len, - int64_t *last_well_formed_len); + int64_t last_escape_check_pos); extern void *parser_alloc(void *malloc_pool, const int64_t alloc_size); extern void *malloc_parentheses_info(const size_t nbyte, void *malloc_pool); diff --git a/src/sql/parser/parse_node.h b/src/sql/parser/parse_node.h index 34c8af527..e66602c4c 100644 --- a/src/sql/parser/parse_node.h +++ b/src/sql/parser/parse_node.h @@ -321,7 +321,7 @@ typedef struct PLParseInfo pl_parse_info_; /*for q-quote*/ ObMinusStatusCtx minus_ctx_; // for fast parser to parse negative value - int64_t last_well_formed_len_; //解析quoted string%parse-param时的一个临时变量，处理连接gbk字符集时遇到的转义字符问题 + int64_t last_escape_check_pos_; //解析quoted string%parse-param时的一个临时变量，处理连接gbk字符集时遇到的转义字符问题 int connection_collation_;//connection collation bool mysql_compatible_comment_; //whether the parser is parsing "/*! xxxx */" bool enable_compatible_comment_; diff --git a/src/sql/parser/sql_parser_base.h b/src/sql/parser/sql_parser_base.h index 7b09de352..4c105b2e3 100644 --- a/src/sql/parser/sql_parser_base.h +++ b/src/sql/parser/sql_parser_base.h @@ -64,7 +64,7 @@ extern int64_t ob_strntoll(const char *ptr, size_t len, int base, char **end, in extern int64_t ob_strntoull(const char *ptr, size_t len, int base, char **end, int *err); extern int store_prentthese_info(int left, int right, ParseResult *result); extern bool check_real_escape(const struct ObCharsetInfo *cs, char *str, int64_t str_len, - int64_t *last_well_formed_len); + int64_t last_escape_check_pos); int add_alias_name(ParseNode *node, ParseResult *result, int end); @@ -692,7 +692,7 @@ do { #define CHECK_REAL_ESCAPE(is_real_escape) \ is_real_escape = check_real_escape(p->charset_info_, p->tmp_literal_, \ - yylval->node->str_len_, &(p->last_well_formed_len_)) + yylval->node->str_len_, p->last_escape_check_pos_) /* do { \ if (NULL != p->charset_info_ && p->charset_info_->escape_with_backslash_is_dangerous) { \ diff --git a/src/sql/parser/sql_parser_mysql_mode.l b/src/sql/parser/sql_parser_mysql_mode.l index 7d0ca2ff1..cc27b6c93 100644 --- a/src/sql/parser/sql_parser_mysql_mode.l +++ b/src/sql/parser/sql_parser_mysql_mode.l @@ -264,7 +264,7 @@ FALSE { check_value(yylval); malloc_new_node(yylval->node, p->malloc_pool_, T_VARCHAR, 0); yylval->node->str_len_ = 0; - p->last_well_formed_len_ = 0; + p->last_escape_check_pos_ = 0; yylval->node->str_value_ = NULL; if (IS_FAST_PARAMETERIZE && !IS_NEED_PARAMETERIZE) { COPY_WRITE(); @@ -327,6 +327,7 @@ FALSE { } else { HANDLE_ESCAPE(p); } + p->last_escape_check_pos_ = yylval->node->str_len_; if (IS_FAST_PARAMETERIZE && !IS_NEED_PARAMETERIZE) { COPY_WRITE(); } @@ -408,7 +409,7 @@ FALSE { } malloc_new_node(yylval->node, p->malloc_pool_, T_VARCHAR, 0); yylval->node->str_len_ = 0; - p->last_well_formed_len_ = 0; + p->last_escape_check_pos_ = 0; if (IS_FAST_PARAMETERIZE && !IS_NEED_PARAMETERIZE) { return OUTLINE_DEFAULT_TOKEN; } @@ -476,6 +477,7 @@ FALSE { } else { HANDLE_ESCAPE(p); } + p->last_escape_check_pos_ = yylval->node->str_len_; if (IS_FAST_PARAMETERIZE && !IS_NEED_PARAMETERIZE) { return OUTLINE_DEFAULT_TOKEN; } @@ -1059,7 +1061,7 @@ Timestamp{whitespace}?\"[^\"]*\" { check_value(yylval); malloc_new_node(yylval->node, p->malloc_pool_, T_VARCHAR, 0); yylval->node->str_len_ = 0; - p->last_well_formed_len_ = 0; + p->last_escape_check_pos_ = 0; yylval->node->str_value_ = NULL; if (IS_FAST_PARAMETERIZE && !IS_NEED_PARAMETERIZE && !p->is_ignore_token_) { COPY_WRITE(); diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index 34d87a913..d0cddb9fb 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -38,6 +38,7 @@ #include "share/backup/ob_backup_io_adapter.h" #include "share/backup/ob_backup_config.h" #include "observer/mysql/ob_query_response_time.h" +#include "rootserver/ob_rs_job_table_operator.h" //ObRsJobType namespace oceanbase { @@ -2600,15 +2601,24 @@ int ObRunUpgradeJobResolver::resolve(const ParseNode &parse_tree) LOG_ERROR("create ObRunUpgradeJobStmt failed", KR(ret)); } else { stmt_ = stmt; - ObString version_str; + ObString str; uint64_t version = OB_INVALID_VERSION; - if (OB_FAIL(Util::resolve_string(parse_tree.children_[0], version_str))) { + if (OB_FAIL(Util::resolve_string(parse_tree.children_[0], str))) { LOG_WARN("resolve string failed", KR(ret)); - } else if (OB_FAIL(ObClusterVersion::get_version(version_str, version))) { - LOG_WARN("fail to get version", KR(ret), K(version_str)); + } else if (0 == str.case_compare(rootserver::ObRsJobTableOperator::get_job_type_str( + rootserver::JOB_TYPE_UPGRADE_SYSTEM_VARIABLE))) { + stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_SYSTEM_VARIABLE; + } else if (0 == str.case_compare(rootserver::ObRsJobTableOperator::get_job_type_str( + rootserver::JOB_TYPE_UPGRADE_SYSTEM_TABLE))) { + stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_SYSTEM_TABLE; } else { - stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::RUN_UPGRADE_JOB; - stmt->get_rpc_arg().version_ = static_cast(version); + // UPGRADE_POST_ACTION + if (OB_FAIL(ObClusterVersion::get_version(str, version))) { + LOG_WARN("fail to get version", KR(ret), K(str)); + } else { + stmt->get_rpc_arg().action_ = obrpc::ObUpgradeJobArg::UPGRADE_POST_ACTION; + stmt->get_rpc_arg().version_ = static_cast(version); + } } } } diff --git a/src/sql/resolver/cmd/ob_show_resolver.cpp b/src/sql/resolver/cmd/ob_show_resolver.cpp index a419bc8f7..3ef81ed42 100644 --- a/src/sql/resolver/cmd/ob_show_resolver.cpp +++ b/src/sql/resolver/cmd/ob_show_resolver.cpp @@ -1958,6 +1958,7 @@ int ObShowResolver::resolve_like_or_where_clause(ObShowResolverContext &ctx) && parse_tree->type_ != T_SHOW_STATUS && parse_tree->type_ != T_SHOW_TABLEGROUPS && parse_tree->type_ != T_SHOW_PROCEDURE_STATUS + && parse_tree->type_ != T_SHOW_FUNCTION_STATUS && parse_tree->type_ != T_SHOW_TRIGGERS)) { // do nothing } else { @@ -2691,7 +2692,7 @@ DEFINE_SHOW_CLAUSE_SET(SHOW_TABLE_STATUS, "name"); DEFINE_SHOW_CLAUSE_SET(SHOW_PROCEDURE_STATUS, NULL, - "select database_name AS `Db`, routine_name AS `Name`, c.type AS `Type`, c.definer AS `Definer`, p.gmt_modified AS `Modified`, p.gmt_create AS `Created`,'DEFINER' AS `Security_type`, p.comment AS `Comment`, character_set_client, collation_connection, collation_database AS `Database Collation`from %s.%s p, %s.%s d, %s.%s c where p.tenant_id = d.tenant_id and p.database_id = d.database_id and d.database_name = c.db and p.routine_name = c.name and (case c.type when 'PROCEDURE' then 1 else 0 end) = p.routine_type and d.database_id = %ld and p.routine_type = %ld ORDER BY name COLLATE utf8mb4_bin ASC", + "select database_name AS `Db`, routine_name AS `Name`, c.type AS `Type`, c.definer AS `Definer`, p.gmt_modified AS `Modified`, p.gmt_create AS `Created`,'DEFINER' AS `Security_type`, p.comment AS `Comment`, character_set_client, collation_connection, collation_database AS `Database Collation`from %s.%s p, %s.%s d, %s.%s c where p.tenant_id = d.tenant_id and p.database_id = d.database_id and d.database_name = c.db and p.routine_name = c.name and (case c.type when 'PROCEDURE' then 1 when 'FUNCTION' then 2 else 0 end) = p.routine_type and d.database_id = %ld and p.routine_type = %ld ORDER BY name COLLATE utf8mb4_bin ASC", NULL, "name"); DEFINE_SHOW_CLAUSE_SET(SHOW_TRIGGERS, diff --git a/src/sql/resolver/ddl/ob_trigger_resolver.cpp b/src/sql/resolver/ddl/ob_trigger_resolver.cpp index 06a2738d6..0d71c248c 100644 --- a/src/sql/resolver/ddl/ob_trigger_resolver.cpp +++ b/src/sql/resolver/ddl/ob_trigger_resolver.cpp @@ -278,6 +278,7 @@ int ObTriggerResolver::resolve_instead_dml_trigger(const ParseNode &parse_node, OV (OB_ISNULL(parse_node.children_[2]), OB_ERR_WHEN_CLAUSE_IN_TRI); OV (OB_NOT_NULL(parse_node.children_[3])); // trigger body. OX (trigger_arg.trigger_info_.add_before_row()); // instead of trigger is always before row. + OX (trigger_arg.trigger_info_.add_instead_row()); OZ (resolve_dml_event_option(*parse_node.children_[0], trigger_arg)); OZ (resolve_reference_names(parse_node.children_[1], trigger_arg)); OZ (resolve_trigger_status(parse_node.int16_values_[1], trigger_arg)); diff --git a/src/sql/rewrite/ob_transform_join_elimination.cpp b/src/sql/rewrite/ob_transform_join_elimination.cpp index a6c3e27fd..3008f7961 100644 --- a/src/sql/rewrite/ob_transform_join_elimination.cpp +++ b/src/sql/rewrite/ob_transform_join_elimination.cpp @@ -1021,8 +1021,10 @@ int ObTransformJoinElimination::eliminate_left_outer_join(ObDMLStmt *stmt, if (OB_UNLIKELY(semi_infos.at(i)->left_table_ids_.empty())) { if (OB_ISNULL(root_table)) { // do nothing - } else if (OB_FAIL(semi_infos.at(i)->left_table_ids_.push_back(root_table->table_id_))) { - LOG_WARN("failed to add semi-info", K(ret)); + } else if (root_table->is_joined_table()) { + ret = append(semi_infos.at(i)->left_table_ids_, static_cast(root_table)->single_table_ids_); + } else { + ret = semi_infos.at(i)->left_table_ids_.push_back(root_table->table_id_); } } } diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index 83e04735b..27bb24448 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -104,6 +104,7 @@ ObBasicSessionInfo::ObBasicSessionInfo() next_tx_isolation_(transaction::ObTxIsolationLevel::INVALID), log_id_level_map_valid_(false), cur_phy_plan_(NULL), + plan_id_(0), capability_(), proxy_capability_(), client_mode_(OB_MIN_CLIENT_MODE), @@ -374,6 +375,7 @@ void ObBasicSessionInfo::reset(bool skip_sys_var) log_id_level_map_valid_ = false; log_id_level_map_.reset_level(); cur_phy_plan_ = NULL; + plan_id_ = 0; capability_.capability_ = 0; proxy_capability_.capability_ = 0; client_mode_ = OB_MIN_CLIENT_MODE; @@ -1927,6 +1929,7 @@ int ObBasicSessionInfo::sys_variable_exists(const ObString &var, bool &is_exists return ret; } +// for query and DML int ObBasicSessionInfo::set_cur_phy_plan(ObPhysicalPlan *cur_phy_plan) { int ret = OB_SUCCESS; @@ -1935,6 +1938,7 @@ int ObBasicSessionInfo::set_cur_phy_plan(ObPhysicalPlan *cur_phy_plan) LOG_WARN("current physical plan is NULL", K(lbt()), K(ret)); } else { cur_phy_plan_ = cur_phy_plan; + plan_id_ = cur_phy_plan->get_plan_id(); int64_t len = cur_phy_plan->stat_.sql_id_.length(); MEMCPY(sql_id_, cur_phy_plan->stat_.sql_id_.ptr(), len); sql_id_[len] = '\0'; @@ -1942,6 +1946,7 @@ int ObBasicSessionInfo::set_cur_phy_plan(ObPhysicalPlan *cur_phy_plan) return ret; } +// for cmd only void ObBasicSessionInfo::set_cur_sql_id(char *sql_id) { if (nullptr == sql_id) { diff --git a/src/sql/session/ob_basic_session_info.h b/src/sql/session/ob_basic_session_info.h index c80278a0c..80312a4af 100644 --- a/src/sql/session/ob_basic_session_info.h +++ b/src/sql/session/ob_basic_session_info.h @@ -1120,6 +1120,7 @@ public: int64_t get_current_execution_id() const { return current_execution_id_; } const common::ObCurTraceId::TraceId &get_last_trace_id() const { return last_trace_id_; } const common::ObCurTraceId::TraceId &get_current_trace_id() const { return curr_trace_id_; } + uint64_t get_current_plan_id() const { return plan_id_; } void set_current_execution_id(int64_t execution_id) { current_execution_id_ = execution_id; } void set_last_trace_id(common::ObCurTraceId::TraceId *trace_id) { @@ -1860,7 +1861,7 @@ private: protected: transaction::ObTxDesc *tx_desc_; transaction::ObTxExecResult tx_result_; // TODO: move to QueryCtx/ExecCtx - palf::SCN unused_read_snapshot_version_;//serialize compatibility preserved + share::SCN unused_read_snapshot_version_;//serialize compatibility preserved transaction::ObXATransID xid_; bool associated_xa_; // session joined distr-xa-trans by xa-start public: @@ -1926,6 +1927,7 @@ private: ObPhysicalPlan *cur_phy_plan_; // sql_id of cur_phy_plan_ sql char sql_id_[common::OB_MAX_SQL_ID_LENGTH + 1]; + uint64_t plan_id_; // for ASH sampling, get current SQL's sql_id & plan_id char flt_trace_id_[common::OB_MAX_UUID_LENGTH + 1]; char flt_span_id_[common::OB_MAX_UUID_LENGTH + 1]; diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index 6c7a32b7c..c3329974a 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -865,6 +865,9 @@ ObPlanCache *ObSQLSessionInfo::get_plan_cache() if (tenant_id > OB_SYS_TENANT_ID && tenant_id <= OB_MAX_RESERVED_TENANT_ID) { // all virtual tenants use sys tenant's plan cache tenant_id = OB_SYS_TENANT_ID; + } else if (OB_INVALID_TENANT_ID == tenant_id) { + // When it is used by threads regardless of tenants, it switches to the system tenant for execution. + tenant_id = OB_SYS_TENANT_ID; } plan_cache_ = plan_cache_manager_->get_or_create_plan_cache(tenant_id, pc_mem_conf); @@ -1215,6 +1218,7 @@ int ObSQLSessionInfo::add_cursor(pl::ObPLCursorInfo *cursor) // open_cursors is 0 to indicate a special state, no limit is set #define NEED_CHECK_SESS_OPEN_CURSORS_LIMIT(v) (0 == v ? false : true) int ret = OB_SUCCESS; + bool add_cursor_success = false; omt::ObTenantConfigGuard tenant_config(TENANT_CONF(get_effective_tenant_id())); CK (tenant_config.is_valid()); CK (OB_NOT_NULL(cursor)); @@ -1248,6 +1252,7 @@ int ObSQLSessionInfo::add_cursor(pl::ObPLCursorInfo *cursor) LOG_WARN("fail insert ps id to hash map", K(id), K(*cursor), K(ret)); } else { cursor->set_id(id); + add_cursor_success = true; if (lib::is_diagnose_info_enabled()) { EVENT_INC(SQL_OPEN_CURSORS_CURRENT); EVENT_INC(SQL_OPEN_CURSORS_CUMULATIVE); @@ -1255,6 +1260,30 @@ int ObSQLSessionInfo::add_cursor(pl::ObPLCursorInfo *cursor) LOG_DEBUG("ps cursor: add cursor", K(ret), K(id), K(get_sessid())); } } + if (!add_cursor_success && OB_NOT_NULL(cursor)) { + int64_t id = cursor->get_id(); + int tmp_ret = close_cursor(cursor); + ret = OB_SUCCESS == ret ? tmp_ret : ret; + if (OB_SUCCESS != tmp_ret) { + LOG_WARN("close cursor fail when add cursor to sesssion.", K(ret), K(id), K(get_sessid())); + } + } + return ret; +} + +int ObSQLSessionInfo::close_cursor(ObPLCursorInfo *&cursor) +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(cursor)) { + int64_t id = cursor->get_id(); + OZ (cursor->close(*this)); + cursor->~ObPLCursorInfo(); + get_cursor_allocator().free(cursor); + cursor = NULL; + LOG_DEBUG("close cursor", K(ret), K(id), K(get_sessid())); + } else { + LOG_DEBUG("close cursor is null", K(get_sessid())); + } return ret; } diff --git a/src/sql/session/ob_sql_session_info.h b/src/sql/session/ob_sql_session_info.h index 74530873e..8d01964a7 100644 --- a/src/sql/session/ob_sql_session_info.h +++ b/src/sql/session/ob_sql_session_info.h @@ -676,6 +676,7 @@ public: pl::ObPLCursorInfo *get_cursor(int64_t cursor_id); pl::ObDbmsCursorInfo *get_dbms_cursor(int64_t cursor_id); int add_cursor(pl::ObPLCursorInfo *cursor); + int close_cursor(pl::ObPLCursorInfo *&cursor); int close_cursor(int64_t cursor_id); int make_cursor(pl::ObPLCursorInfo *&cursor); int init_cursor_cache(); diff --git a/src/storage/access/ob_sstable_row_lock_checker.cpp b/src/storage/access/ob_sstable_row_lock_checker.cpp index b799081aa..45852235d 100644 --- a/src/storage/access/ob_sstable_row_lock_checker.cpp +++ b/src/storage/access/ob_sstable_row_lock_checker.cpp @@ -14,6 +14,7 @@ #include "ob_sstable_row_lock_checker.h" #include "storage/tx/ob_trans_define.h" using namespace oceanbase::common; +using namespace oceanbase::share; using namespace oceanbase::blocksstable; namespace oceanbase { @@ -109,7 +110,7 @@ int ObSSTableRowLockChecker::check_row_locked(ObStoreRowLockState &lock_state) if (OB_UNLIKELY(OB_ITER_END != ret)) { LOG_WARN("Fail to get next row", K(ret), K_(multi_version_range)); } - } else if (palf::SCN::min_scn() != lock_state.trans_version_ || lock_state.is_locked_) { + } else if (SCN::min_scn() != lock_state.trans_version_ || lock_state.is_locked_) { break; } } @@ -119,7 +120,7 @@ int ObSSTableRowLockChecker::check_row_locked(ObStoreRowLockState &lock_state) } if (OB_SUCC(ret) && transaction::ObTransVersion::INVALID_TRANS_VERSION != prefetcher_.row_lock_check_version_) { - if (OB_UNLIKELY(lock_state.trans_version_ != palf::SCN::min_scn() || lock_state.is_locked_)) { + if (OB_UNLIKELY(lock_state.trans_version_ != SCN::min_scn() || lock_state.is_locked_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected lock state", K(ret), K_(lock_state.trans_version), K_(lock_state.is_locked)); } else if (OB_FAIL(lock_state.trans_version_.convert_for_tx(prefetcher_.row_lock_check_version_))) { diff --git a/src/storage/access/ob_table_access_context.h b/src/storage/access/ob_table_access_context.h index 8dae6504c..2633c9421 100644 --- a/src/storage/access/ob_table_access_context.h +++ b/src/storage/access/ob_table_access_context.h @@ -16,7 +16,7 @@ #include "ob_table_access_param.h" #include "storage/lob/ob_lob_locator.h" #include "storage/tx/ob_defensive_check_mgr.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -141,7 +141,7 @@ public: int64_t out_cnt_; common::ObVersionRange trans_version_range_; const common::ObSEArray *range_array_pos_; - palf::SCN merge_scn_; + share::SCN merge_scn_; ObLobLocatorHelper *lob_locator_helper_; ObStoreRowIterPool *iter_pool_; ObBlockRowStore *block_row_store_; diff --git a/src/storage/backup/ob_backup_handler.cpp b/src/storage/backup/ob_backup_handler.cpp index f2262e00f..e6eb33359 100644 --- a/src/storage/backup/ob_backup_handler.cpp +++ b/src/storage/backup/ob_backup_handler.cpp @@ -26,7 +26,7 @@ namespace backup { int ObBackupHandler::schedule_backup_meta_dag(const ObBackupJobDesc &job_desc, const ObBackupDest &backup_dest, const uint64_t tenant_id, const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, - const int64_t turn_id, const int64_t retry_id, const palf::SCN &start_scn) + const int64_t turn_id, const int64_t retry_id, const SCN &start_scn) { int ret = OB_SUCCESS; MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); @@ -184,7 +184,7 @@ int ObBackupHandler::schedule_build_tenant_level_index_dag(const ObBackupJobDesc int ObBackupHandler::schedule_backup_complement_log_dag(const ObBackupJobDesc &job_desc, const share::ObBackupDest &backup_dest, const uint64_t tenant_id, const share::ObBackupSetDesc &backup_set_desc, - const share::ObLSID &ls_id, const palf::SCN &start_scn, const palf::SCN &end_scn) + const share::ObLSID &ls_id, const SCN &start_scn, const SCN &end_scn) { int ret = OB_SUCCESS; MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); diff --git a/src/storage/backup/ob_backup_handler.h b/src/storage/backup/ob_backup_handler.h index 5f14ee2d2..3296739f9 100644 --- a/src/storage/backup/ob_backup_handler.h +++ b/src/storage/backup/ob_backup_handler.h @@ -24,7 +24,7 @@ class ObBackupHandler { public: static int schedule_backup_meta_dag(const ObBackupJobDesc &job_desc, const share::ObBackupDest &backup_dest, const uint64_t tenant_id, const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, - const int64_t turn_id, const int64_t retry_id, const palf::SCN &start_scn); + const int64_t turn_id, const int64_t retry_id, const share::SCN &start_scn); static int schedule_backup_data_dag(const ObBackupJobDesc &job_desc, const share::ObBackupDest &backup_dest, const uint64_t tenant_id, const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, const int64_t turn_id, const int64_t retry_id, const share::ObBackupDataType &backup_data_type); @@ -33,7 +33,7 @@ public: const int64_t turn_id, const int64_t retry_id, const share::ObBackupDataType &backup_data_type); static int schedule_backup_complement_log_dag(const ObBackupJobDesc &job_desc, const share::ObBackupDest &backup_dest, const uint64_t tenant_id, const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, - const palf::SCN &start_scn, const palf::SCN &end_scn); + const share::SCN &start_scn, const share::SCN &end_scn); }; } // namespace backup diff --git a/src/storage/backup/ob_backup_task.cpp b/src/storage/backup/ob_backup_task.cpp index 8fbb35a67..bfa97be93 100644 --- a/src/storage/backup/ob_backup_task.cpp +++ b/src/storage/backup/ob_backup_task.cpp @@ -68,7 +68,7 @@ static int get_ls_handle(const uint64_t tenant_id, const share::ObLSID &ls_id, s } static int advance_checkpoint_by_flush(const uint64_t tenant_id, const share::ObLSID &ls_id, - const palf::SCN &start_scn, storage::ObLS *ls) + const SCN &start_scn, storage::ObLS *ls) { int ret = OB_SUCCESS; const int64_t advance_checkpoint_timeout = GCONF._advance_checkpoint_timeout; @@ -100,7 +100,7 @@ static int advance_checkpoint_by_flush(const uint64_t tenant_id, const share::Ob } else if (OB_FAIL(ls->get_ls_meta_package(ls_meta_package))) { LOG_WARN("failed to get ls meta package", K(ret), K(tenant_id), K(ls_id)); } else { - const palf::SCN clog_checkpoint_scn = ls_meta_package.ls_meta_.get_clog_checkpoint_scn(); + const SCN clog_checkpoint_scn = ls_meta_package.ls_meta_.get_clog_checkpoint_scn(); if (clog_checkpoint_scn >= start_scn) { LOG_INFO("clog checkpoint scn has passed start scn", K(i), @@ -1040,7 +1040,7 @@ ObLSBackupMetaDag::~ObLSBackupMetaDag() {} int ObLSBackupMetaDag::init( - const palf::SCN &start_scn, const ObLSBackupDagInitParam ¶m, const ObBackupReportCtx &report_ctx) + const SCN &start_scn, const ObLSBackupDagInitParam ¶m, const ObBackupReportCtx &report_ctx) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -1828,7 +1828,7 @@ ObLSBackupComplementLogDag::~ObLSBackupComplementLogDag() int ObLSBackupComplementLogDag::init(const ObBackupJobDesc &job_desc, const ObBackupDest &backup_dest, const uint64_t tenant_id, const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, - const int64_t turn_id, const int64_t retry_id, const palf::SCN &start_scn, const palf::SCN &end_scn, + const int64_t turn_id, const int64_t retry_id, const SCN &start_scn, const SCN &end_scn, const ObBackupReportCtx &report_ctx) { int ret = OB_SUCCESS; @@ -3383,7 +3383,7 @@ ObLSBackupMetaTask::~ObLSBackupMetaTask() {} int ObLSBackupMetaTask::init( - const palf::SCN &start_scn, const ObLSBackupDagInitParam ¶m, const ObBackupReportCtx &report_ctx) + const SCN &start_scn, const ObLSBackupDagInitParam ¶m, const ObBackupReportCtx &report_ctx) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -3409,7 +3409,7 @@ int ObLSBackupMetaTask::process() MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); SERVER_EVENT_SYNC_ADD("backup_data", "before_backup_meta"); DEBUG_SYNC(BEFORE_BACKUP_META); - const palf::SCN start_scn = start_scn_; + const SCN start_scn = start_scn_; const int64_t task_id = param_.job_desc_.task_id_; const uint64_t tenant_id = param_.tenant_id_; const share::ObLSID ls_id = param_.ls_id_; @@ -3465,7 +3465,7 @@ int ObLSBackupMetaTask::process() } int ObLSBackupMetaTask::advance_checkpoint_by_flush_( - const uint64_t tenant_id, const share::ObLSID &ls_id, const palf::SCN &start_scn) + const uint64_t tenant_id, const share::ObLSID &ls_id, const SCN &start_scn) { int ret = OB_SUCCESS; checkpoint::ObCheckpointExecutor *checkpoint_executor = NULL; @@ -3527,7 +3527,7 @@ int ObLSBackupMetaTask::backup_ls_meta_package_(const ObBackupLSMetaInfo &ls_met } int ObLSBackupMetaTask::backup_ls_tablet_list_( - const palf::SCN &scn, const share::ObLSID &ls_id, const common::ObIArray &tablet_id_list) + const SCN &scn, const share::ObLSID &ls_id, const common::ObIArray &tablet_id_list) { int ret = OB_SUCCESS; share::ObBackupDataStore store; @@ -3546,7 +3546,7 @@ int ObLSBackupMetaTask::backup_ls_tablet_list_( return ret; } -int ObLSBackupMetaTask::build_backup_data_ls_tablet_desc_(const share::ObLSID &ls_id, const palf::SCN &scn, +int ObLSBackupMetaTask::build_backup_data_ls_tablet_desc_(const share::ObLSID &ls_id, const SCN &scn, const common::ObIArray &tablet_id_list, share::ObBackupDataTabletToLSDesc &desc) { int ret = OB_SUCCESS; @@ -3775,7 +3775,7 @@ int ObLSBackupPrepareTask::may_need_advance_checkpoint_() { int ret = OB_SUCCESS; int64_t rebuild_seq = 0; - palf::SCN backup_clog_checkpoint_scn; + SCN backup_clog_checkpoint_scn; if (OB_ISNULL(ls_backup_ctx_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls backup ctx should not be null", K(ret)); @@ -3811,7 +3811,7 @@ int ObLSBackupPrepareTask::may_need_advance_checkpoint_() return ret; } -int ObLSBackupPrepareTask::fetch_backup_ls_meta_(int64_t &rebuild_seq, palf::SCN &clog_checkpoint_scn) +int ObLSBackupPrepareTask::fetch_backup_ls_meta_(int64_t &rebuild_seq, SCN &clog_checkpoint_scn) { int ret = OB_SUCCESS; rebuild_seq = 0; @@ -3835,8 +3835,8 @@ int ObLSBackupPrepareTask::fetch_backup_ls_meta_(int64_t &rebuild_seq, palf::SCN int ObLSBackupPrepareTask::check_tx_data_can_explain_user_data_() { int ret = OB_SUCCESS; - palf::SCN backup_filled_tx_scn = palf::SCN::max_scn(); - palf::SCN cur_min_filled_tx_scn = palf::SCN::max_scn(); + SCN backup_filled_tx_scn = SCN::max_scn(); + SCN cur_min_filled_tx_scn = SCN::max_scn(); if (!backup_data_type_.is_minor_backup()) { // do nothing } else if (OB_FAIL(get_backup_tx_data_table_filled_tx_scn_(backup_filled_tx_scn))) { @@ -3854,10 +3854,10 @@ int ObLSBackupPrepareTask::check_tx_data_can_explain_user_data_() return ret; } -int ObLSBackupPrepareTask::get_backup_tx_data_table_filled_tx_scn_(palf::SCN &filled_tx_scn) +int ObLSBackupPrepareTask::get_backup_tx_data_table_filled_tx_scn_(SCN &filled_tx_scn) { int ret = OB_SUCCESS; - filled_tx_scn = palf::SCN::max_scn(); + filled_tx_scn = SCN::max_scn(); const common::ObTabletID &tx_data_tablet_id = LS_TX_DATA_TABLET; const ObBackupMetaType meta_type = ObBackupMetaType::BACKUP_SSTABLE_META; ObBackupDataType sys_backup_data_type; @@ -3878,7 +3878,7 @@ int ObLSBackupPrepareTask::get_backup_tx_data_table_filled_tx_scn_(palf::SCN &fi backup_path.get_obstr(), param_.backup_dest_.get_storage_info(), meta_index, meta_array))) { LOG_WARN("failed to read sstable metas", K(ret), K(backup_path), K(meta_index)); } else if (meta_array.empty()) { - filled_tx_scn = palf::SCN::min_scn(); + filled_tx_scn = SCN::min_scn(); LOG_INFO("the log stream do not have tx data sstable", K(ret)); } else { filled_tx_scn = meta_array.at(0).sstable_meta_.basic_meta_.filled_tx_scn_; @@ -3941,10 +3941,10 @@ int ObLSBackupPrepareTask::prepare_meta_index_store_param_( return ret; } -int ObLSBackupPrepareTask::get_cur_ls_min_filled_tx_scn_(palf::SCN &min_filled_tx_scn) +int ObLSBackupPrepareTask::get_cur_ls_min_filled_tx_scn_(SCN &min_filled_tx_scn) { int ret = OB_SUCCESS; - min_filled_tx_scn = palf::SCN::max_scn(); + min_filled_tx_scn = SCN::max_scn(); ObLSTabletIterator iterator(ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US); storage::ObLSHandle ls_handle; storage::ObLS *ls = NULL; @@ -3962,7 +3962,7 @@ int ObLSBackupPrepareTask::get_cur_ls_min_filled_tx_scn_(palf::SCN &min_filled_t STORAGE_LOG(WARN, "build ls table iter failed.", KR(ret)); } else { while (OB_SUCC(iterator.get_next_tablet(tablet_handle))) { - palf::SCN tmp_filled_tx_scn = palf::SCN::max_scn(); + SCN tmp_filled_tx_scn = SCN::max_scn(); bool has_minor_sstable = false; if (OB_FAIL(get_tablet_min_filled_tx_scn_(tablet_handle, tmp_filled_tx_scn, has_minor_sstable))) { STORAGE_LOG(WARN, "get min end_log_ts from a single tablet failed.", KR(ret)); @@ -3981,11 +3981,11 @@ int ObLSBackupPrepareTask::get_cur_ls_min_filled_tx_scn_(palf::SCN &min_filled_t } int ObLSBackupPrepareTask::get_tablet_min_filled_tx_scn_( - ObTabletHandle &tablet_handle, palf::SCN &min_filled_tx_scn, bool &has_minor_sstable) + ObTabletHandle &tablet_handle, SCN &min_filled_tx_scn, bool &has_minor_sstable) { int ret = OB_SUCCESS; has_minor_sstable = false; - min_filled_tx_scn = palf::SCN::max_scn(); + min_filled_tx_scn = SCN::max_scn(); ObTablet *tablet = nullptr; if (OB_ISNULL(tablet = tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; @@ -4418,7 +4418,7 @@ ObLSBackupComplementLogTask::~ObLSBackupComplementLogTask() int ObLSBackupComplementLogTask::init(const ObBackupJobDesc &job_desc, const ObBackupDest &backup_dest, const uint64_t tenant_id, const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, - const palf::SCN &start_scn, const palf::SCN &end_scn, const int64_t turn_id, const int64_t retry_id, + const SCN &start_scn, const SCN &end_scn, const int64_t turn_id, const int64_t retry_id, const ObBackupReportCtx &report_ctx) { int ret = OB_SUCCESS; @@ -4514,8 +4514,8 @@ int ObLSBackupComplementLogTask::calc_backup_file_range_(common::ObIArray piece_list; @@ -4558,7 +4558,7 @@ int ObLSBackupComplementLogTask::check_pieces_continue_(const common::ObIArray &piece_list, const palf::SCN &start_scn, const palf::SCN &end_scn, + const common::ObIArray &piece_list, const SCN &start_scn, const SCN &end_scn, common::ObIArray &piece_file_list) { int ret = OB_SUCCESS; @@ -4647,7 +4647,7 @@ int ObLSBackupComplementLogTask::inner_get_piece_file_list_( } int ObLSBackupComplementLogTask::locate_archive_file_id_by_ts_(const uint64_t tenant_id, const int64_t round_id, - const int64_t piece_id, const palf::SCN &scn, const bool is_upper_bound, int64_t &file_id) + const int64_t piece_id, const SCN &scn, const bool is_upper_bound, int64_t &file_id) { int ret = OB_SUCCESS; // TODO(yangyi.yyy): duotian has provide this interface, wait merge code in 4.1 diff --git a/src/storage/backup/ob_backup_task.h b/src/storage/backup/ob_backup_task.h index 6bab98315..fe2763fb4 100644 --- a/src/storage/backup/ob_backup_task.h +++ b/src/storage/backup/ob_backup_task.h @@ -34,7 +34,7 @@ #include "storage/blocksstable/ob_logic_macro_id.h" namespace oceanbase { -namespace palf +namespace share { class SCN; } @@ -61,10 +61,10 @@ struct ObLSBackupDagNetInitParam : public share::ObIDagInitParam { int64_t turn_id_; int64_t retry_id_; ObBackupReportCtx report_ctx_; - palf::SCN start_scn_; // for backup meta + share::SCN start_scn_; // for backup meta share::ObBackupDataType backup_data_type_; // for build index - palf::SCN compl_start_scn_; // for complemnt log - palf::SCN compl_end_scn_; // for complemet log + share::SCN compl_start_scn_; // for complemnt log + share::SCN compl_end_scn_; // for complemet log }; struct ObLSBackupDagInitParam : public share::ObIDagInitParam { @@ -133,7 +133,7 @@ private: bool is_inited_; ObLSBackupDagInitParam param_; ObBackupReportCtx report_ctx_; - palf::SCN start_scn_; + share::SCN start_scn_; DISALLOW_COPY_AND_ASSIGN(ObLSBackupMetaDagNet); }; @@ -215,8 +215,8 @@ private: bool is_inited_; ObLSBackupDagInitParam param_; ObBackupReportCtx report_ctx_; - palf::SCN compl_start_scn_; - palf::SCN compl_end_scn_; + share::SCN compl_start_scn_; + share::SCN compl_end_scn_; DISALLOW_COPY_AND_ASSIGN(ObLSBackupComplementLogDagNet); }; @@ -236,7 +236,7 @@ class ObLSBackupMetaDag : public ObBackupDag { public: ObLSBackupMetaDag(); virtual ~ObLSBackupMetaDag(); - int init(const palf::SCN &start_scn, const ObLSBackupDagInitParam ¶m, const ObBackupReportCtx &report_ctx); + int init(const share::SCN &start_scn, const ObLSBackupDagInitParam ¶m, const ObBackupReportCtx &report_ctx); virtual int create_first_task() override; virtual bool operator==(const ObIDag &other) const override; virtual int fill_comment(char *buf, const int64_t buf_len) const override; @@ -247,7 +247,7 @@ public: private: bool is_inited_; - palf::SCN start_scn_; + share::SCN start_scn_; ObLSBackupDagInitParam param_; ObBackupReportCtx report_ctx_; DISALLOW_COPY_AND_ASSIGN(ObLSBackupMetaDag); @@ -390,7 +390,7 @@ public: virtual ~ObLSBackupComplementLogDag(); int init(const ObBackupJobDesc &job_desc, const share::ObBackupDest &backup_dest, const uint64_t tenant_id, const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, const int64_t turn_id, - const int64_t retry_id, const palf::SCN &start_scn, const palf::SCN &end_scn, const ObBackupReportCtx &report_ctx); + const int64_t retry_id, const share::SCN &start_scn, const share::SCN &end_scn, const ObBackupReportCtx &report_ctx); virtual int create_first_task() override; virtual int fill_comment(char *buf, const int64_t buf_len) const override; virtual int fill_dag_key(char *buf, const int64_t buf_len) const override; @@ -407,8 +407,8 @@ private: share::ObLSID ls_id_; int64_t turn_id_; int64_t retry_id_; - palf::SCN compl_start_scn_; - palf::SCN compl_end_scn_; + share::SCN compl_start_scn_; + share::SCN compl_end_scn_; ObBackupReportCtx report_ctx_; DISALLOW_COPY_AND_ASSIGN(ObLSBackupComplementLogDag); @@ -418,23 +418,23 @@ class ObLSBackupMetaTask : public share::ObITask { public: ObLSBackupMetaTask(); virtual ~ObLSBackupMetaTask(); - int init(const palf::SCN &start_scn, const ObLSBackupDagInitParam ¶m, const ObBackupReportCtx &report_ctx); + int init(const share::SCN &start_scn, const ObLSBackupDagInitParam ¶m, const ObBackupReportCtx &report_ctx); virtual int process() override; private: - int advance_checkpoint_by_flush_(const uint64_t tenant_id, const share::ObLSID &ls_id, const palf::SCN &start_scn); + int advance_checkpoint_by_flush_(const uint64_t tenant_id, const share::ObLSID &ls_id, const share::SCN &start_scn); int get_backup_meta_ctx_(const uint64_t tenant_id, const share::ObLSID &ls_id, ObBackupLSMetaInfo &ls_meta_info, common::ObArray &tablet_id_list); int backup_ls_meta_package_(const ObBackupLSMetaInfo &ls_meta_info); int backup_ls_tablet_list_( - const palf::SCN &scn, const share::ObLSID &ls, const common::ObIArray &tablet_id_list); - int build_backup_data_ls_tablet_desc_(const share::ObLSID &ls_id, const palf::SCN &scn, + const share::SCN &scn, const share::ObLSID &ls, const common::ObIArray &tablet_id_list); + int build_backup_data_ls_tablet_desc_(const share::ObLSID &ls_id, const share::SCN &scn, const common::ObIArray &tablet_id, share::ObBackupDataTabletToLSDesc &info); private: bool is_inited_; - palf::SCN start_scn_; + share::SCN start_scn_; ObLSBackupDagInitParam param_; ObBackupReportCtx report_ctx_; DISALLOW_COPY_AND_ASSIGN(ObLSBackupMetaTask); @@ -452,14 +452,14 @@ public: private: int may_need_advance_checkpoint_(); int check_tx_data_can_explain_user_data_(); - int get_backup_tx_data_table_filled_tx_scn_(palf::SCN &filled_tx_scn); + int get_backup_tx_data_table_filled_tx_scn_(share::SCN &filled_tx_scn); int prepare_meta_index_store_(ObBackupMetaIndexStore &meta_index_store); int get_sys_ls_retry_id_(int64_t &retry_id); int prepare_meta_index_store_param_(const int64_t retry_id, ObBackupIndexStoreParam ¶m); - int get_cur_ls_min_filled_tx_scn_(palf::SCN &min_filled_tx_scn); + int get_cur_ls_min_filled_tx_scn_(share::SCN &min_filled_tx_scn); int get_tablet_min_filled_tx_scn_(ObTabletHandle &tablet_handle, - palf::SCN &min_filled_tx_scn, bool &has_minor_sstable); - int fetch_backup_ls_meta_(int64_t &rebuild_seq, palf::SCN &clog_checkpoint_scn); + share::SCN &min_filled_tx_scn, bool &has_minor_sstable); + int fetch_backup_ls_meta_(int64_t &rebuild_seq, share::SCN &clog_checkpoint_scn); private: bool is_inited_; @@ -678,24 +678,24 @@ public: ObLSBackupComplementLogTask(); virtual ~ObLSBackupComplementLogTask(); int init(const ObBackupJobDesc &job_desc, const share::ObBackupDest &backup_dest, const uint64_t tenant_id, - const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, const palf::SCN &start_scn, - const palf::SCN &end_scn, const int64_t turn_id, const int64_t retry_id, const ObBackupReportCtx &report_ctx); + const share::ObBackupSetDesc &backup_set_desc, const share::ObLSID &ls_id, const share::SCN &start_scn, + const share::SCN &end_scn, const int64_t turn_id, const int64_t retry_id, const ObBackupReportCtx &report_ctx); virtual int process() override; private: int get_complement_log_dir_path_(share::ObBackupPath &backup_path); int calc_backup_file_range_(common::ObIArray &file_list); int check_pieces_continue_(const common::ObIArray &rounds); - int get_piece_id_by_ts_(const uint64_t tenant_id, const palf::SCN &scn, int64_t &piece_id); + int get_piece_id_by_ts_(const uint64_t tenant_id, const share::SCN &scn, int64_t &piece_id); int get_all_pieces_(const uint64_t tenant_id, const int64_t start_piece_id, const int64_t end_piece_id, common::ObArray &piece_list); int get_all_piece_file_list_(const uint64_t tenant_id, - const common::ObIArray &piece_list, const palf::SCN &start_scn, const palf::SCN &end_scn, + const common::ObIArray &piece_list, const share::SCN &start_scn, const share::SCN &end_scn, common::ObIArray &piece_file_list); int inner_get_piece_file_list_( const int64_t round_id, const int64_t piece_id, common::ObIArray &piece_file_list); int locate_archive_file_id_by_ts_(const uint64_t tenant_id, const int64_t round_id, const int64_t piece_id, - const palf::SCN &scn, const bool is_upper_bound, int64_t &file_id); + const share::SCN &scn, const bool is_upper_bound, int64_t &file_id); int filter_file_id_smaller_than_(const int64_t file_id, common::ObIArray &list); int filter_file_id_larger_than_(const int64_t file_id, common::ObIArray &list); int get_src_backup_piece_dir_(const int64_t round_id, const int64_t piece_id, share::ObBackupPath &backup_path); @@ -716,8 +716,8 @@ private: uint64_t tenant_id_; share::ObBackupSetDesc backup_set_desc_; share::ObLSID ls_id_; - palf::SCN compl_start_scn_; - palf::SCN compl_end_scn_; + share::SCN compl_start_scn_; + share::SCN compl_end_scn_; int64_t turn_id_; int64_t retry_id_; diff --git a/src/storage/backup/ob_backup_utils.cpp b/src/storage/backup/ob_backup_utils.cpp index 3021e9d3c..fc48e2c61 100644 --- a/src/storage/backup/ob_backup_utils.cpp +++ b/src/storage/backup/ob_backup_utils.cpp @@ -31,7 +31,7 @@ #include "storage/tx_storage/ob_ls_service.h" #include "storage/high_availability/ob_storage_ha_utils.h" #include "observer/ob_server_event_history_table_operator.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/blocksstable/ob_logic_macro_id.h" #include @@ -225,8 +225,8 @@ int ObBackupUtils::check_tablet_minor_sstable_validity_(const storage::ObTabletH ObTablet *tablet = NULL; ObITable *last_table_ptr = NULL; ObTabletID tablet_id; - palf::SCN start_scn = SCN::min_scn(); - palf::SCN clog_checkpoint_scn = SCN::min_scn(); + SCN start_scn = SCN::min_scn(); + SCN clog_checkpoint_scn = SCN::min_scn(); if (OB_ISNULL(tablet = tablet_handle.get_obj())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid tablet handle", K(ret), K(tablet_handle)); diff --git a/src/storage/blocksstable/encoding/ob_column_equal_decoder.cpp b/src/storage/blocksstable/encoding/ob_column_equal_decoder.cpp index f70f53334..acba9aecc 100644 --- a/src/storage/blocksstable/encoding/ob_column_equal_decoder.cpp +++ b/src/storage/blocksstable/encoding/ob_column_equal_decoder.cpp @@ -53,7 +53,8 @@ int ObColumnEqualDecoder::decode(ObColumnDecoderCtx &ctx, ObObj &cell, const int if (!has_exc(ctx)) { ref = -1; } else { - const ObObjTypeClass tc = ob_obj_type_class(ctx.obj_meta_.get_type()); + const ObObjType store_type = ctx.col_header_->get_store_obj_type(); + const ObObjTypeClass tc = ob_obj_type_class(store_type); switch (get_store_class_map()[tc]) { case ObUIntSC: case ObIntSC: { @@ -61,7 +62,7 @@ int ObColumnEqualDecoder::decode(ObColumnDecoderCtx &ctx, ObObj &cell, const int meta_header_->payload_, ctx.micro_block_header_->row_count_, ctx.is_bit_packing(), row_id, ctx.col_header_->length_ - sizeof(ObColumnEqualMetaHeader), - ref, cell, ctx.obj_meta_))) { + ref, cell, store_type))) { LOG_WARN("meta_reader_ read failed", K(ret), K(row_id), K(ctx)); } break; @@ -71,7 +72,7 @@ int ObColumnEqualDecoder::decode(ObColumnDecoderCtx &ctx, ObObj &cell, const int meta_header_->payload_, ctx.micro_block_header_->row_count_, ctx.is_bit_packing(), row_id, ctx.col_header_->length_ - sizeof(ObColumnEqualMetaHeader), - ref, cell, ctx.obj_meta_))) { + ref, cell, store_type))) { LOG_WARN("meta_reader_ read failed", K(ret), K(row_id), K(ctx)); } break; @@ -83,7 +84,7 @@ int ObColumnEqualDecoder::decode(ObColumnDecoderCtx &ctx, ObObj &cell, const int meta_header_->payload_, ctx.micro_block_header_->row_count_, ctx.is_bit_packing(), row_id, ctx.col_header_->length_ - sizeof(ObColumnEqualMetaHeader), - ref, cell, ctx.obj_meta_))) { + ref, cell, store_type))) { LOG_WARN("meta_reader_ read failed", K(ret), K(row_id), K(ctx)); } break; @@ -93,7 +94,7 @@ int ObColumnEqualDecoder::decode(ObColumnDecoderCtx &ctx, ObObj &cell, const int meta_header_->payload_, ctx.micro_block_header_->row_count_, ctx.is_bit_packing(), row_id, ctx.col_header_->length_ - sizeof(ObColumnEqualMetaHeader), - ref, cell, ctx.obj_meta_))) { + ref, cell, store_type))) { LOG_WARN("meta_reader_ read failed", K(ret), K(row_id), K(ctx)); } break; @@ -103,7 +104,7 @@ int ObColumnEqualDecoder::decode(ObColumnDecoderCtx &ctx, ObObj &cell, const int meta_header_->payload_, ctx.micro_block_header_->row_count_, ctx.is_bit_packing(), row_id, ctx.col_header_->length_ - sizeof(ObColumnEqualMetaHeader), - ref, cell, ctx.obj_meta_))) { + ref, cell, store_type))) { LOG_WARN("meta_reader_ read failed", K(ret), K(row_id), K(ctx)); } break; diff --git a/src/storage/blocksstable/encoding/ob_column_equal_decoder.h b/src/storage/blocksstable/encoding/ob_column_equal_decoder.h index f81beb21e..d9119ee42 100644 --- a/src/storage/blocksstable/encoding/ob_column_equal_decoder.h +++ b/src/storage/blocksstable/encoding/ob_column_equal_decoder.h @@ -34,9 +34,10 @@ public: ObColumnEqualDecoder(); virtual ~ObColumnEqualDecoder(); - OB_INLINE int init(const common::ObObjMeta &obj_meta, + OB_INLINE int init( const ObMicroBlockHeader µ_block_header, - const ObColumnHeader &column_header, const char *block_data); + const ObColumnHeader &column_header, + const char *block_data); void reset(); OB_INLINE void reuse(); virtual int decode(ObColumnDecoderCtx &ctx, common::ObObj &cell, const int64_t row_id, @@ -58,12 +59,12 @@ private: const ObColumnEqualMetaHeader *meta_header_; }; -OB_INLINE int ObColumnEqualDecoder::init(const common::ObObjMeta &obj_meta, +OB_INLINE int ObColumnEqualDecoder::init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *block_data) { - UNUSEDx(obj_meta, micro_block_header, column_header); + UNUSEDx(micro_block_header, column_header); int ret = common::OB_SUCCESS; // performance critical, don't check params if (inited_) { diff --git a/src/storage/blocksstable/encoding/ob_const_decoder.cpp b/src/storage/blocksstable/encoding/ob_const_decoder.cpp index cae528034..54d370667 100644 --- a/src/storage/blocksstable/encoding/ob_const_decoder.cpp +++ b/src/storage/blocksstable/encoding/ob_const_decoder.cpp @@ -178,7 +178,7 @@ int ObConstDecoder::batch_decode( } else if (OB_FAIL(extract_ref_and_null_count(row_ids, row_cap, datums, unused_null_cnt))) { LOG_WARN("Failed to extract refs",K(ret)); } else if (OB_FAIL(dict_decoder_.batch_decode_dict( - ctx.obj_meta_.get_type(), + ctx.col_header_->get_store_obj_type(), cell_datas, row_cap, ctx.col_header_->length_ - meta_header_->offset_, diff --git a/src/storage/blocksstable/encoding/ob_const_decoder.h b/src/storage/blocksstable/encoding/ob_const_decoder.h index 8a9b40c6b..bb487f921 100644 --- a/src/storage/blocksstable/encoding/ob_const_decoder.h +++ b/src/storage/blocksstable/encoding/ob_const_decoder.h @@ -36,7 +36,7 @@ public: } virtual ~ObConstDecoder() {} - OB_INLINE int init(const common::ObObjMeta &obj_meta, + OB_INLINE int init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *block_data); @@ -127,12 +127,12 @@ private: ObDictDecoder dict_decoder_; }; -OB_INLINE int ObConstDecoder::init(const common::ObObjMeta &obj_meta, +OB_INLINE int ObConstDecoder::init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta_data) { - UNUSEDx(micro_block_header, column_header); + UNUSEDx(micro_block_header); // performance critical, don't check params int ret = common::OB_SUCCESS; if (OB_UNLIKELY(is_inited())) { @@ -143,7 +143,7 @@ OB_INLINE int ObConstDecoder::init(const common::ObObjMeta &obj_meta, meta_header_ = reinterpret_cast(meta_data); const char *dict_data = meta_data + meta_header_->offset_; if (meta_header_->count_ > 0) { - if (OB_FAIL(dict_decoder_.init(obj_meta, dict_data))) { + if (OB_FAIL(dict_decoder_.init(column_header.get_store_obj_type(), dict_data))) { STORAGE_LOG(WARN, "failed to init dict decoder", K(ret), KP(dict_data)); meta_header_ = NULL; } diff --git a/src/storage/blocksstable/encoding/ob_dict_decoder.cpp b/src/storage/blocksstable/encoding/ob_dict_decoder.cpp index 7b5a27671..5464cc1da 100644 --- a/src/storage/blocksstable/encoding/ob_dict_decoder.cpp +++ b/src/storage/blocksstable/encoding/ob_dict_decoder.cpp @@ -189,17 +189,16 @@ bool init_dict_cmp_ref_funcs() bool dict_cmp_ref_funcs_inited = init_dict_cmp_ref_funcs(); -int ObDictDecoder::init(const ObObjMeta &obj_meta, const char *meta_header) +int ObDictDecoder::init(const common::ObObjType &store_obj_type, const char *meta_header) { - UNUSEDx(obj_meta); int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited())) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); } else { - store_class_ = get_store_class_map()[ob_obj_type_class(obj_meta.get_type())]; - if (ObIntTC == ob_obj_type_class(obj_meta.get_type())) { - int64_t type_store_size = get_type_size_map()[obj_meta.get_type()]; + store_class_ = get_store_class_map()[ob_obj_type_class(store_obj_type)]; + if (ObIntTC == ob_obj_type_class(store_obj_type)) { + int64_t type_store_size = get_type_size_map()[store_obj_type]; integer_mask_ = ~INTEGER_MASK_TABLE[type_store_size]; } else { integer_mask_ = 0; @@ -402,7 +401,8 @@ int ObDictDecoder::batch_decode( ret = OB_NOT_INIT; LOG_WARN("Not init", K(ret)); } else if (fast_decode_valid(ctx)) { - const ObObjTypeStoreClass store_class = get_store_class_map()[ctx.obj_meta_.get_type_class()]; + const ObObjType store_obj_type = ctx.col_header_->get_store_obj_type(); + const ObObjTypeStoreClass store_class = get_store_class_map()[ob_obj_type_class(store_obj_type)]; const char *ref_data = reinterpret_cast( const_cast(meta_header_)) + ctx.col_header_->length_; @@ -410,9 +410,9 @@ int ObDictDecoder::batch_decode( if (meta_header_->is_fix_length_dict()) { // Only need store_len for UIntSC/IntSC const int64_t store_size = meta_header_->data_size_; - const ObObjDatumMapType map_type = ObDatum::get_obj_datum_map_type(ctx.obj_meta_.get_type()); + const ObObjDatumMapType map_type = ObDatum::get_obj_datum_map_type(store_obj_type); bool read_as_signed_data = ObIntSC == store_class - && store_size == get_type_size_map()[ctx.obj_meta_.get_type()]; + && store_size == get_type_size_map()[store_obj_type]; const int64_t func_entry_store_size_idx = (store_class == ObIntSC || store_class == ObUIntSC) ? store_size : 0; dict_fix_batch_decode_func decode_func = dict_fix_batch_decode_funcs @@ -423,7 +423,7 @@ int ObDictDecoder::batch_decode( [get_store_class_tag_map()[store_class]]; decode_func(ref_data, base_data, store_size, meta_header_->count_, row_ids, row_cap, datums); LOG_DEBUG("[batch_decode] Run fix dict fast batch decode", - K(ret), K(store_class), K(store_size), K(map_type), K(ctx.obj_meta_.get_type()), + K(ret), K(store_class), K(store_size), K(map_type), K(store_obj_type), K(read_as_signed_data), K(func_entry_store_size_idx)); } else { dict_var_batch_decode_func decode_func = dict_var_batch_decode_funcs @@ -456,7 +456,7 @@ int ObDictDecoder::batch_decode( if (OB_FAIL(ret)) { } else if (OB_FAIL(batch_decode_dict( - ctx.obj_meta_.get_type(), + ctx.col_header_->get_store_obj_type(), cell_datas, row_cap, ctx.col_header_->length_, @@ -542,7 +542,8 @@ int ObDictDecoder::batch_decode_dict( bool ObDictDecoder::fast_decode_valid(const ObColumnDecoderCtx &ctx) const { bool valid = false; - const ObObjTypeStoreClass store_class = get_store_class_map()[ctx.obj_meta_.get_type_class()]; + const ObObjTypeStoreClass store_class = + get_store_class_map()[ob_obj_type_class(ctx.col_header_->get_store_obj_type())]; if (meta_header_->is_fix_length_dict()) { valid = !ctx.is_bit_packing() && meta_header_->row_ref_size_ <= 2 diff --git a/src/storage/blocksstable/encoding/ob_dict_decoder.h b/src/storage/blocksstable/encoding/ob_dict_decoder.h index 59a9fea64..231067421 100644 --- a/src/storage/blocksstable/encoding/ob_dict_decoder.h +++ b/src/storage/blocksstable/encoding/ob_dict_decoder.h @@ -60,11 +60,11 @@ public: {} virtual ~ObDictDecoder() {} - OB_INLINE int init(const common::ObObjMeta &obj_meta, + OB_INLINE int init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta); - int init(const common::ObObjMeta &obj_meta, const char *meta_header); + int init(const common::ObObjType &store_obj_type, const char *meta_header); virtual int decode(ObColumnDecoderCtx &ctx, common::ObObj &cell, const int64_t row_id, const ObBitStream &bs, const char *data, const int64_t len) const override; @@ -201,7 +201,7 @@ private: const char *var_data_; }; -OB_INLINE int ObDictDecoder::init(const common::ObObjMeta &obj_meta, +OB_INLINE int ObDictDecoder::init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta) @@ -213,10 +213,10 @@ OB_INLINE int ObDictDecoder::init(const common::ObObjMeta &obj_meta, ret = common::OB_INIT_TWICE; STORAGE_LOG(WARN, "init twice", K(ret)); } else { - const common::ObObjTypeClass type_class = ob_obj_type_class(obj_meta.get_type()); + const common::ObObjTypeClass type_class = ob_obj_type_class(column_header.get_store_obj_type()); store_class_ = get_store_class_map()[type_class]; if (common::ObIntTC == type_class) { - int64_t type_store_size = get_type_size_map()[obj_meta.get_type()]; + int64_t type_store_size = get_type_size_map()[column_header.get_store_obj_type()]; integer_mask_ = ~INTEGER_MASK_TABLE[type_store_size]; } else { integer_mask_ = 0; diff --git a/src/storage/blocksstable/encoding/ob_encoding_bitset.h b/src/storage/blocksstable/encoding/ob_encoding_bitset.h index 8ca795ad3..6e1b515e0 100644 --- a/src/storage/blocksstable/encoding/ob_encoding_bitset.h +++ b/src/storage/blocksstable/encoding/ob_encoding_bitset.h @@ -553,17 +553,17 @@ class ObBitMapMetaReader public: static int read(const char *buf, const int64_t row_count, const bool bit_packing, const int64_t row_id, const int64_t len, - int64_t &ref, common::ObObj &cell, const common::ObObjMeta type); + int64_t &ref, common::ObObj &cell, const common::ObObjType type); OB_INLINE static int read_exc_cell(const char *buf, const ObBitMapMetaHeader *meta, const bool bit_packing, const int64_t ref, const int64_t len, - common::ObObj &cell, const uint64_t integer_mask, const common::ObObjMeta type); + common::ObObj &cell, const uint64_t integer_mask, const common::ObObjType type); }; template int ObBitMapMetaReader::read(const char *buf, const int64_t row_count, const bool bit_packing, const int64_t row_id, const int64_t len, - int64_t &ref, common::ObObj &cell, const common::ObObjMeta type) + int64_t &ref, common::ObObj &cell, const common::ObObjType type) { int ret = common::OB_SUCCESS; if (OB_ISNULL(buf) @@ -592,8 +592,8 @@ int ObBitMapMetaReader::read(const char *buf, const int64_t row_coun } // read data uint64_t integer_mask = 0; - if (common::ObIntTC == ob_obj_type_class(type.get_type())) { - integer_mask = ~INTEGER_MASK_TABLE[get_type_size_map()[type.get_type()]]; + if (common::ObIntTC == ob_obj_type_class(type)) { + integer_mask = ~INTEGER_MASK_TABLE[get_type_size_map()[type]]; } if (OB_FAIL(ret)) { } else if (STORED_NOT_EXT != ext_val) { @@ -610,7 +610,7 @@ int ObBitMapMetaReader::read(const char *buf, const int64_t row_coun template OB_INLINE int ObBitMapMetaReader::read_exc_cell(const char *buf, const ObBitMapMetaHeader *meta, const bool bit_packing, const int64_t ref, - const int64_t len, common::ObObj &cell, const uint64_t integer_mask, const common::ObObjMeta type) + const int64_t len, common::ObObj &cell, const uint64_t integer_mask, const common::ObObjType type) { UNUSED(type); int ret = common::OB_SUCCESS; @@ -637,7 +637,7 @@ OB_INLINE int ObBitMapMetaReader::read_exc_cell(const char *buf, template <> OB_INLINE int ObBitMapMetaReader::read_exc_cell(const char *buf, const ObBitMapMetaHeader *meta, const bool bit_packing, const int64_t ref, - const int64_t len, common::ObObj &cell, const uint64_t integer_mask, const common::ObObjMeta type) + const int64_t len, common::ObObj &cell, const uint64_t integer_mask, const common::ObObjType type) { int ret = common::OB_SUCCESS; UNUSEDx(bit_packing, integer_mask, type); @@ -664,7 +664,7 @@ OB_INLINE int ObBitMapMetaReader::read_exc_cell(const char *buf, template <> OB_INLINE int ObBitMapMetaReader::read_exc_cell(const char *buf, const ObBitMapMetaHeader *meta, const bool bit_packing, const int64_t ref, - const int64_t len, common::ObObj &cell, const uint64_t integer_mask, const common::ObObjMeta type) + const int64_t len, common::ObObj &cell, const uint64_t integer_mask, const common::ObObjType type) { int ret = common::OB_SUCCESS; UNUSEDx(bit_packing, integer_mask, type); @@ -698,7 +698,7 @@ OB_INLINE int ObBitMapMetaReader::read_exc_cell(const char *buf, template <> OB_INLINE int ObBitMapMetaReader::read_exc_cell(const char *buf, const ObBitMapMetaHeader *meta, const bool bit_packing, const int64_t ref, - const int64_t len, common::ObObj &cell, const uint64_t integer_mask, const common::ObObjMeta type) + const int64_t len, common::ObObj &cell, const uint64_t integer_mask, const common::ObObjType type) { int ret = common::OB_SUCCESS; UNUSEDx(bit_packing, integer_mask); @@ -719,12 +719,14 @@ OB_INLINE int ObBitMapMetaReader::read_exc_cell(const char *buf, if (OB_SUCC(ret)) { ObStorageDatum tmp_datum; // TODO: remove + ObObjMeta tmp_obj_meta; + tmp_obj_meta.set_type(type); - ObObjDatumMapType datum_type = ObDatum::get_obj_datum_map_type(type.get_type()); + ObObjDatumMapType datum_type = ObDatum::get_obj_datum_map_type(type); const uint32_t size = ObDatum::get_reserved_size(datum_type); MEMCPY(const_cast(tmp_datum.ptr_), buf + meta->data_offset_ + offset, size); tmp_datum.len_ = size; - if (OB_FAIL(tmp_datum.to_obj(cell, type))) { + if (OB_FAIL(tmp_datum.to_obj(cell, tmp_obj_meta))) { STORAGE_LOG(WARN, "Failed to read datum", K(ret)); } } @@ -734,7 +736,7 @@ OB_INLINE int ObBitMapMetaReader::read_exc_cell(const char *buf, template <> OB_INLINE int ObBitMapMetaReader::read_exc_cell(const char *buf, const ObBitMapMetaHeader *meta, const bool bit_packing, const int64_t ref, - const int64_t len, common::ObObj &cell, const uint64_t integer_mask, const common::ObObjMeta type) + const int64_t len, common::ObObj &cell, const uint64_t integer_mask, const common::ObObjType type) { int ret = common::OB_SUCCESS; UNUSEDx(bit_packing, integer_mask); @@ -755,12 +757,14 @@ OB_INLINE int ObBitMapMetaReader::read_exc_cell(const char *buf, if (OB_SUCC(ret)) { ObStorageDatum tmp_datum; // TODO: remove + ObObjMeta tmp_obj_meta; + tmp_obj_meta.set_type(type); - ObObjDatumMapType datum_type = ObDatum::get_obj_datum_map_type(type.get_type()); + ObObjDatumMapType datum_type = ObDatum::get_obj_datum_map_type(type); const uint32_t size = ObDatum::get_reserved_size(datum_type); MEMCPY(const_cast(tmp_datum.ptr_), buf + meta->data_offset_ + offset, size); tmp_datum.len_ = size; - if (OB_FAIL(tmp_datum.to_obj(cell, type))) { + if (OB_FAIL(tmp_datum.to_obj(cell, tmp_obj_meta))) { STORAGE_LOG(WARN, "Failed to read datum", K(ret)); } } diff --git a/src/storage/blocksstable/encoding/ob_hex_string_decoder.h b/src/storage/blocksstable/encoding/ob_hex_string_decoder.h index 3a59b186b..ed8604a0c 100644 --- a/src/storage/blocksstable/encoding/ob_hex_string_decoder.h +++ b/src/storage/blocksstable/encoding/ob_hex_string_decoder.h @@ -34,7 +34,7 @@ public: ObHexStringDecoder(); ~ObHexStringDecoder(); - OB_INLINE int init(const common::ObObjMeta &obj_meta, + OB_INLINE int init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta); @@ -69,13 +69,13 @@ private: const ObHexStringHeader *header_; }; -OB_INLINE int ObHexStringDecoder::init(const common::ObObjMeta &obj_meta, +OB_INLINE int ObHexStringDecoder::init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta) { // performance critical, don't check params, already checked upper layer - UNUSEDx(obj_meta, micro_block_header); + UNUSEDx(micro_block_header); int ret = common::OB_SUCCESS; if (is_inited()) { ret = common::OB_INIT_TWICE; diff --git a/src/storage/blocksstable/encoding/ob_integer_base_diff_decoder.h b/src/storage/blocksstable/encoding/ob_integer_base_diff_decoder.h index f4ccf80a6..96f28ccf3 100644 --- a/src/storage/blocksstable/encoding/ob_integer_base_diff_decoder.h +++ b/src/storage/blocksstable/encoding/ob_integer_base_diff_decoder.h @@ -35,7 +35,7 @@ public: {} virtual ~ObIntegerBaseDiffDecoder() {} - OB_INLINE int init(const common::ObObjMeta &obj_metan, + OB_INLINE int init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta); @@ -126,7 +126,7 @@ private: uint64_t base_; }; -OB_INLINE int ObIntegerBaseDiffDecoder::init(const common::ObObjMeta &obj_meta, +OB_INLINE int ObIntegerBaseDiffDecoder::init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta) { @@ -137,11 +137,11 @@ OB_INLINE int ObIntegerBaseDiffDecoder::init(const common::ObObjMeta &obj_meta, ret = common::OB_INIT_TWICE; STORAGE_LOG(WARN, "init twice", K(ret)); } else { - const int64_t store_size = get_type_size_map()[obj_meta.get_type()]; - ObObjTypeStoreClass sc = get_store_class_map()[ob_obj_type_class(obj_meta.get_type())]; + const int64_t store_size = get_type_size_map()[column_header.get_store_obj_type()]; + ObObjTypeStoreClass sc = get_store_class_map()[ob_obj_type_class(column_header.get_store_obj_type())]; if (ObIntSC != sc && ObUIntSC != sc) { ret = common::OB_INNER_STAT_ERROR; - STORAGE_LOG(WARN, "not supported store class", K(ret), K(obj_meta), K(sc)); + STORAGE_LOG(WARN, "not supported store class", K(ret), K(column_header), K(sc)); } else { meta += column_header.offset_; header_ = reinterpret_cast(meta); diff --git a/src/storage/blocksstable/encoding/ob_inter_column_substring_decoder.cpp b/src/storage/blocksstable/encoding/ob_inter_column_substring_decoder.cpp index d7bfab85a..e63778afa 100644 --- a/src/storage/blocksstable/encoding/ob_inter_column_substring_decoder.cpp +++ b/src/storage/blocksstable/encoding/ob_inter_column_substring_decoder.cpp @@ -55,7 +55,7 @@ int ObInterColSubStrDecoder::decode(ObColumnDecoderCtx &ctx, ObObj &cell, const ctx.micro_block_header_->row_count_, ctx.is_bit_packing(), row_id, ctx.col_header_->length_ - sizeof(ObInterColSubStrMetaHeader), - ref, cell, ctx.obj_meta_))) { + ref, cell, ctx.col_header_->get_store_obj_type()))) { LOG_WARN("meta_reader_ read failed", K(ret), K(row_id)); } } diff --git a/src/storage/blocksstable/encoding/ob_inter_column_substring_decoder.h b/src/storage/blocksstable/encoding/ob_inter_column_substring_decoder.h index 6377926fb..6e4a8f02d 100644 --- a/src/storage/blocksstable/encoding/ob_inter_column_substring_decoder.h +++ b/src/storage/blocksstable/encoding/ob_inter_column_substring_decoder.h @@ -34,7 +34,7 @@ public: ObInterColSubStrDecoder(); virtual ~ObInterColSubStrDecoder(); - OB_INLINE int init(const common::ObObjMeta &obj_meta, + OB_INLINE int init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta); @@ -61,13 +61,13 @@ private: const ObInterColSubStrMetaHeader *meta_header_; }; -OB_INLINE int ObInterColSubStrDecoder::init(const common::ObObjMeta &obj_meta, +OB_INLINE int ObInterColSubStrDecoder::init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta) { int ret = common::OB_SUCCESS; - UNUSEDx(obj_meta, micro_block_header, column_header); + UNUSEDx(micro_block_header, column_header); // performance critical, don't check params, already checked upper layer if (OB_UNLIKELY(is_inited())) { ret = common::OB_INIT_TWICE; diff --git a/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp b/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp index 0f00e56a9..45329a781 100644 --- a/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp +++ b/src/storage/blocksstable/encoding/ob_micro_block_decoder.cpp @@ -541,7 +541,7 @@ int ObIEncodeBlockReader::add_decoder(const int64_t store_idx, const ObObjMeta & } else { const ObIColumnDecoder *decoder = nullptr; const ObIColumnDecoder *ref_decoder = nullptr; - if (OB_FAIL(acquire(obj_meta, store_idx, decoder))) { + if (OB_FAIL(acquire(store_idx, decoder))) { LOG_WARN("acquire decoder failed", K(ret), K(obj_meta), K(store_idx)); } else { dest.decoder_ = decoder; @@ -557,7 +557,7 @@ int ObIEncodeBlockReader::add_decoder(const int64_t store_idx, const ObObjMeta & } } if (OB_SUCC(ret) && ref_col_idx >= 0) { - if (OB_FAIL(acquire(obj_meta, ref_col_idx, ref_decoder))) { + if (OB_FAIL(acquire(ref_col_idx, ref_decoder))) { LOG_WARN("acquire decoder failed", K(ret), K(obj_meta), K(ref_col_idx)); } else { dest.ctx_->ref_decoder_ = ref_decoder; @@ -573,15 +573,14 @@ int ObIEncodeBlockReader::add_decoder(const int64_t store_idx, const ObObjMeta & // called before inited // performance critical, do not check parameters -int ObIEncodeBlockReader::acquire(const ObObjMeta &obj_meta, const int64_t store_idx, - const ObIColumnDecoder *&decoder) +int ObIEncodeBlockReader::acquire(const int64_t store_idx, const ObIColumnDecoder *&decoder) { int ret = OB_SUCCESS; if (NULL != cached_decocer_ && store_idx < cached_decocer_->count_) { decoder = &cached_decocer_->at(store_idx); } else { if (OB_FAIL(ObIEncodeBlockReader::acquire_funcs_[col_header_[store_idx].type_]( - *allocator_, obj_meta, *header_, col_header_[store_idx], meta_data_, decoder))) { + *allocator_, *header_, col_header_[store_idx], meta_data_, decoder))) { LOG_WARN("acquire decoder failed", K(ret), K(store_idx), K(col_header_[store_idx])); } else { need_release_decoders_[need_release_decoder_cnt_++] = decoder; @@ -604,7 +603,6 @@ int ObIEncodeBlockReader::setup_row(const uint64_t row_id, int64_t &row_len, con template int ObIEncodeBlockReader::acquire_decoder(ObDecoderAllocator &allocator, - const ObObjMeta &obj_meta, const ObMicroBlockHeader &header, const ObColumnHeader &col_header, const char *meta_data, @@ -614,7 +612,10 @@ int ObIEncodeBlockReader::acquire_decoder(ObDecoderAllocator &allocator, Decoder *d = nullptr; if (OB_FAIL(allocator.alloc(d))) { LOG_WARN("alloc failed", K(ret)); - } else if (OB_FAIL(d->init(obj_meta, header, col_header, meta_data))) { + } else if (OB_UNLIKELY(!col_header.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid column header", K(ret), K(header), K(col_header)); + } else if (OB_FAIL(d->init(header, col_header, meta_data))) { LOG_WARN("init decoder failed", K(ret)); } else { decoder = d; @@ -886,15 +887,17 @@ ObMicroBlockDecoder::~ObMicroBlockDecoder() template int ObMicroBlockDecoder::acquire( - Allocator &allocator, const ObObjMeta &obj_meta, - const ObMicroBlockHeader &header, const ObColumnHeader &col_header, - const char *meta_data, const ObIColumnDecoder *&decoder) + Allocator &allocator, + const ObMicroBlockHeader &header, + const ObColumnHeader &col_header, + const char *meta_data, + const ObIColumnDecoder *&decoder) { int ret = OB_SUCCESS; decoder = NULL; - if (!obj_meta.is_valid()) { + if (OB_UNLIKELY(!col_header.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid obj_meta", K(ret), K(obj_meta)); + LOG_WARN("invalid column header", K(ret), K(col_header)); } else { switch (col_header.type_) { @@ -902,7 +905,7 @@ int ObMicroBlockDecoder::acquire( ObRawDecoder *d = NULL; if (OB_FAIL(allocator.alloc(d))) { LOG_WARN("alloc failed", K(ret)); - } else if (OB_FAIL(d->init(obj_meta, header, col_header, meta_data))) { + } else if (OB_FAIL(d->init(header, col_header, meta_data))) { LOG_WARN("init raw decoder failed", K(ret)); } else { decoder = d; @@ -914,7 +917,7 @@ int ObMicroBlockDecoder::acquire( if (OB_FAIL(allocator.alloc(d))) { LOG_WARN("alloc failed", K(ret)); } else { - if (OB_FAIL(d->init(obj_meta, header, col_header, meta_data))) { + if (OB_FAIL(d->init(header, col_header, meta_data))) { LOG_WARN("init dict decoder failed", K(ret)); } else { decoder = d; @@ -927,7 +930,7 @@ int ObMicroBlockDecoder::acquire( if (OB_FAIL(allocator.alloc(d))) { LOG_WARN("alloc failed", K(ret)); } else { - if (OB_FAIL(d->init(obj_meta, header, col_header, meta_data))) { + if (OB_FAIL(d->init(header, col_header, meta_data))) { LOG_WARN("init integer base diff decoder failed", K(ret)); } else { decoder = d; @@ -939,7 +942,7 @@ int ObMicroBlockDecoder::acquire( ObStringDiffDecoder *d = NULL; if (OB_FAIL(allocator.alloc(d))) { LOG_WARN("alloc failed", K(ret)); - } else if (OB_FAIL(d->init(obj_meta, header, col_header, meta_data))) { + } else if (OB_FAIL(d->init(header, col_header, meta_data))) { LOG_WARN("init string diff decoder failed", K(ret)); } else { decoder = d; @@ -950,7 +953,7 @@ int ObMicroBlockDecoder::acquire( ObHexStringDecoder *d = NULL; if (OB_FAIL(allocator.alloc(d))) { LOG_WARN("alloc failed", K(ret)); - } else if (OB_FAIL(d->init(obj_meta, header, col_header, meta_data))) { + } else if (OB_FAIL(d->init(header, col_header, meta_data))) { LOG_WARN("init hex packing decoder failed", K(ret)); } else { decoder = d; @@ -961,7 +964,7 @@ int ObMicroBlockDecoder::acquire( ObRLEDecoder *d = NULL; if (OB_FAIL(allocator.alloc(d))) { LOG_WARN("alloc failed", K(ret)); - } else if (OB_FAIL(d->init(obj_meta, header, col_header, meta_data))) { + } else if (OB_FAIL(d->init(header, col_header, meta_data))) { LOG_WARN("init rle decoder failed", K(ret)); } else { decoder = d; @@ -972,7 +975,7 @@ int ObMicroBlockDecoder::acquire( ObConstDecoder *d = NULL; if (OB_FAIL(allocator.alloc(d))) { LOG_WARN("alloc failed", K(ret)); - } else if (OB_FAIL(d->init(obj_meta, header, col_header, meta_data))) { + } else if (OB_FAIL(d->init(header, col_header, meta_data))) { LOG_WARN("init const decoder failed", K(ret)); } else { decoder = d; @@ -983,7 +986,7 @@ int ObMicroBlockDecoder::acquire( ObStringPrefixDecoder *d = NULL; if (OB_FAIL(allocator.alloc(d))) { LOG_WARN("alloc failed", K(ret)); - } else if (OB_FAIL(d->init(obj_meta, header, col_header, meta_data))) { + } else if (OB_FAIL(d->init(header, col_header, meta_data))) { LOG_WARN("init string prefix decoder failed", K(ret)); } else { decoder = d; @@ -994,7 +997,7 @@ int ObMicroBlockDecoder::acquire( ObColumnEqualDecoder *d = NULL; if (OB_FAIL(allocator.alloc(d))) { LOG_WARN("alloc failed", K(ret)); - } else if (OB_FAIL(d->init(obj_meta, header, col_header, meta_data))) { + } else if (OB_FAIL(d->init(header, col_header, meta_data))) { LOG_WARN("init column equal decoder failed", K(ret)); } else { decoder = d; @@ -1005,7 +1008,7 @@ int ObMicroBlockDecoder::acquire( ObInterColSubStrDecoder *d = NULL; if (OB_FAIL(allocator.alloc(d))) { LOG_WARN("alloc failed", K(ret)); - } else if (OB_FAIL(d->init(obj_meta, header, col_header, meta_data))) { + } else if (OB_FAIL(d->init(header, col_header, meta_data))) { LOG_WARN("init column substr decoder failed", K(ret)); } else { decoder = d; @@ -1026,17 +1029,13 @@ int ObMicroBlockDecoder::acquire( // called before inited // performance critical, do not check parameters -int ObMicroBlockDecoder::acquire( - const ObObjMeta &obj_meta, - const int64_t store_idx, - const ObIColumnDecoder *&decoder) +int ObMicroBlockDecoder::acquire(const int64_t store_idx, const ObIColumnDecoder *&decoder) { int ret = OB_SUCCESS; if (NULL != cached_decoder_ && store_idx < cached_decoder_->count_) { decoder = &cached_decoder_->at(store_idx); } else { - if (OB_FAIL(acquire(*allocator_, obj_meta, *header_, - col_header_[store_idx], meta_data_, decoder))) { + if (OB_FAIL(acquire(*allocator_, *header_, col_header_[store_idx], meta_data_, decoder))) { LOG_WARN("acquire decoder failed", K(ret), K(store_idx), "column_header", col_header_[store_idx]); } else if (OB_FAIL(need_release_decoders_.push_back(decoder))) { @@ -1140,7 +1139,7 @@ int ObMicroBlockDecoder::add_decoder(const int64_t store_idx, const ObObjMeta &o dest.ctx_ = &none_exist_column_decoder_ctx_; } else { const ObIColumnDecoder *decoder = NULL; - if (OB_FAIL(acquire(obj_meta, store_idx, decoder))) { + if (OB_FAIL(acquire(store_idx, decoder))) { LOG_WARN("acquire decoder failed", K(ret), K(obj_meta), K(store_idx)); } else { dest.decoder_ = decoder; @@ -1157,7 +1156,7 @@ int ObMicroBlockDecoder::add_decoder(const int64_t store_idx, const ObObjMeta &o } if (OB_SUCC(ret) && ref_col_idx >= 0) { - if (OB_FAIL(acquire(obj_meta, ref_col_idx, decoder))) { + if (OB_FAIL(acquire(ref_col_idx, decoder))) { LOG_WARN("acquire decoder failed", K(ret), K(obj_meta), K(ref_col_idx)); } else { dest.ctx_->ref_decoder_ = decoder; @@ -1751,8 +1750,7 @@ int ObMicroBlockDecoder::cache_decoders( for (int64_t i = 0; OB_SUCC(ret) && i < h->count_; ++i) { const ObIColumnDecoder *d = nullptr; int64_t ref_col_idx = -1; - ObObjMeta obj_meta = full_schema_cols.at(i).col_type_; - if (OB_FAIL(acquire(allocator, obj_meta, *header, col_header[i], meta_data, d))) { + if (OB_FAIL(acquire(allocator, *header, col_header[i], meta_data, d))) { LOG_WARN("acquire allocator failed", K(ret), "micro_block_header", *header, "col_header", col_header[i]); } else if (OB_FAIL(d->get_ref_col_idx(ref_col_idx))) { diff --git a/src/storage/blocksstable/encoding/ob_micro_block_decoder.h b/src/storage/blocksstable/encoding/ob_micro_block_decoder.h index 25fe9a8c7..3333bdc71 100644 --- a/src/storage/blocksstable/encoding/ob_micro_block_decoder.h +++ b/src/storage/blocksstable/encoding/ob_micro_block_decoder.h @@ -75,7 +75,6 @@ public: class ObIEncodeBlockReader { typedef int (*decode_acquire_func)(ObDecoderAllocator &allocator, - const ObObjMeta &obj_meta, const ObMicroBlockHeader &header, const ObColumnHeader &col_header, const char *meta_data, @@ -98,8 +97,7 @@ protected: int add_decoder(const int64_t store_idx, const common::ObObjMeta &obj_meta, ObColumnDecoder &dest); void free_decoders(); void release(const ObIColumnDecoder *decoder); - int acquire(const common::ObObjMeta &obj_meta, int64_t store_idx, - const ObIColumnDecoder *&decoder); + int acquire(const int64_t store_idx, const ObIColumnDecoder *&decoder); int setup_row(const uint64_t row_id, int64_t &row_len, const char *&row_data); protected: static const int64_t DEFAULT_DECODER_CNT = 16; @@ -134,11 +132,10 @@ protected: static ObColumnDecoderCtx none_exist_column_decoder_ctx_; template static int acquire_decoder(ObDecoderAllocator &allocator, - const ObObjMeta &obj_meta, - const ObMicroBlockHeader &header, - const ObColumnHeader &col_header, - const char *meta_data, - const ObIColumnDecoder *&decoder); + const ObMicroBlockHeader &header, + const ObColumnHeader &col_header, + const char *meta_data, + const ObIColumnDecoder *&decoder); static decode_acquire_func acquire_funcs_[ObColumnHeader::MAX_TYPE]; }; @@ -316,11 +313,13 @@ private: const char *&meta_data, const char *block, const int64_t block_size); template - static int acquire(Allocator &allocator, const common::ObObjMeta &obj_meta, - const ObMicroBlockHeader &header, const ObColumnHeader &col_header, - const char *meta_data, const ObIColumnDecoder *&decoder); - int acquire(const common::ObObjMeta &obj_meta, int64_t store_idx, + static int acquire( + Allocator &allocator, + const ObMicroBlockHeader &header, + const ObColumnHeader &col_header, + const char *meta_data, const ObIColumnDecoder *&decoder); + int acquire(const int64_t store_idx, const ObIColumnDecoder *&decoder); void release(const ObIColumnDecoder *decoder); private: diff --git a/src/storage/blocksstable/encoding/ob_raw_decoder.cpp b/src/storage/blocksstable/encoding/ob_raw_decoder.cpp index a9f93ae79..e29173cb9 100644 --- a/src/storage/blocksstable/encoding/ob_raw_decoder.cpp +++ b/src/storage/blocksstable/encoding/ob_raw_decoder.cpp @@ -355,14 +355,15 @@ int ObRawDecoder::batch_decode( LOG_WARN("Raw decoder not inited", K(ret)); } else if (fast_decode_valid(ctx)) { // Optimized decode for byte-packing data - const ObObjTypeStoreClass store_class = get_store_class_map()[ctx.obj_meta_.get_type_class()]; + const ObObjType store_type = ctx.col_header_->get_store_obj_type(); + const ObObjTypeStoreClass store_class = get_store_class_map()[ob_obj_type_class(store_type)]; if (ctx.is_fix_length()) { - const ObObjDatumMapType map_type = ObDatum::get_obj_datum_map_type(ctx.obj_meta_.get_type()); + const ObObjDatumMapType map_type = ObDatum::get_obj_datum_map_type(store_type); // Only need store_len for UIntSC/IntSC uint32_t store_len = ctx.col_header_->length_ > 8 ? 0 : ctx.col_header_->length_; raw_fix_batch_decode_func decode_func = raw_fix_batch_decode_funcs [ObIntSC == store_class - && ctx.col_header_->length_ == get_type_size_map()[ctx.obj_meta_.get_type()]] + && ctx.col_header_->length_ == get_type_size_map()[store_type]] [get_value_len_tag_map()[store_len]] [get_value_len_tag_map()[get_datum_store_len(map_type)]] [get_store_class_tag_map()[store_class]]; @@ -396,7 +397,8 @@ bool ObRawDecoder::fast_decode_valid(const ObColumnDecoderCtx &ctx) const bool valid = false; const ObColumnHeader *col_header = ctx.col_header_; const ObMicroBlockHeader *block_header = ctx.micro_block_header_; - const ObObjTypeStoreClass store_class = get_store_class_map()[ctx.obj_meta_.get_type_class()]; + const ObObjTypeStoreClass store_class = + get_store_class_map()[ob_obj_type_class(ctx.col_header_->get_store_obj_type())]; if (col_header->is_fix_length()) { valid = !col_header->is_bit_packing() && !col_header->has_extend_value() @@ -452,7 +454,7 @@ int ObRawDecoder::batch_decode_general( uint32_t datum_len = 0; uint64_t value = 0; if (OB_FAIL(get_uint_data_datum_len( - ObDatum::get_obj_datum_map_type(ctx.obj_meta_.get_type()), + ObDatum::get_obj_datum_map_type(ctx.col_header_->get_store_obj_type()), datum_len))) { LOG_WARN("Failed to get datum len for int data", K(ret)); } @@ -498,7 +500,7 @@ int ObRawDecoder::batch_decode_general( if (OB_FAIL(ret)) { } else if (OB_FAIL(batch_load_data_to_datum( - ctx.obj_meta_.get_type(), + ctx.col_header_->get_store_obj_type(), cell_datas, row_cap, integer_mask_, @@ -612,7 +614,7 @@ bool ObRawDecoder::fast_filter_valid( } } if (valid) { - switch (ctx.obj_meta_.get_type_class()) { + switch (ob_obj_type_class(ctx.col_header_->get_store_obj_type())) { case ObIntTC: case ObDateTimeTC: case ObDateTC: @@ -730,7 +732,7 @@ int ObRawDecoder::fast_comparison_operator( ret = OB_INVALID_ARGUMENT; LOG_WARN("Filter pushdown operator: Invalid argument", K(ret), K(col_ctx), K(fix_len_tag)); } else { - const int64_t type_store_size = get_type_size_map()[col_ctx.obj_meta_.get_type()]; + const int64_t type_store_size = get_type_size_map()[col_ctx.col_header_->get_store_obj_type()]; const uint64_t node_value = filter.get_objs().at(0).v_.uint64_; const sql::ObWhiteFilterOperatorType &op_type = filter.get_op_type(); bool exceed_stored_value_range = ~INTEGER_MASK_TABLE[col_ctx.col_header_->length_] diff --git a/src/storage/blocksstable/encoding/ob_raw_decoder.h b/src/storage/blocksstable/encoding/ob_raw_decoder.h index adcedfac0..cd37a24bd 100644 --- a/src/storage/blocksstable/encoding/ob_raw_decoder.h +++ b/src/storage/blocksstable/encoding/ob_raw_decoder.h @@ -60,7 +60,7 @@ public: : store_class_(ObExtendSC), integer_mask_(0), meta_data_(nullptr), is_out_row_column_(false) {} virtual ~ObRawDecoder() {} - OB_INLINE int init(const common::ObObjMeta &obj_meta, + OB_INLINE int init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta); @@ -308,18 +308,18 @@ int ObRawDecoder::batch_locate_cell_data( return ret; } -OB_INLINE int ObRawDecoder::init(const common::ObObjMeta &obj_meta, +OB_INLINE int ObRawDecoder::init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta) { - UNUSED(micro_block_header); + UNUSEDx(micro_block_header); int ret = common::OB_SUCCESS; if (is_inited()) { ret = common::OB_INIT_TWICE; STORAGE_LOG(WARN, "init twice", K(ret)); } else { - const ObObjType store_type = column_header.is_out_row() ? ObVarcharType : obj_meta.get_type(); + const ObObjType store_type = column_header.is_out_row() ? ObVarcharType : column_header.get_store_obj_type(); const common::ObObjTypeClass type_class = ob_obj_type_class(store_type); store_class_ = get_store_class_map()[type_class]; if (common::ObIntTC == type_class) { diff --git a/src/storage/blocksstable/encoding/ob_rle_decoder.cpp b/src/storage/blocksstable/encoding/ob_rle_decoder.cpp index 1e12e4be8..70964e99d 100644 --- a/src/storage/blocksstable/encoding/ob_rle_decoder.cpp +++ b/src/storage/blocksstable/encoding/ob_rle_decoder.cpp @@ -91,7 +91,7 @@ int ObRLEDecoder::batch_decode( } else if (OB_FAIL(extract_ref_and_null_count(row_ids, row_cap, datums, unused_null_cnt))) { LOG_WARN("Failed to extract refs",K(ret)); } else if (OB_FAIL(dict_decoder_.batch_decode_dict( - ctx.obj_meta_.get_type(), + ctx.col_header_->get_store_obj_type(), cell_datas, row_cap, ctx.col_header_->length_ - meta_header_->offset_, diff --git a/src/storage/blocksstable/encoding/ob_rle_decoder.h b/src/storage/blocksstable/encoding/ob_rle_decoder.h index f8c615e7e..71fb5ad33 100644 --- a/src/storage/blocksstable/encoding/ob_rle_decoder.h +++ b/src/storage/blocksstable/encoding/ob_rle_decoder.h @@ -38,7 +38,7 @@ public: } virtual ~ObRLEDecoder() {} - OB_INLINE int init(const common::ObObjMeta &obj_meta, + OB_INLINE int init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *block_data); @@ -134,12 +134,12 @@ private: ObDictDecoder dict_decoder_; }; -OB_INLINE int ObRLEDecoder::init(const common::ObObjMeta &obj_meta, +OB_INLINE int ObRLEDecoder::init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *block_data) { - UNUSEDx(micro_block_header, column_header); + UNUSEDx(micro_block_header); // performance critical, don't check params int ret = common::OB_SUCCESS; if (OB_UNLIKELY(is_inited())) { @@ -151,7 +151,7 @@ OB_INLINE int ObRLEDecoder::init(const common::ObObjMeta &obj_meta, const char *dict_meta_header = block_data + column_header.offset_ + meta_header_->offset_; - if (OB_FAIL(dict_decoder_.init(obj_meta, dict_meta_header))) { + if (OB_FAIL(dict_decoder_.init(column_header.get_store_obj_type(), dict_meta_header))) { STORAGE_LOG(WARN, "failed to init dict decoder", K(ret), KP(dict_meta_header)); meta_header_ = NULL; } else { diff --git a/src/storage/blocksstable/encoding/ob_string_diff_decoder.h b/src/storage/blocksstable/encoding/ob_string_diff_decoder.h index 322721553..d6de0c833 100644 --- a/src/storage/blocksstable/encoding/ob_string_diff_decoder.h +++ b/src/storage/blocksstable/encoding/ob_string_diff_decoder.h @@ -34,7 +34,7 @@ public: ObStringDiffDecoder(); virtual ~ObStringDiffDecoder(); - OB_INLINE int init(const common::ObObjMeta &obj_metan, + OB_INLINE int init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta); @@ -61,12 +61,12 @@ private: const ObStringDiffHeader *header_; }; -OB_INLINE int ObStringDiffDecoder::init(const common::ObObjMeta &obj_meta, +OB_INLINE int ObStringDiffDecoder::init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta) { - UNUSEDx(obj_meta, micro_block_header, column_header); + UNUSEDx(micro_block_header, column_header); // performance critical, don't check params, already checked upper layer int ret = common::OB_SUCCESS; if (is_inited()) { diff --git a/src/storage/blocksstable/encoding/ob_string_prefix_decoder.h b/src/storage/blocksstable/encoding/ob_string_prefix_decoder.h index c11a633f9..af8c2487a 100644 --- a/src/storage/blocksstable/encoding/ob_string_prefix_decoder.h +++ b/src/storage/blocksstable/encoding/ob_string_prefix_decoder.h @@ -35,7 +35,7 @@ public: {} virtual ~ObStringPrefixDecoder(); - OB_INLINE int init(const common::ObObjMeta &obj_meta, + OB_INLINE int init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta); @@ -67,12 +67,12 @@ private: const char *meta_data_; }; -OB_INLINE int ObStringPrefixDecoder::init(const common::ObObjMeta &obj_meta, +OB_INLINE int ObStringPrefixDecoder::init( const ObMicroBlockHeader µ_block_header, const ObColumnHeader &column_header, const char *meta) { - UNUSEDx(obj_meta, micro_block_header, column_header); + UNUSEDx(micro_block_header, column_header); // performance critical, don't check params, already checked upper layer int ret = common::OB_SUCCESS; if (OB_UNLIKELY(is_inited())) { diff --git a/src/storage/blocksstable/ob_block_sstable_struct.cpp b/src/storage/blocksstable/ob_block_sstable_struct.cpp index bf0592732..11042ae2f 100644 --- a/src/storage/blocksstable/ob_block_sstable_struct.cpp +++ b/src/storage/blocksstable/ob_block_sstable_struct.cpp @@ -18,7 +18,7 @@ #include "lib/utility/ob_serialization_helper.h" #include "lib/utility/serialization.h" #include "lib/utility/utility.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "ob_block_manager.h" #include "ob_data_buffer.h" #include "share/config/ob_server_config.h" diff --git a/src/storage/blocksstable/ob_block_sstable_struct.h b/src/storage/blocksstable/ob_block_sstable_struct.h index d91458119..7ba391dd2 100644 --- a/src/storage/blocksstable/ob_block_sstable_struct.h +++ b/src/storage/blocksstable/ob_block_sstable_struct.h @@ -30,7 +30,7 @@ #include "storage/ob_i_store.h" #include "storage/ob_i_table.h" #include "storage/blocksstable/ob_logic_macro_id.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -1159,7 +1159,7 @@ public: ObString data_buffer_; ObDDLMacroBlockType block_type_; ObLogicMacroBlockId logic_id_; - palf::SCN start_scn_; + share::SCN start_scn_; }; }//end namespace blocksstable diff --git a/src/storage/blocksstable/ob_macro_block.h b/src/storage/blocksstable/ob_macro_block.h index ef033ee5c..e759f3ab1 100644 --- a/src/storage/blocksstable/ob_macro_block.h +++ b/src/storage/blocksstable/ob_macro_block.h @@ -21,7 +21,7 @@ #include "ob_sstable_meta.h" #include "share/ob_encryption_util.h" #include "storage/blocksstable/ob_macro_block_meta.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { namespace storage { @@ -63,7 +63,7 @@ struct ObDataStoreDesc ObSSTableIndexBuilder *sstable_index_builder_; ObCompressorType compressor_type_; int64_t snapshot_version_; - palf::SCN end_scn_; + share::SCN end_scn_; int64_t progressive_merge_round_; int64_t encrypt_id_; bool need_prebuild_bloomfilter_; diff --git a/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp b/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp index 6a01a7fd1..9d9d4e3a6 100644 --- a/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp +++ b/src/storage/blocksstable/ob_micro_block_row_lock_checker.cpp @@ -17,6 +17,7 @@ namespace oceanbase { namespace blocksstable { +using namespace share; ObMicroBlockRowLockChecker::ObMicroBlockRowLockChecker(common::ObIAllocator &allocator) : ObMicroBlockRowScanner(allocator), @@ -76,7 +77,7 @@ int ObMicroBlockRowLockChecker::get_next_row(const ObDatumRow *&row) STORAGE_LOG(DEBUG, "check row lock", K(ret), KPC_(range), K(read_trans_id), K(trans_id), K(sql_sequence), KPC_(lock_state)); - if (palf::SCN::min_scn() != lock_state_->trans_version_ || // trans is commit + if (SCN::min_scn() != lock_state_->trans_version_ || // trans is commit lock_state_->is_locked_) { break; } diff --git a/src/storage/blocksstable/ob_micro_block_row_scanner.cpp b/src/storage/blocksstable/ob_micro_block_row_scanner.cpp index 15ca49443..7b3394b22 100644 --- a/src/storage/blocksstable/ob_micro_block_row_scanner.cpp +++ b/src/storage/blocksstable/ob_micro_block_row_scanner.cpp @@ -24,7 +24,7 @@ namespace oceanbase { using namespace storage; -using namespace palf; +using namespace share; namespace blocksstable { diff --git a/src/storage/blocksstable/ob_sstable.cpp b/src/storage/blocksstable/ob_sstable.cpp index 2ca4731d1..a856f5e27 100644 --- a/src/storage/blocksstable/ob_sstable.cpp +++ b/src/storage/blocksstable/ob_sstable.cpp @@ -34,6 +34,7 @@ namespace oceanbase { using namespace common::hash; using namespace storage; +using namespace share; namespace blocksstable { @@ -664,7 +665,7 @@ int ObSSTable::check_row_locked(ObStoreCtx &ctx, ObSSTableRowLockChecker *row_checker = NULL; const int64_t read_snapshot = ctx.mvcc_acc_ctx_.get_snapshot_version().get_val_for_tx(); ObArenaAllocator allocator(ObModIds::OB_STORE_ROW_LOCK_CHECKER); - lock_state.trans_version_ = palf::SCN::min_scn(); + lock_state.trans_version_ = SCN::min_scn(); lock_state.is_locked_ = false; if (OB_UNLIKELY(!is_valid())) { diff --git a/src/storage/blocksstable/ob_sstable.h b/src/storage/blocksstable/ob_sstable.h index 0a9b23df1..126a91f54 100644 --- a/src/storage/blocksstable/ob_sstable.h +++ b/src/storage/blocksstable/ob_sstable.h @@ -15,7 +15,7 @@ #include "storage/blocksstable/ob_sstable_meta.h" #include "storage/blocksstable/ob_macro_block_meta.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "ob_datum_range.h" namespace oceanbase @@ -121,7 +121,7 @@ public: { return meta_.basic_meta_.recycle_version_; } - palf::SCN get_filled_tx_scn() const + share::SCN get_filled_tx_scn() const { return meta_.basic_meta_.filled_tx_scn_; } diff --git a/src/storage/blocksstable/ob_sstable_meta.cpp b/src/storage/blocksstable/ob_sstable_meta.cpp index 7538fc01e..f08a392e1 100644 --- a/src/storage/blocksstable/ob_sstable_meta.cpp +++ b/src/storage/blocksstable/ob_sstable_meta.cpp @@ -21,6 +21,7 @@ using namespace oceanbase::common; using namespace oceanbase::storage; +using namespace oceanbase::share; using namespace oceanbase::share::schema; namespace oceanbase @@ -51,7 +52,7 @@ ObSSTableBasicMeta::ObSSTableBasicMeta() max_merged_trans_version_(0), recycle_version_(0), ddl_scn_(), - filled_tx_scn_(palf::SCN::min_scn()), + filled_tx_scn_(SCN::min_scn()), data_index_tree_height_(0), table_mode_(), status_(0), diff --git a/src/storage/blocksstable/ob_sstable_meta.h b/src/storage/blocksstable/ob_sstable_meta.h index 9b5b9b3f6..3de1e0656 100644 --- a/src/storage/blocksstable/ob_sstable_meta.h +++ b/src/storage/blocksstable/ob_sstable_meta.h @@ -17,7 +17,7 @@ #include "storage/ob_storage_schema.h" #include "storage/ob_i_table.h" #include "storage/blocksstable/ob_sstable_meta_info.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -57,9 +57,9 @@ public: } OB_INLINE int64_t get_upper_trans_version() const { return ATOMIC_LOAD(&upper_trans_version_); } OB_INLINE int64_t get_max_merged_trans_version() const { return max_merged_trans_version_; } - OB_INLINE palf::SCN get_ddl_scn() const { return ddl_scn_; } + OB_INLINE share::SCN get_ddl_scn() const { return ddl_scn_; } OB_INLINE int64_t get_create_snapshot_version() const { return create_snapshot_version_; } - OB_INLINE palf::SCN get_filled_tx_scn() const { return filled_tx_scn_; } + OB_INLINE share::SCN get_filled_tx_scn() const { return filled_tx_scn_; } OB_INLINE int16_t get_data_index_tree_height() const { return data_index_tree_height_; } int set_upper_trans_version(const int64_t upper_trans_version); @@ -103,8 +103,8 @@ public: int64_t max_merged_trans_version_; // recycle_version only avaliable for minor sstable, recored recycled multi version start int64_t recycle_version_; - palf::SCN ddl_scn_; // only used in DDL SSTable, all MB in DDL SSTable should have the same scn(start_scn) - palf::SCN filled_tx_scn_; // only for rebuild + share::SCN ddl_scn_; // only used in DDL SSTable, all MB in DDL SSTable should have the same scn(start_scn) + share::SCN filled_tx_scn_; // only for rebuild int16_t data_index_tree_height_; share::schema::ObTableMode table_mode_; uint8_t status_; diff --git a/src/storage/blocksstable/ob_sstable_printer.cpp b/src/storage/blocksstable/ob_sstable_printer.cpp index 18ec92277..e629e1a25 100644 --- a/src/storage/blocksstable/ob_sstable_printer.cpp +++ b/src/storage/blocksstable/ob_sstable_printer.cpp @@ -530,6 +530,7 @@ void ObSSTablePrinter::print_encoding_column_header(const ObColumnHeader *col_he print_line("is bit packing", col_header->is_bit_packing()); print_line("is last var field", col_header->is_last_var_field()); print_line("extend value index", col_header->extend_value_index_); + print_line("store object type", col_header->obj_type_); print_line("offset", col_header->offset_); print_line("length", col_header->length_); print_end_line(); diff --git a/src/storage/checkpoint/ob_checkpoint_executor.cpp b/src/storage/checkpoint/ob_checkpoint_executor.cpp index 730b54325..13027d3cb 100644 --- a/src/storage/checkpoint/ob_checkpoint_executor.cpp +++ b/src/storage/checkpoint/ob_checkpoint_executor.cpp @@ -21,7 +21,7 @@ namespace oceanbase { -using namespace palf; +using namespace share; using namespace logservice; namespace storage { @@ -266,7 +266,7 @@ bool ObCheckpointExecutor::need_flush() { int ret = OB_SUCCESS; bool need_flush = false; - palf::SCN end_scn; + SCN end_scn; if (OB_FAIL(loghandler_->get_end_scn(end_scn))) { STORAGE_LOG(WARN, "get_end_scn failed", K(ret)); } else if (end_scn.convert_to_ts() - ls_->get_clog_checkpoint_scn().convert_to_ts() @@ -290,7 +290,7 @@ bool ObCheckpointExecutor::is_wait_advance_checkpoint() return ATOMIC_LOAD(&wait_advance_checkpoint_); } -void ObCheckpointExecutor::set_wait_advance_checkpoint(palf::SCN &checkpoint_scn) +void ObCheckpointExecutor::set_wait_advance_checkpoint(SCN &checkpoint_scn) { ObSpinLockGuard guard(lock_); if (checkpoint_scn == ls_->get_clog_checkpoint_scn()) { diff --git a/src/storage/checkpoint/ob_checkpoint_executor.h b/src/storage/checkpoint/ob_checkpoint_executor.h index 58fd13a4a..56fb4c67d 100644 --- a/src/storage/checkpoint/ob_checkpoint_executor.h +++ b/src/storage/checkpoint/ob_checkpoint_executor.h @@ -16,15 +16,10 @@ #include "lib/lock/ob_spin_lock.h" #include "logservice/ob_log_base_type.h" #include "logservice/ob_log_handler.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { - -namespace palf -{ -class SCN; -} namespace storage { class ObLS; @@ -33,7 +28,7 @@ namespace checkpoint struct ObCheckpointVTInfo { - palf::SCN rec_scn; + share::SCN rec_scn; int service_type; TO_STRING_KV( @@ -63,7 +58,7 @@ public: // the service will flush and advance checkpoint // after flush, checkpoint_scn will be equal or greater than recycle_scn - int advance_checkpoint_by_flush(palf::SCN recycle_scn); + int advance_checkpoint_by_flush(share::SCN recycle_scn); // for __all_virtual_checkpoint int get_checkpoint_info(ObIArray &checkpoint_array); @@ -73,7 +68,7 @@ public: bool is_wait_advance_checkpoint(); - void set_wait_advance_checkpoint(palf::SCN &checkpoint_scn); + void set_wait_advance_checkpoint(share::SCN &checkpoint_scn); int64_t get_cannot_recycle_log_size(); diff --git a/src/storage/checkpoint/ob_common_checkpoint.h b/src/storage/checkpoint/ob_common_checkpoint.h index e36682e02..6b52182f8 100644 --- a/src/storage/checkpoint/ob_common_checkpoint.h +++ b/src/storage/checkpoint/ob_common_checkpoint.h @@ -16,7 +16,7 @@ #include #include "share/ob_ls_id.h" #include "common/ob_tablet_id.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -66,7 +66,7 @@ int common_checkpoint_type_to_string(const ObCommonCheckpointType common_checkpo struct ObCommonCheckpointVTInfo { ObTabletID tablet_id; - palf::SCN rec_scn; + share::SCN rec_scn; int checkpoint_type; bool is_flushing; @@ -91,8 +91,8 @@ inline bool is_valid_log_base_type(const ObCommonCheckpointType &type) class ObCommonCheckpoint { public: - virtual palf::SCN get_rec_scn() = 0; - virtual int flush(palf::SCN recycle_scn, bool need_freeze = true) = 0; + virtual share::SCN get_rec_scn() = 0; + virtual int flush(share::SCN recycle_scn, bool need_freeze = true) = 0; virtual ObTabletID get_tablet_id() const = 0; virtual bool is_flushing() const = 0; diff --git a/src/storage/checkpoint/ob_data_checkpoint.cpp b/src/storage/checkpoint/ob_data_checkpoint.cpp index 35bdda86f..d56710313 100644 --- a/src/storage/checkpoint/ob_data_checkpoint.cpp +++ b/src/storage/checkpoint/ob_data_checkpoint.cpp @@ -16,7 +16,7 @@ namespace oceanbase { -using namespace palf; +using namespace share; namespace storage { namespace checkpoint @@ -235,7 +235,7 @@ int ObDataCheckpoint::flush(SCN recycle_scn, bool need_freeze) return ret; } -int ObDataCheckpoint::ls_freeze(palf::SCN rec_scn) +int ObDataCheckpoint::ls_freeze(SCN rec_scn) { int ret = OB_SUCCESS; ObCheckPointService *checkpoint_srv = MTL(ObCheckPointService *); @@ -290,7 +290,7 @@ void ObDataCheckpoint::print_list_(ObCheckpointDList &list) } } -void ObDataCheckpoint::road_to_flush(palf::SCN rec_scn) +void ObDataCheckpoint::road_to_flush(SCN rec_scn) { if (OB_UNLIKELY(!is_inited_)) { STORAGE_LOG(WARN, "ObDataCheckpoint not init", K(is_inited_)); diff --git a/src/storage/checkpoint/ob_data_checkpoint.h b/src/storage/checkpoint/ob_data_checkpoint.h index 82ce8df97..4698c752a 100644 --- a/src/storage/checkpoint/ob_data_checkpoint.h +++ b/src/storage/checkpoint/ob_data_checkpoint.h @@ -17,7 +17,7 @@ #include "storage/checkpoint/ob_common_checkpoint.h" #include "lib/lock/ob_spin_lock.h" #include "storage/checkpoint/ob_freeze_checkpoint.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -40,8 +40,8 @@ struct ObCheckpointDList int unlink(ObFreezeCheckpoint *ob_freeze_checkpoint); int insert(ObFreezeCheckpoint *ob_freeze_checkpoint, bool ordered = true); void get_iterator(ObCheckpointIterator &iterator); - palf::SCN get_min_rec_scn_in_list(bool ordered = true); - ObFreezeCheckpoint *get_first_greater(const palf::SCN rec_scn); + share::SCN get_min_rec_scn_in_list(bool ordered = true); + ObFreezeCheckpoint *get_first_greater(const share::SCN rec_scn); int get_freezecheckpoint_info( ObIArray &freeze_checkpoint_array); @@ -88,16 +88,16 @@ public: static const uint64_t LS_DATA_CHECKPOINT_TABLET_ID = 40000; int init(ObLS *ls); int safe_to_destroy(); - palf::SCN get_rec_scn(); + share::SCN get_rec_scn(); // if min_rec_scn <= the input rec_scn // logstream freeze - int flush(palf::SCN recycle_scn, bool need_freeze = true); + int flush(share::SCN recycle_scn, bool need_freeze = true); // if min_rec_scn <= the input rec_scn // add ls_freeze task // logstream freeze optimization - int ls_freeze(palf::SCN rec_scn); + int ls_freeze(share::SCN rec_scn); // logstream_freeze schedule and minor merge schedule - void road_to_flush(palf::SCN rec_scn); + void road_to_flush(share::SCN rec_scn); // ObFreezeCheckpoint register into ObDataCheckpoint int add_to_new_create(ObFreezeCheckpoint *ob_freeze_checkpoint); // remove from prepare_list when finish minor_merge diff --git a/src/storage/checkpoint/ob_freeze_checkpoint.h b/src/storage/checkpoint/ob_freeze_checkpoint.h index 2e33f3d38..8682e3ab6 100644 --- a/src/storage/checkpoint/ob_freeze_checkpoint.h +++ b/src/storage/checkpoint/ob_freeze_checkpoint.h @@ -17,7 +17,7 @@ #include "lib/utility/ob_print_utils.h" #include "share/ob_ls_id.h" #include "common/ob_tablet_id.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -61,7 +61,7 @@ int freeze_checkpoint_location_to_string(const ObFreezeCheckpointLocation locati struct ObFreezeCheckpointVTInfo { ObTabletID tablet_id; - palf::SCN rec_scn; + share::SCN rec_scn; ObFreezeCheckpointLocation location; bool rec_scn_is_stable; @@ -87,7 +87,7 @@ public: ObFreezeCheckpoint() : location_(OUT), data_checkpoint_(nullptr) {} virtual ~ObFreezeCheckpoint() {} virtual void remove_from_data_checkpoint(bool need_lock_data_checkpoint = true); - virtual palf::SCN get_rec_scn() = 0; + virtual share::SCN get_rec_scn() = 0; virtual int flush(share::ObLSID ls_id) = 0; // judge rec_scn of the checkpoint unit won't get smaller // by comparing with max_consequent_callbacked_scn diff --git a/src/storage/compaction/ob_compaction_diagnose.cpp b/src/storage/compaction/ob_compaction_diagnose.cpp index cdb56dda4..7f4eb849a 100644 --- a/src/storage/compaction/ob_compaction_diagnose.cpp +++ b/src/storage/compaction/ob_compaction_diagnose.cpp @@ -32,6 +32,13 @@ using namespace share; namespace compaction { +int64_t ObScheduleSuspectInfo::hash() const +{ + int64_t hash_value = ObMergeDagHash::inner_hash(); + hash_value = common::murmurhash(&tenant_id_, sizeof(tenant_id_), hash_value); + return hash_value; +} + bool ObScheduleSuspectInfo::is_valid() const { bool bret = true; @@ -45,6 +52,7 @@ bool ObScheduleSuspectInfo::is_valid() const ObScheduleSuspectInfo & ObScheduleSuspectInfo::operator = (const ObScheduleSuspectInfo &other) { + tenant_id_ = other.tenant_id_; merge_type_ = other.merge_type_; ls_id_ = other.ls_id_; tablet_id_ = other.tablet_id_; @@ -320,13 +328,14 @@ int ObCompactionDiagnoseMgr::get_suspect_info( ObScheduleSuspectInfo &ret_info) { int ret = OB_SUCCESS; - compaction::ObMergeDagHash dag_hash; - dag_hash.merge_type_ = merge_type; - dag_hash.ls_id_ = ls_id; - dag_hash.tablet_id_ = tablet_id; - if (OB_FAIL(ObScheduleSuspectInfoMgr::get_instance().get_suspect_info(dag_hash.inner_hash(), ret_info))) { + ObScheduleSuspectInfo input_info; + input_info.tenant_id_ = MTL_ID(); + input_info.merge_type_ = merge_type; + input_info.ls_id_ = ls_id; + input_info.tablet_id_ = tablet_id; + if (OB_FAIL(ObScheduleSuspectInfoMgr::get_instance().get_suspect_info(input_info.hash(), ret_info))) { if (OB_HASH_NOT_EXIST != ret) { - LOG_WARN("failed to get suspect info", K(ret), K(dag_hash)); + LOG_WARN("failed to get suspect info", K(ret), K(input_info)); } } else if (ret_info.add_time_ + SUSPECT_INFO_WARNING_THRESHOLD < ObTimeUtility::fast_current_time()) { ret = OB_ENTRY_NOT_EXIST; @@ -413,7 +422,7 @@ int ObCompactionDiagnoseMgr::diagnose_tenant_tablet() SET_DIAGNOSE_INFO( info_array_[idx_++], MINI_MERGE, - MTL_ID(), + ret_info.tenant_id_, ls_id, ObTabletID(INT64_MAX), ObCompactionDiagnoseInfo::DIA_STATUS_FAILED, diff --git a/src/storage/compaction/ob_compaction_diagnose.h b/src/storage/compaction/ob_compaction_diagnose.h index 241e2c3b7..b11c33fec 100644 --- a/src/storage/compaction/ob_compaction_diagnose.h +++ b/src/storage/compaction/ob_compaction_diagnose.h @@ -32,13 +32,16 @@ struct ObScheduleSuspectInfo : public common::ObDLinkBase { ObScheduleSuspectInfo() : ObMergeDagHash(), + tenant_id_(OB_INVALID_ID), add_time_(0), suspect_info_("\0") {} + int64_t hash() const; bool is_valid() const; ObScheduleSuspectInfo & operator = (const ObScheduleSuspectInfo &other); - TO_STRING_KV(K_(merge_type), K_(ls_id), K_(tablet_id), K_(add_time), K_(suspect_info)); + TO_STRING_KV(K_(tenant_id), K_(merge_type), K_(ls_id), K_(tablet_id), K_(add_time), K_(suspect_info)); + int64_t tenant_id_; int64_t add_time_; char suspect_info_[common::OB_DIAGNOSE_INFO_LENGTH]; }; @@ -212,6 +215,7 @@ private: int64_t __pos = 0; \ int ret = OB_SUCCESS; \ compaction::ObScheduleSuspectInfo info; \ + info.tenant_id_ = MTL_ID(); \ info.merge_type_ = type; \ info.ls_id_ = ls_id; \ info.tablet_id_ = tablet_id; \ @@ -226,7 +230,7 @@ private: buf[__pos++] = '.'; \ } \ SIMPLE_TO_STRING_##n \ - if (OB_FAIL(ObScheduleSuspectInfoMgr::get_instance().add_suspect_info(info.inner_hash(), info))) { \ + if (OB_FAIL(ObScheduleSuspectInfoMgr::get_instance().add_suspect_info(info.hash(), info))) { \ STORAGE_LOG(WARN, "failed to add suspect info", K(ret), K(info)); \ } else { \ STORAGE_LOG(DEBUG, "success to add suspect info", K(ret), K(info)); \ diff --git a/src/storage/compaction/ob_i_compaction_filter.cpp b/src/storage/compaction/ob_i_compaction_filter.cpp index 4f3767eef..59a4d97d9 100644 --- a/src/storage/compaction/ob_i_compaction_filter.cpp +++ b/src/storage/compaction/ob_i_compaction_filter.cpp @@ -19,6 +19,7 @@ namespace oceanbase { using namespace common; using namespace storage; +using namespace share; namespace compaction { @@ -78,7 +79,7 @@ int64_t ObICompactionFilter::ObFilterStatistics::to_string(char *buf, const int6 return pos; } -int ObTransStatusFilter::init(const palf::SCN &filter_val, const int64_t filter_col_idx) +int ObTransStatusFilter::init(const SCN &filter_val, const int64_t filter_col_idx) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!filter_val.is_valid() || filter_val.is_min() || filter_col_idx < 0)) { @@ -113,12 +114,12 @@ int ObTransStatusFilter::filter( LOG_WARN("uncommitted row in trans state table or multi version row", K(ret), K(row)); } else { const int64_t trans_end_scn_storage_val = row.storage_datums_[filter_col_idx_].get_int(); - palf::SCN trans_end_scn; + SCN trans_end_scn; if (OB_FAIL(trans_end_scn.convert_for_tx(trans_end_scn_storage_val))) { LOG_WARN("failed to convert for tx", K(ret), K(trans_end_scn_storage_val)); } else if (trans_end_scn <= filter_val_) { filter_ret = FILTER_RET_REMOVE; - max_filtered_end_scn_ = palf::SCN::max(max_filtered_end_scn_, trans_end_scn); + max_filtered_end_scn_ = SCN::max(max_filtered_end_scn_, trans_end_scn); LOG_DEBUG("filter row", K(ret), K(row), K(filter_val_)); } else { filter_ret = FILTER_RET_NOT_CHANGE; diff --git a/src/storage/compaction/ob_i_compaction_filter.h b/src/storage/compaction/ob_i_compaction_filter.h index 4081c0e3f..acc0b7340 100644 --- a/src/storage/compaction/ob_i_compaction_filter.h +++ b/src/storage/compaction/ob_i_compaction_filter.h @@ -15,7 +15,7 @@ #include "lib/utility/ob_print_utils.h" #include "share/schema/ob_table_param.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { namespace blocksstable @@ -91,7 +91,7 @@ public: max_filtered_end_scn_.set_min(); } ~ObTransStatusFilter() {} - int init(const palf::SCN &filter_val, const int64_t filter_col_idx); + int init(const share::SCN &filter_val, const int64_t filter_col_idx); OB_INLINE virtual void reset() override { ObICompactionFilter::reset(); @@ -107,14 +107,14 @@ public: K_(filter_col_idx), K_(max_filtered_end_scn)); public: - palf::SCN get_max_filtered_end_scn() { return max_filtered_end_scn_; } - palf::SCN get_recycle_scn() { return filter_val_; } + share::SCN get_max_filtered_end_scn() { return max_filtered_end_scn_; } + share::SCN get_recycle_scn() { return filter_val_; } private: bool is_inited_; - palf::SCN filter_val_; + share::SCN filter_val_; int64_t filter_col_idx_; - palf::SCN max_filtered_end_scn_; + share::SCN max_filtered_end_scn_; }; } // namespace compaction diff --git a/src/storage/compaction/ob_partition_merge_iter.cpp b/src/storage/compaction/ob_partition_merge_iter.cpp index 9ea787469..449adc294 100644 --- a/src/storage/compaction/ob_partition_merge_iter.cpp +++ b/src/storage/compaction/ob_partition_merge_iter.cpp @@ -20,6 +20,7 @@ namespace oceanbase { using namespace share::schema; +using namespace share; using namespace common; using namespace memtable; using namespace storage; @@ -94,7 +95,7 @@ int ObPartitionMergeIter::init_query_base_params(const ObMergeParameter &merge_p { int ret = OB_SUCCESS; ObSSTable *sstable = nullptr; - palf::SCN snapshot_version; + SCN snapshot_version; if (OB_UNLIKELY(nullptr == column_ids_ || nullptr == table_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected null column_ids", K(ret), KPC(this)); diff --git a/src/storage/compaction/ob_partition_merge_policy.cpp b/src/storage/compaction/ob_partition_merge_policy.cpp index 4dcceece0..64b18de39 100644 --- a/src/storage/compaction/ob_partition_merge_policy.cpp +++ b/src/storage/compaction/ob_partition_merge_policy.cpp @@ -23,7 +23,7 @@ #include "storage/compaction/ob_compaction_diagnose.h" #include "storage/compaction/ob_tenant_compaction_progress.h" #include "observer/omt/ob_tenant_config_mgr.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" using namespace oceanbase; using namespace common; @@ -136,7 +136,7 @@ int ObPartitionMergePolicy::find_mini_merge_tables( // Keep max_snapshot_version currently because major merge must be done step by step int64_t max_snapshot_version = freeze_info.next.freeze_scn.get_val_for_tx(); ObITable *last_table = tablet.get_table_store().get_minor_sstables().get_boundary_table(true/*last*/); - const palf::SCN last_minor_scn = nullptr == last_table ? tablet.get_clog_checkpoint_scn() : last_table->get_end_scn(); + const SCN last_minor_scn = nullptr == last_table ? tablet.get_clog_checkpoint_scn() : last_table->get_end_scn(); // Freezing in the restart phase may not satisfy end >= last_max_sstable, // so the memtable cannot be filtered by scn diff --git a/src/storage/compaction/ob_tablet_merge_ctx.cpp b/src/storage/compaction/ob_tablet_merge_ctx.cpp index d7e789ce3..6533c9eef 100644 --- a/src/storage/compaction/ob_tablet_merge_ctx.cpp +++ b/src/storage/compaction/ob_tablet_merge_ctx.cpp @@ -27,6 +27,7 @@ namespace oceanbase { +using namespace share; namespace compaction { @@ -276,8 +277,8 @@ int ObTabletMergeInfo::record_start_tx_scn_for_tx_data(const ObTabletMergeCtx &c if (OB_NOT_NULL(compaction_filter_)) { // if compaction_filter is valid, update filled_tx_log_ts if recycled some tx data - palf::SCN recycled_scn; - if (compaction_filter_->get_max_filtered_end_scn() > palf::SCN::min_scn()) { + SCN recycled_scn; + if (compaction_filter_->get_max_filtered_end_scn() > SCN::min_scn()) { recycled_scn = compaction_filter_->get_max_filtered_end_scn(); } else { recycled_scn = compaction_filter_->get_recycle_scn(); @@ -767,7 +768,7 @@ int ObTabletMergeCtx::update_tablet_or_release_memtable(const ObGetMergeTablesRe update_table_store_flag = true; } - const palf::SCN release_memtable_scn = get_merge_table_result.scn_range_.end_scn_; + const SCN release_memtable_scn = get_merge_table_result.scn_range_.end_scn_; if (OB_FAIL(ret)) { } else if (update_table_store_flag && OB_FAIL(update_tablet_directly(get_merge_table_result))) { LOG_WARN("failed to update tablet directly", K(ret), K(get_merge_table_result), K(update_table_store_flag)); @@ -784,7 +785,7 @@ int ObTabletMergeCtx::update_tablet_directly(const ObGetMergeTablesResult &get_m int ret = OB_SUCCESS; const int64_t rebuild_seq = ls_handle_.get_ls()->get_rebuild_seq(); scn_range_ = get_merge_table_result.scn_range_; - palf::SCN clog_checkpoint_scn = get_merge_table_result.scn_range_.end_scn_; + SCN clog_checkpoint_scn = get_merge_table_result.scn_range_.end_scn_; ObTableHandleV2 empty_table_handle; ObUpdateTableStoreParam param( diff --git a/src/storage/compaction/ob_tablet_merge_ctx.h b/src/storage/compaction/ob_tablet_merge_ctx.h index 9708dadb7..56a12de8d 100644 --- a/src/storage/compaction/ob_tablet_merge_ctx.h +++ b/src/storage/compaction/ob_tablet_merge_ctx.h @@ -21,7 +21,7 @@ #include "storage/compaction/ob_tablet_merge_task.h" #include "storage/tx_storage/ob_ls_map.h" #include "storage/tx_storage/ob_ls_handle.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -206,7 +206,7 @@ struct ObTabletMergeCtx // 2. filled in ObPartitionStore::get_merge_tables ObVersionRange sstable_version_range_;// version range for new sstable share::ObScnRange scn_range_; - palf::SCN merge_scn_; + share::SCN merge_scn_; int64_t create_snapshot_version_; storage::ObTablesHandleArray tables_handle_; diff --git a/src/storage/compaction/ob_tablet_merge_task.cpp b/src/storage/compaction/ob_tablet_merge_task.cpp index a722fc769..7cdae2dbd 100644 --- a/src/storage/compaction/ob_tablet_merge_task.cpp +++ b/src/storage/compaction/ob_tablet_merge_task.cpp @@ -32,8 +32,11 @@ namespace oceanbase { -using namespace blocksstable; +using namespace common; +using namespace share; using namespace storage; +using namespace memtable; +using namespace blocksstable; namespace compaction { @@ -602,6 +605,9 @@ int ObTabletMergePrepareTask::process() && !MTL(ObTenantTabletScheduler *)->could_major_merge_start())) { ret = OB_CANCELED; LOG_INFO("Merge has been paused", K(ret), K(ctx)); + } else if (ctx->ls_handle_.get_ls()->is_offline()) { + ret = OB_CANCELED; + LOG_INFO("ls offline, skip merge", K(ret), K(ctx)); } else if (FALSE_IT(ctx->time_guard_.click(ObCompactionTimeGuard::DAG_WAIT_TO_SCHEDULE))) { } else if (OB_FAIL(ctx->ls_handle_.get_ls()->get_tablet(ctx->param_.tablet_id_, ctx->tablet_handle_, @@ -879,9 +885,7 @@ int ObTabletMergeFinishTask::process() } if (OB_SUCC(ret) && ctx.param_.is_major_merge() && NULL != ctx.param_.report_) { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(ctx.param_.report_->submit_tablet_checksums_task(MTL_ID(), ctx.param_.ls_id_, tablet_id))) { - LOG_WARN("failed to submit tablet checksums task to report", K(tmp_ret), K(MTL_ID()), K(ctx.param_.ls_id_), K(tablet_id)); - } else if (OB_TMP_FAIL(ctx.param_.report_->submit_tablet_update_task(MTL_ID(), ctx.param_.ls_id_, tablet_id))) { + if (OB_TMP_FAIL(ctx.param_.report_->submit_tablet_update_task(MTL_ID(), ctx.param_.ls_id_, tablet_id))) { LOG_WARN("failed to submit tablet update task to report", K(tmp_ret), K(MTL_ID()), K(ctx.param_.ls_id_), K(tablet_id)); } else if (OB_TMP_FAIL(ctx.ls_handle_.get_ls()->get_tablet_svr()->update_tablet_report_status(tablet_id))) { LOG_WARN("failed to update tablet report status", K(tmp_ret), K(MTL_ID()), K(tablet_id)); @@ -890,6 +894,10 @@ int ObTabletMergeFinishTask::process() if (OB_SUCC(ret) && OB_NOT_NULL(ctx.merge_progress_)) { int tmp_ret = OB_SUCCESS; + // update merge info + if (OB_TMP_FAIL(ctx.merge_progress_->update_merge_info(ctx.merge_info_.get_sstable_merge_info()))) { + STORAGE_LOG(WARN, "fail to update update merge info", K(tmp_ret)); + } if (OB_TMP_FAIL(compaction::ObCompactionSuggestionMgr::get_instance().analyze_merge_info( ctx.merge_info_, *ctx.merge_progress_))) { @@ -963,7 +971,7 @@ int ObTabletMergeFinishTask::add_sstable_for_merge(ObTabletMergeCtx &ctx) } if (OB_SUCC(ret)) { - palf::SCN clog_checkpoint_scn = ctx.param_.is_mini_merge() ? ctx.merged_table_handle_.get_table()->get_end_scn() : palf::SCN::min_scn(); + SCN clog_checkpoint_scn = ctx.param_.is_mini_merge() ? ctx.merged_table_handle_.get_table()->get_end_scn() : SCN::min_scn(); ObUpdateTableStoreParam param(ctx.merged_table_handle_, ctx.sstable_version_range_.snapshot_version_, ctx.sstable_version_range_.multi_version_start_, @@ -977,7 +985,9 @@ int ObTabletMergeFinishTask::add_sstable_for_merge(ObTabletMergeCtx &ctx) if (ctx.param_.tablet_id_.is_special_merge_tablet()) { param.multi_version_start_ = 1; } - if (OB_FAIL(ret)) { + // for mini merge, read all msd from frozen memtable + if (ctx.param_.is_mini_merge() && OB_FAIL(read_msd_from_memtable(ctx, param))) { + LOG_WARN("failed to read msd from memtable", K(ret), K(ctx)); } else if (OB_FAIL(ctx.ls_handle_.get_ls()->update_tablet_table_store( ctx.param_.tablet_id_, param, new_tablet_handle))) { LOG_WARN("failed to update tablet table store", K(ret), K(param)); @@ -1011,6 +1021,69 @@ int ObTabletMergeFinishTask::add_sstable_for_merge(ObTabletMergeCtx &ctx) return ret; } +int ObTabletMergeFinishTask::read_msd_from_memtable(ObTabletMergeCtx &ctx, ObUpdateTableStoreParam ¶m) +{ + int ret = OB_SUCCESS; + + if (OB_FAIL(traverse_all_memtables(ctx, ¶m.tx_data_, MultiSourceDataUnitType::TABLET_TX_DATA))) { + LOG_WARN("failed to read tx data from memtables", K(ret)); + } else if (OB_FAIL(traverse_all_memtables(ctx, ¶m.binding_info_, MultiSourceDataUnitType::TABLET_BINDING_INFO))) { + LOG_WARN("failed to read tx data from memtables", K(ret)); + } else if (OB_FAIL(traverse_all_memtables(ctx, ¶m.auto_inc_seq_, MultiSourceDataUnitType::TABLET_SEQ))) { + LOG_WARN("failed to read tx data from memtables", K(ret)); + } else { + LOG_INFO("succeeded to read msd from memtable", K(ret), + "ls_id", ctx.param_.ls_id_, + "tablet_id", ctx.param_.tablet_id_, + "tx_data", param.tx_data_, + "binding_info", param.binding_info_, + "auto_inc_seq", param.auto_inc_seq_); + } + + return ret; +} + +int ObTabletMergeFinishTask::traverse_all_memtables( + ObTabletMergeCtx &ctx, + ObIMultiSourceDataUnit *msd, + const MultiSourceDataUnitType &type) +{ + int ret = OB_SUCCESS; + ObIArray &tables = ctx.tables_handle_.get_tables(); + ObITable *table = nullptr; + ObMemtable *memtable = nullptr; + + if (OB_ISNULL(msd)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid args", K(ret)); + } + + for (int64_t i = tables.count() - 1; OB_SUCC(ret) && i >= 0; --i) { + if (OB_ISNULL(table = tables.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table is null", K(ret), K(tables), KP(table)); + } else if (OB_UNLIKELY(!table->is_memtable())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table is not memtable", K(ret), K(tables), KPC(table)); + } else if (OB_UNLIKELY(!table->is_frozen_memtable())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table is not frozen memtable", K(ret), K(tables), KPC(table)); + } else if (table->is_data_memtable()) { + memtable = static_cast(table); + if (memtable->has_multi_source_data_unit(type)) { + if (OB_FAIL(memtable->get_multi_source_data_unit(msd, nullptr/*allocator*/))) { + LOG_WARN("failed to get msd from memtable", K(ret), K(type)); + } else { + // succeeded to get msd, just break + break; + } + } + } + } + + return ret; +} + int ObTabletMergeFinishTask::try_schedule_compaction_after_mini( ObTabletMergeCtx &ctx, ObTabletHandle &tablet_handle) diff --git a/src/storage/compaction/ob_tablet_merge_task.h b/src/storage/compaction/ob_tablet_merge_task.h index b3cbbef9e..2404ab518 100644 --- a/src/storage/compaction/ob_tablet_merge_task.h +++ b/src/storage/compaction/ob_tablet_merge_task.h @@ -35,6 +35,13 @@ class ObITable; struct ObGetMergeTablesResult; class ObTablet; class ObTabletHandle; +struct ObUpdateTableStoreParam; +} + +namespace memtable +{ +enum class MultiSourceDataUnitType; +class ObIMultiSourceDataUnit; } namespace blocksstable @@ -145,6 +152,8 @@ private: int get_merged_sstable(ObTabletMergeCtx &ctx, blocksstable::ObSSTable *&sstable); int add_sstable_for_merge(ObTabletMergeCtx &ctx); int try_schedule_compaction_after_mini(ObTabletMergeCtx &ctx, storage::ObTabletHandle &tablet_handle); + int read_msd_from_memtable(ObTabletMergeCtx &ctx, storage::ObUpdateTableStoreParam ¶m); + int traverse_all_memtables(ObTabletMergeCtx &ctx, memtable::ObIMultiSourceDataUnit *msd, const memtable::MultiSourceDataUnitType &type); private: bool is_inited_; ObBasicTabletMergeDag *merge_dag_; diff --git a/src/storage/compaction/ob_tenant_compaction_progress.cpp b/src/storage/compaction/ob_tenant_compaction_progress.cpp index aaa33af49..b86409fbb 100644 --- a/src/storage/compaction/ob_tenant_compaction_progress.cpp +++ b/src/storage/compaction/ob_tenant_compaction_progress.cpp @@ -151,6 +151,7 @@ int ObTenantCompactionProgressMgr::loop_major_sstable_( int64_t &size) { int ret = OB_SUCCESS; + common::ObTimeGuard timeguard("loop_major_sstable_to_calc_progress_size", 30 * 1000 * 1000); // 30s ObSharedGuard ls_iter_guard; ObLS *ls = nullptr; if (OB_FAIL(MTL(ObLSService *)->get_ls_iter(ls_iter_guard, ObLSGetMod::STORAGE_MOD))) { diff --git a/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp b/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp index fec93d517..245e462b3 100644 --- a/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp +++ b/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp @@ -43,7 +43,6 @@ using namespace share; using namespace share::schema; using common::hash::ObHashSet; -using palf::SCN; namespace storage { @@ -131,9 +130,9 @@ void ObTenantFreezeInfoMgr::destroy() TG_DESTROY(tg_id_); } -palf::SCN ObTenantFreezeInfoMgr::get_latest_frozen_scn() +SCN ObTenantFreezeInfoMgr::get_latest_frozen_scn() { - palf::SCN frozen_scn = palf::SCN::min_scn(); + SCN frozen_scn = SCN::min_scn(); RLockGuard lock_guard(lock_); ObIArray &info_list = info_list_[cur_idx_]; @@ -153,6 +152,7 @@ int ObTenantFreezeInfoMgr::get_min_dependent_freeze_info(FreezeInfo &freeze_info idx = info_list.count() - MIN_DEPENDENT_FREEZE_INFO_GAP; } ret = get_info_nolock(idx, freeze_info); + LOG_INFO("get min dependent freeze info", K(ret), K(freeze_info)); // diagnose code for issue 45841468 return ret; } @@ -490,7 +490,7 @@ int ObTenantFreezeInfoMgr::diagnose_min_reserved_snapshot( ret = OB_SUCCESS; } } - snapshot_version = std::max(0L, snapshot_gc_ts_ - duration * 1000L * 1000L); + snapshot_version = std::max(0L, snapshot_gc_ts_ - duration * 1000L * 1000L * 1000L); snapshot_from_type = "undo_retention"; if (freeze_info.freeze_scn.get_val_for_tx() < snapshot_version) { snapshot_version = freeze_info.freeze_scn.get_val_for_tx(); @@ -561,7 +561,7 @@ int ObTenantFreezeInfoMgr::get_reserve_points( // return ret; // } -int ObTenantFreezeInfoMgr::get_latest_freeze_scn(palf::SCN &freeze_scn) +int ObTenantFreezeInfoMgr::get_latest_freeze_scn(SCN &freeze_scn) { int ret = OB_SUCCESS; @@ -582,7 +582,7 @@ int ObTenantFreezeInfoMgr::get_latest_freeze_scn(palf::SCN &freeze_scn) return ret; } -int ObTenantFreezeInfoMgr::prepare_new_info_list(const palf::SCN &min_major_snapshot) +int ObTenantFreezeInfoMgr::prepare_new_info_list(const SCN &min_major_snapshot) { int ret = OB_SUCCESS; @@ -591,7 +591,7 @@ int ObTenantFreezeInfoMgr::prepare_new_info_list(const palf::SCN &min_major_snap snapshots_[next_idx].reset(); for (int64_t i = 0; i < info_list_[cur_idx_].count() && OB_SUCC(ret); ++i) { - if (palf::SCN::max_scn() == min_major_snapshot || // no garbage collection is necessary + if (SCN::max_scn() == min_major_snapshot || // no garbage collection is necessary // or version is bigger or equal than the smallest major version currently info_list_[cur_idx_].at(i).freeze_scn >= min_major_snapshot) { if (OB_FAIL(info_list_[next_idx].push_back(info_list_[cur_idx_].at(i)))) { @@ -654,7 +654,7 @@ int ObTenantFreezeInfoMgr::update_info( const int64_t snapshot_gc_ts, const ObIArray &info_list, const ObIArray &snapshots, - const palf::SCN &min_major_snapshot, + const SCN &min_major_snapshot, bool& gc_snapshot_ts_changed) { int ret = OB_SUCCESS; @@ -721,7 +721,7 @@ int ObTenantFreezeInfoMgr::ReloadTask::get_global_info(int64_t &snapshot_gc_ts) int ret = OB_SUCCESS; uint64_t tenant_id = MTL_ID(); ObGlobalStatProxy stat_proxy(*sql_proxy_, tenant_id); - palf::SCN snapshot_gc_scn; + SCN snapshot_gc_scn; if (OB_FAIL(stat_proxy.get_snapshot_gc_scn(snapshot_gc_scn))) { if (OB_TENANT_NOT_EXIST != ret) { STORAGE_LOG(WARN, "fail to get global info", K(ret), K(tenant_id)); @@ -735,12 +735,12 @@ int ObTenantFreezeInfoMgr::ReloadTask::get_global_info(int64_t &snapshot_gc_ts) } int ObTenantFreezeInfoMgr::ReloadTask::get_freeze_info( - palf::SCN &min_major_version, + SCN &min_major_version, ObIArray &freeze_info) { int ret = OB_SUCCESS; - palf::SCN freeze_scn = palf::SCN::min_scn(); + SCN freeze_scn = SCN::min_scn(); min_major_version.set_max(); ObSEArray tmp; ObFreezeInfoProxy freeze_info_proxy(MTL_ID()); @@ -840,7 +840,7 @@ int ObTenantFreezeInfoMgr::ReloadTask::try_update_info() ObSEArray snapshots; bool changed = false; observer::ObService *ob_service = GCTX.ob_service_; - palf::SCN min_major_snapshot = palf::SCN::max_scn(); + SCN min_major_snapshot = SCN::max_scn(); if (OB_FAIL(get_global_info(snapshot_gc_ts))) { if (OB_TENANT_NOT_EXIST != ret) { diff --git a/src/storage/compaction/ob_tenant_freeze_info_mgr.h b/src/storage/compaction/ob_tenant_freeze_info_mgr.h index a1d607997..dde5b1788 100644 --- a/src/storage/compaction/ob_tenant_freeze_info_mgr.h +++ b/src/storage/compaction/ob_tenant_freeze_info_mgr.h @@ -22,7 +22,7 @@ #include "share/ob_freeze_info_proxy.h" #include "share/ob_snapshot_table_proxy.h" #include "share/ob_zone_info.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -32,11 +32,6 @@ namespace common class ObISQLClient; } -namespace palf -{ -class SCN; -} - namespace storage { @@ -47,12 +42,12 @@ class ObTenantFreezeInfoMgr public: struct FreezeInfo { - palf::SCN freeze_scn; + share::SCN freeze_scn; int64_t schema_version; int64_t cluster_version; FreezeInfo() : freeze_scn(), schema_version(-1), cluster_version(0) {} - FreezeInfo(const palf::SCN &scn, const int64_t schema_ver, const int64_t cluster_ver) + FreezeInfo(const share::SCN &scn, const int64_t schema_ver, const int64_t cluster_ver) : freeze_scn(scn), schema_version(schema_ver), cluster_version(cluster_ver) {} FreezeInfo &operator =(const FreezeInfo &o) { @@ -90,7 +85,7 @@ public: void stop(); void destroy(); - palf::SCN get_latest_frozen_scn(); + share::SCN get_latest_frozen_scn(); int get_freeze_info_behind_major_snapshot(const int64_t major_snapshot, common::ObIArray &freeze_infos); int get_freeze_info_by_snapshot_version(const int64_t snapshot_version, FreezeInfo &freeze_info); @@ -122,7 +117,7 @@ public: const int64_t snapshot_gc_ts, const common::ObIArray &info_list, const common::ObIArray &snapshots, - const palf::SCN &min_major_snapshot, + const share::SCN &min_major_snapshot, bool& changed); int64_t get_snapshot_gc_ts(); @@ -142,11 +137,11 @@ private: common::MODIFY_GC_SNAPSHOT_INTERVAL + 10L * 1000L * 1000L; static const int64_t MIN_DEPENDENT_FREEZE_INFO_GAP = 2; - int get_latest_freeze_scn(palf::SCN &freeze_scn); + int get_latest_freeze_scn(share::SCN &freeze_scn); int64_t get_next_idx() { return 1L - cur_idx_; } void switch_info() { cur_idx_ = get_next_idx(); } int get_info_nolock(const int64_t idx, FreezeInfo &freeze_info); - int prepare_new_info_list(const palf::SCN &min_major_snapshot); + int prepare_new_info_list(const share::SCN &min_major_snapshot); virtual int get_multi_version_duration(int64_t &duration) const; int inner_get_neighbour_major_freeze(const int64_t snapshot_version, NeighbourFreezeInfo &info); @@ -171,7 +166,7 @@ private: private: int get_global_info(int64_t &snapshot_gc_ts); - int get_freeze_info(palf::SCN &min_major_snapshot, + int get_freeze_info(share::SCN &min_major_snapshot, common::ObIArray &freeze_info); bool inited_; diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp index a4ac19452..8f7a2afeb 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp @@ -31,7 +31,7 @@ #include "ob_tenant_freeze_info_mgr.h" #include "ob_tenant_compaction_progress.h" #include "ob_server_compaction_event_history.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -348,6 +348,26 @@ int ObTenantTabletScheduler::update_upper_trans_version_and_gc_sstable() return ret; } +int ObTenantTabletScheduler::wait_ls_compaction_finish(const share::ObLSID &ls_id) +{ + int ret = OB_SUCCESS; + bool exist = false; + if (OB_UNLIKELY(!ls_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(ls_id)); + } + while (OB_SUCC(ret)) { + if (OB_FAIL(MTL(ObTenantDagScheduler*)->check_ls_compaction_dag_exist(ls_id, exist))) { + LOG_WARN("failed to check ls compaction dag", K(ret), K(ls_id)); + } else if (!exist) { + break; + } else { + ob_usleep(100 * 1000); // 100ms + } + } + return ret; +} + int ObTenantTabletScheduler::schedule_build_bloomfilter( const uint64_t table_id, const blocksstable::MacroBlockId ¯o_id, @@ -452,7 +472,7 @@ bool ObTenantTabletScheduler::check_weak_read_ts_ready( ObLS &ls) { bool is_ready_for_compaction = false; - palf::SCN weak_read_scn; + SCN weak_read_scn; if (FALSE_IT(weak_read_scn = ls.get_ls_wrs_handler()->get_ls_weak_read_ts())) { } else if (weak_read_scn.get_val_for_tx() < merge_version) { @@ -508,11 +528,11 @@ int64_t ObTenantTabletScheduler::get_frozen_version() const return frozen_version_; } -bool ObTenantTabletScheduler::check_tx_table_ready(ObLS &ls, const palf::SCN &check_scn) +bool ObTenantTabletScheduler::check_tx_table_ready(ObLS &ls, const SCN &check_scn) { int ret = OB_SUCCESS; bool tx_table_ready = false; - palf::SCN max_decided_scn; + SCN max_decided_scn; if (OB_FAIL(ls.get_max_decided_scn(max_decided_scn))) { LOG_WARN("failed to get max decided log_ts", K(ret), "ls_id", ls.get_ls_id()); } else if (check_scn <= max_decided_scn) { @@ -529,6 +549,8 @@ int ObTenantTabletScheduler::check_ls_state(ObLS &ls, bool &need_merge) need_merge = false; if (ls.is_deleted()) { LOG_INFO("ls is deleted", K(ret), K(ls)); + } else if (ls.is_offline()) { + LOG_INFO("ls is offline", K(ret), K(ls)); } else { need_merge = true; } @@ -659,9 +681,7 @@ int ObTenantTabletScheduler::schedule_tablet_major_merge( } if (OB_SUCC(ret) && tablet_merge_finish && tablet.get_tablet_meta().report_status_.need_report()) { - if (OB_TMP_FAIL(GCTX.ob_service_->submit_tablet_checksums_task(MTL_ID(), ls_id, tablet_id))) { - LOG_WARN("failed to submit tablet checksums task to report", K(tmp_ret), K(MTL_ID()), K(tablet_id)); - } else if (OB_TMP_FAIL(GCTX.ob_service_->submit_tablet_update_task(MTL_ID(), ls_id, tablet_id))) { + if (OB_TMP_FAIL(GCTX.ob_service_->submit_tablet_update_task(MTL_ID(), ls_id, tablet_id))) { LOG_WARN("failed to submit tablet update task to report", K(tmp_ret), K(MTL_ID()), K(tablet_id)); } else if (OB_TMP_FAIL(ls.get_tablet_svr()->update_tablet_report_status(tablet_id))) { LOG_WARN("failed to update tablet report status", K(tmp_ret), K(MTL_ID()), K(tablet_id)); diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.h b/src/storage/compaction/ob_tenant_tablet_scheduler.h index 958437b7d..5d77e8c14 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.h +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.h @@ -112,6 +112,7 @@ public: int merge_all(); int schedule_merge(const int64_t broadcast_version); int update_upper_trans_version_and_gc_sstable(); + int wait_ls_compaction_finish(const share::ObLSID &ls_id); // Schedule an async task to build bloomfilter for the given macro block. // The bloomfilter build task will be ignored if a same build task exists in the queue. @@ -120,7 +121,7 @@ public: const blocksstable::MacroBlockId ¯o_id, const int64_t prefix_len); int schedule_load_bloomfilter(const blocksstable::MacroBlockId ¯o_id); - static bool check_tx_table_ready(ObLS &ls, const palf::SCN &check_scn); + static bool check_tx_table_ready(ObLS &ls, const share::SCN &check_scn); static int check_ls_state(ObLS &ls, bool &need_merge); static int schedule_tablet_minor_merge(const share::ObLSID ls_id, ObTablet &tablet); static int schedule_tablet_major_merge( diff --git a/src/storage/compaction/ob_tx_table_merge_task.cpp b/src/storage/compaction/ob_tx_table_merge_task.cpp index 2b6ccc6a1..f6b38a294 100644 --- a/src/storage/compaction/ob_tx_table_merge_task.cpp +++ b/src/storage/compaction/ob_tx_table_merge_task.cpp @@ -67,7 +67,7 @@ int ObTxTableMergeDag::init_by_param(const ObIDagInitParam *param) } else if (merge_param->tablet_id_.is_ls_tx_data_tablet() && merge_param->is_minor_merge()) { // init compaction filter for minor merge in TxDataTable ObTxTableGuard guard; - palf::SCN recycle_scn = palf::SCN::min_scn(); + SCN recycle_scn = SCN::min_scn(); int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(ctx_->ls_handle_.get_ls()->get_tx_table_guard(guard))) { LOG_WARN("failed to get tx table", K(tmp_ret), KPC(merge_param)); diff --git a/src/storage/ddl/ob_complement_data_task.cpp b/src/storage/ddl/ob_complement_data_task.cpp index d79cf50c0..6ba910dbc 100644 --- a/src/storage/ddl/ob_complement_data_task.cpp +++ b/src/storage/ddl/ob_complement_data_task.cpp @@ -310,7 +310,7 @@ int ObComplementDataContext::write_start_log(const ObComplementDataParam ¶m) } else if (OB_UNLIKELY(!hidden_table_key.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid table key", K(ret), K(hidden_table_key)); - } else if (OB_FAIL(data_sstable_redo_writer_.start_ddl_redo(hidden_table_key))) { + } else if (OB_FAIL(data_sstable_redo_writer_.start_ddl_redo(hidden_table_key, ddl_kv_mgr_handle_))) { LOG_WARN("fail write start log", K(ret), K(hidden_table_key), K(param)); } else { LOG_INFO("complement task start ddl redo success", K(hidden_table_key)); @@ -328,6 +328,7 @@ void ObComplementDataContext::destroy() allocator_.free(index_builder_); index_builder_ = nullptr; } + ddl_kv_mgr_handle_.reset(); allocator_.reset(); } @@ -1144,7 +1145,7 @@ int ObComplementMergeTask::add_build_hidden_table_sstable() ObTablet *tablet = nullptr; ObTabletHandle tablet_handle; ObITable::TableKey hidden_table_key; - palf::SCN prepare_scn; + SCN prepare_scn; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObComplementMergetask has not been inited", K(ret)); @@ -1177,13 +1178,17 @@ int ObComplementMergeTask::add_build_hidden_table_sstable() 1/*execution_id*/, param_->task_id_, prepare_scn))) { - LOG_WARN("fail write ddl prepare log", K(ret), K(hidden_table_key)); + if (OB_TASK_EXPIRED == ret) { + LOG_INFO("ddl task expired, but return success", K(ret), K(hidden_table_key), KPC(param_)); + } else { + LOG_WARN("fail write ddl prepare log", K(ret), K(hidden_table_key)); + } } else { ObTabletHandle new_tablet_handle; // no use here ObDDLKvMgrHandle ddl_kv_mgr_handle; const ObLSID &ls_id = param_->ls_id_; const ObTabletID &tablet_id = tablet->get_tablet_meta().tablet_id_; - const palf::SCN &ddl_start_scn = static_cast(get_dag())->get_context().data_sstable_redo_writer_.get_start_scn(); + const SCN &ddl_start_scn = static_cast(get_dag())->get_context().data_sstable_redo_writer_.get_start_scn(); if (OB_FAIL(tablet->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { LOG_WARN("get ddl kv manager failed", K(ret)); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_prepare(ddl_start_scn, @@ -1191,9 +1196,17 @@ int ObComplementMergeTask::add_build_hidden_table_sstable() param_->hidden_table_schema_->get_table_id(), 1/*execution_id*/, param_->task_id_))) { - LOG_WARN("commit ddl log failed", K(ret), K(ddl_start_scn), K(prepare_scn), K(hidden_table_key)); + if (OB_TASK_EXPIRED == ret) { + LOG_INFO("ddl task expired, but return success", K(ret), K(ls_id), K(tablet_id), + K(ddl_start_scn), "new_ddl_start_scn", ddl_kv_mgr_handle.get_obj()->get_start_scn()); + ret = OB_SUCCESS; + } else { + LOG_WARN("commit ddl log failed", K(ret), K(ddl_start_scn), K(prepare_scn), K(hidden_table_key)); + } } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->wait_ddl_commit(ddl_start_scn, prepare_scn))) { if (OB_TASK_EXPIRED == ret) { + LOG_INFO("ddl task expired, but return success", K(ret), K(ls_id), K(tablet_id), + K(ddl_start_scn), "new_ddl_start_scn", ddl_kv_mgr_handle.get_obj()->get_start_scn()); ret = OB_SUCCESS; } else { LOG_WARN("wait ddl commit failed", K(ret), K(ddl_start_scn), K(hidden_table_key)); @@ -1498,7 +1511,7 @@ int ObLocalScan::construct_range_ctx(ObQueryFlag &query_flag, trans_version_range.snapshot_version_ = snapshot_version_; trans_version_range.multi_version_start_ = snapshot_version_; trans_version_range.base_version_ = 0; - palf::SCN tmp_scn; + SCN tmp_scn; if (OB_FAIL(tmp_scn.convert_for_tx(snapshot_version_))) { LOG_WARN("convert fail", K(ret), K(ls_id), K_(snapshot_version)); } else if (OB_FAIL(ctx_.init_for_read(ls_id, diff --git a/src/storage/ddl/ob_complement_data_task.h b/src/storage/ddl/ob_complement_data_task.h index fe007374d..f1af0a520 100644 --- a/src/storage/ddl/ob_complement_data_task.h +++ b/src/storage/ddl/ob_complement_data_task.h @@ -121,6 +121,7 @@ public: int64_t concurrent_cnt_; ObDDLSSTableRedoWriter data_sstable_redo_writer_; blocksstable::ObSSTableIndexBuilder *index_builder_; + ObDDLKvMgrHandle ddl_kv_mgr_handle_; // for keeping ddl kv mgr alive }; class ObComplementPrepareTask; diff --git a/src/storage/ddl/ob_ddl_clog.cpp b/src/storage/ddl/ob_ddl_clog.cpp index c419b9c07..ccb28626e 100644 --- a/src/storage/ddl/ob_ddl_clog.cpp +++ b/src/storage/ddl/ob_ddl_clog.cpp @@ -22,6 +22,7 @@ namespace oceanbase { using namespace blocksstable; +using namespace share; namespace storage { @@ -242,7 +243,7 @@ ObDDLPrepareLog::ObDDLPrepareLog() } int ObDDLPrepareLog::init(const ObITable::TableKey &table_key, - const palf::SCN &start_scn) + const SCN &start_scn) { int ret = OB_SUCCESS; if (!table_key.is_valid() || !start_scn.is_valid()) { @@ -263,8 +264,8 @@ ObDDLCommitLog::ObDDLCommitLog() } int ObDDLCommitLog::init(const ObITable::TableKey &table_key, - const palf::SCN &start_scn, - const palf::SCN &prepare_scn) + const SCN &start_scn, + const SCN &prepare_scn) { int ret = OB_SUCCESS; if (!table_key.is_valid() || !start_scn.is_valid() || !prepare_scn.is_valid()) { diff --git a/src/storage/ddl/ob_ddl_clog.h b/src/storage/ddl/ob_ddl_clog.h index 9cb7117f0..5e831b7d3 100644 --- a/src/storage/ddl/ob_ddl_clog.h +++ b/src/storage/ddl/ob_ddl_clog.h @@ -151,14 +151,14 @@ public: ObDDLPrepareLog(); ~ObDDLPrepareLog() = default; int init(const ObITable::TableKey &table_key, - const palf::SCN &start_scn); + const share::SCN &start_scn); bool is_valid() const { return table_key_.is_valid() && start_scn_.is_valid(); } ObITable::TableKey get_table_key() const { return table_key_; } - palf::SCN get_start_scn() const { return start_scn_; } + share::SCN get_start_scn() const { return start_scn_; } TO_STRING_KV(K_(table_key), K_(start_scn)); private: ObITable::TableKey table_key_; - palf::SCN start_scn_; + share::SCN start_scn_; }; class ObDDLCommitLog final @@ -168,17 +168,17 @@ public: ObDDLCommitLog(); ~ObDDLCommitLog() = default; int init(const ObITable::TableKey &table_key, - const palf::SCN &start_scn, - const palf::SCN &prepare_scn); + const share::SCN &start_scn, + const share::SCN &prepare_scn); bool is_valid() const { return table_key_.is_valid() && start_scn_.is_valid() && prepare_scn_.is_valid(); } ObITable::TableKey get_table_key() const { return table_key_; } - palf::SCN get_start_scn() const { return start_scn_; } - palf::SCN get_prepare_scn() const { return prepare_scn_; } + share::SCN get_start_scn() const { return start_scn_; } + share::SCN get_prepare_scn() const { return prepare_scn_; } TO_STRING_KV(K_(table_key), K_(start_scn), K_(prepare_scn)); private: ObITable::TableKey table_key_; - palf::SCN start_scn_; - palf::SCN prepare_scn_; + share::SCN start_scn_; + share::SCN prepare_scn_; }; class ObTabletSchemaVersionChangeLog final diff --git a/src/storage/ddl/ob_ddl_merge_task.cpp b/src/storage/ddl/ob_ddl_merge_task.cpp index 8b2f573f7..d66d0b852 100644 --- a/src/storage/ddl/ob_ddl_merge_task.cpp +++ b/src/storage/ddl/ob_ddl_merge_task.cpp @@ -13,7 +13,7 @@ #define USING_LOG_PREFIX STORAGE_COMPACTION #include "storage/ddl/ob_ddl_merge_task.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_ddl_checksum.h" #include "share/ob_get_compat_mode.h" #include "share/schema/ob_table_schema.h" @@ -107,6 +107,9 @@ int ObDDLTableMergeDag::create_first_task() LOG_WARN("get tablet failed", K(ret), K(ddl_param_)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { LOG_WARN("get ddl kv mgr failed", K(ret), K(ddl_param_)); + } else if (ddl_param_.start_scn_ < ddl_kv_mgr_handle.get_obj()->get_start_scn()) { + ret = OB_TASK_EXPIRED; + LOG_WARN("ddl task expired, skip it", K(ret), K(ddl_param_), "new_start_scn", ddl_kv_mgr_handle.get_obj()->get_start_scn()); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->freeze_ddl_kv())) { LOG_WARN("ddl kv manager try freeze failed", K(ret), K(ddl_param_)); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_ddl_kvs(true/*frozen_only*/, ddl_kvs_handle))) { @@ -203,7 +206,7 @@ ObDDLTableDumpTask::~ObDDLTableDumpTask() int ObDDLTableDumpTask::init(const share::ObLSID &ls_id, const ObTabletID &tablet_id, - const palf::SCN &freeze_scn) + const SCN &freeze_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { @@ -340,6 +343,8 @@ int ObDDLTableMergeTask::process() LOG_INFO("tablet me says with major but no major, meaning its a migrated deleted tablet, skip"); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_ddl_param(ddl_param))) { LOG_WARN("get tablet ddl param failed", K(ret)); + } else if (merge_param_.start_scn_ > SCN::min_scn() && merge_param_.start_scn_ < ddl_param.start_scn_) { + LOG_INFO("ddl merge task expired, do nothing", K(merge_param_), "new_start_scn", ddl_param.start_scn_); } else if (merge_param_.is_commit_ && OB_FAIL(check_data_integrity(ddl_sstable_handles, ddl_param.start_scn_, merge_param_.rec_scn_, @@ -381,8 +386,6 @@ int ObDDLTableMergeTask::process() merge_param_.ddl_task_id_, sstable->get_meta().get_col_checksum()))) { LOG_WARN("report ddl column checksum failed", K(ret), K(merge_param_)); - } else if (OB_FAIL(GCTX.ob_service_->submit_tablet_checksums_task(tenant_id, merge_param_.ls_id_, merge_param_.tablet_id_))) { - LOG_WARN("fail to submit tablet checksums task", K(ret), K(tenant_id), K(merge_param_)); } else if (OB_FAIL(GCTX.ob_service_->submit_tablet_update_task(tenant_id, merge_param_.ls_id_, merge_param_.tablet_id_))) { LOG_WARN("fail to submit tablet update task", K(ret), K(tenant_id), K(merge_param_)); } @@ -398,8 +401,8 @@ int ObDDLTableMergeTask::process() } int ObDDLTableMergeTask::check_data_integrity(const ObTablesHandleArray &ddl_sstables, - const palf::SCN &start_scn, - const palf::SCN &prepare_scn, + const SCN &start_scn, + const SCN &prepare_scn, bool &is_data_complete) { int ret = OB_SUCCESS; @@ -416,7 +419,7 @@ int ObDDLTableMergeTask::check_data_integrity(const ObTablesHandleArray &ddl_sst LOG_INFO("prepare log ts not match", K(last_ddl_sstable->get_key()), K(prepare_scn)); } else { is_data_complete = true; - palf::SCN last_end_scn = first_ddl_sstable->get_end_scn(); + SCN last_end_scn = first_ddl_sstable->get_end_scn(); for (int64_t i = 1; OB_SUCC(ret) && i < ddl_sstables.get_count(); ++i) { const ObSSTable *cur_ddl_sstable = static_cast(ddl_sstables.get_tables().at(i)); if (cur_ddl_sstable->get_start_scn() != last_end_scn) { diff --git a/src/storage/ddl/ob_ddl_merge_task.h b/src/storage/ddl/ob_ddl_merge_task.h index 07c92b6b8..33ffdcabe 100644 --- a/src/storage/ddl/ob_ddl_merge_task.h +++ b/src/storage/ddl/ob_ddl_merge_task.h @@ -13,7 +13,7 @@ #ifndef OCEANBASE_STORAGE_DDL_MERGE_TASK_ #define OCEANBASE_STORAGE_DDL_MERGE_TASK_ -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/meta_mem/ob_tablet_handle.h" #include "share/scheduler/ob_dag_scheduler.h" #include "storage/blocksstable/ob_index_block_builder.h" @@ -46,18 +46,21 @@ public: table_id_(0), execution_id_(0), ddl_task_id_(0) - {} + { + start_scn_.set_min(); + } bool is_valid() const { - return ls_id_.is_valid() && tablet_id_.is_valid(); + return ls_id_.is_valid() && tablet_id_.is_valid() && start_scn_.is_valid_and_not_min(); } virtual ~ObDDLTableMergeDagParam() = default; - TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(rec_scn), K_(is_commit), K_(table_id), K_(execution_id), K_(ddl_task_id)); + TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(rec_scn), K_(is_commit), K_(start_scn), K_(table_id), K_(execution_id), K_(ddl_task_id)); public: share::ObLSID ls_id_; ObTabletID tablet_id_; - palf::SCN rec_scn_; + share::SCN rec_scn_; bool is_commit_; + share::SCN start_scn_; // start log ts at schedule, for skipping expired task uint64_t table_id_; // used for report ddl checksum int64_t execution_id_; // used for report ddl checksum int64_t ddl_task_id_; // used for report ddl checksum @@ -94,14 +97,14 @@ class ObDDLTableDumpTask : public share::ObITask public: ObDDLTableDumpTask(); virtual ~ObDDLTableDumpTask(); - int init(const share::ObLSID &ls_id, const ObTabletID &tablet_id, const palf::SCN &freeze_scn); + int init(const share::ObLSID &ls_id, const ObTabletID &tablet_id, const share::SCN &freeze_scn); virtual int process() override; TO_STRING_KV(K_(is_inited), K_(ls_id), K_(tablet_id), K_(freeze_scn)); private: bool is_inited_; share::ObLSID ls_id_; ObTabletID tablet_id_; - palf::SCN freeze_scn_; + share::SCN freeze_scn_; DISALLOW_COPY_AND_ASSIGN(ObDDLTableDumpTask); }; @@ -113,8 +116,8 @@ public: int init(const ObDDLTableMergeDagParam &ddl_dag_param); virtual int process() override; static int check_data_integrity(const ObTablesHandleArray &ddl_sstables, - const palf::SCN &start_scn, - const palf::SCN &prepare_scn, + const share::SCN &start_scn, + const share::SCN &prepare_scn, bool &is_data_complete); TO_STRING_KV(K_(is_inited), K_(merge_param)); private: @@ -134,7 +137,7 @@ public: uint64_t tenant_id_; share::ObLSID ls_id_; ObITable::TableKey table_key_; - palf::SCN start_scn_; + share::SCN start_scn_; int64_t snapshot_version_; int64_t cluster_version_; }; diff --git a/src/storage/ddl/ob_ddl_redo_log_replayer.cpp b/src/storage/ddl/ob_ddl_redo_log_replayer.cpp index 9305eda6f..a420a14b5 100644 --- a/src/storage/ddl/ob_ddl_redo_log_replayer.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_replayer.cpp @@ -23,6 +23,7 @@ using namespace oceanbase::common; using namespace oceanbase::lib; using namespace oceanbase::blocksstable; using namespace oceanbase::storage; +using namespace oceanbase::share; ObDDLRedoLogReplayer::ObDDLRedoLogReplayer() : is_inited_(false), ls_(nullptr), allocator_() @@ -51,7 +52,7 @@ int ObDDLRedoLogReplayer::init(ObLS *ls) return ret; } -int ObDDLRedoLogReplayer::replay_start(const ObDDLStartLog &log, const palf::SCN &scn) +int ObDDLRedoLogReplayer::replay_start(const ObDDLStartLog &log, const SCN &scn) { int ret = OB_SUCCESS; ObTabletHandle tablet_handle; @@ -84,7 +85,7 @@ int ObDDLRedoLogReplayer::replay_start(const ObDDLStartLog &log, const palf::SCN return ret; } -int ObDDLRedoLogReplayer::replay_redo(const ObDDLRedoLog &log, const palf::SCN &scn) +int ObDDLRedoLogReplayer::replay_redo(const ObDDLRedoLog &log, const SCN &scn) { int ret = OB_SUCCESS; const ObDDLMacroBlockRedoInfo &redo_info = log.get_redo_info(); @@ -147,7 +148,7 @@ int ObDDLRedoLogReplayer::replay_redo(const ObDDLRedoLog &log, const palf::SCN & return ret; } -int ObDDLRedoLogReplayer::replay_prepare(const ObDDLPrepareLog &log, const palf::SCN &scn) +int ObDDLRedoLogReplayer::replay_prepare(const ObDDLPrepareLog &log, const SCN &scn) { int ret = OB_SUCCESS; ObTabletHandle tablet_handle; @@ -185,7 +186,7 @@ int ObDDLRedoLogReplayer::replay_prepare(const ObDDLPrepareLog &log, const palf: return ret; } -int ObDDLRedoLogReplayer::replay_commit(const ObDDLCommitLog &log, const palf::SCN &scn) +int ObDDLRedoLogReplayer::replay_commit(const ObDDLCommitLog &log, const SCN &scn) { int ret = OB_SUCCESS; ObTabletHandle tablet_handle; @@ -227,8 +228,8 @@ int ObDDLRedoLogReplayer::replay_commit(const ObDDLCommitLog &log, const palf::S } int ObDDLRedoLogReplayer::check_need_replay_ddl_log(const ObITable::TableKey &table_key, - const palf::SCN &ddl_start_scn, - const palf::SCN &scn, + const SCN &ddl_start_scn, + const SCN &scn, bool &need_replay, ObTabletHandle &tablet_handle) { diff --git a/src/storage/ddl/ob_ddl_redo_log_replayer.h b/src/storage/ddl/ob_ddl_redo_log_replayer.h index 5ba7c94b9..ea25cfa26 100644 --- a/src/storage/ddl/ob_ddl_redo_log_replayer.h +++ b/src/storage/ddl/ob_ddl_redo_log_replayer.h @@ -31,15 +31,15 @@ public: ~ObDDLRedoLogReplayer(); int init(ObLS *ls); void reset() { destroy(); } - int replay_start(const ObDDLStartLog &log, const palf::SCN &scn); - int replay_redo(const ObDDLRedoLog &log, const palf::SCN &scn); - int replay_prepare(const ObDDLPrepareLog &log, const palf::SCN &scn); - int replay_commit(const ObDDLCommitLog &log, const palf::SCN &scn); + int replay_start(const ObDDLStartLog &log, const share::SCN &scn); + int replay_redo(const ObDDLRedoLog &log, const share::SCN &scn); + int replay_prepare(const ObDDLPrepareLog &log, const share::SCN &scn); + int replay_commit(const ObDDLCommitLog &log, const share::SCN &scn); private: void destroy(); int check_need_replay_ddl_log(const ObITable::TableKey &table_key, - const palf::SCN &ddl_start_scn, - const palf::SCN &scn, + const share::SCN &ddl_start_scn, + const share::SCN &scn, bool &need_replay, ObTabletHandle &tablet_handle); private: diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_redo_log_writer.cpp index bf05f7e41..517b93434 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_writer.cpp @@ -16,7 +16,7 @@ #include "logservice/archiveservice/ob_archive_service.h" #include "logservice/ob_log_handler.h" #include "logservice/ob_log_service.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/blocksstable/ob_block_manager.h" #include "storage/blocksstable/ob_index_block_builder.h" #include "storage/blocksstable/ob_macro_block_struct.h" @@ -557,8 +557,8 @@ int ObDDLRedoLogWriter::write( palf::LSN lsn; const bool need_nonblock= false; - palf::SCN base_scn = palf::SCN::min_scn(); - palf::SCN scn; + SCN base_scn = SCN::min_scn(); + SCN scn; if (!log.is_valid() || nullptr == log_handler || !ls_id.is_valid() || OB_INVALID_TENANT_ID == tenant_id || nullptr == buffer) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(log), K(ls_id), K(tenant_id), KP(buffer)); @@ -602,7 +602,7 @@ int ObDDLRedoLogWriter::write( return ret; } -int ObDDLRedoLogWriter::write_ddl_start_log(const ObDDLStartLog &log, ObLogHandler *log_handler, palf::SCN &start_scn) +int ObDDLRedoLogWriter::write_ddl_start_log(const ObDDLStartLog &log, ObLogHandler *log_handler, SCN &start_scn) { int ret = OB_SUCCESS; start_scn.reset(); @@ -619,7 +619,7 @@ int ObDDLRedoLogWriter::write_ddl_start_log(const ObDDLStartLog &log, ObLogHandl palf::LSN lsn; const bool need_nonblock= false; - palf::SCN scn; + SCN scn; bool is_external_consistent = false; if (OB_ISNULL(cb = op_alloc(ObDDLClogCb))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -632,7 +632,7 @@ int ObDDLRedoLogWriter::write_ddl_start_log(const ObDDLStartLog &log, ObLogHandl LOG_WARN("fail to seriaize ddl start log", K(ret)); } else if (OB_FAIL(log_handler->append(buffer, buffer_size, - palf::SCN::base_scn(), + SCN::base_scn(), need_nonblock, cb, lsn, @@ -691,8 +691,8 @@ int ObDDLRedoLogWriter::write_ddl_finish_log(const T &log, const ObDDLClogType c palf::LSN lsn; const bool need_nonblock= false; - palf::SCN base_scn = palf::SCN::base_scn(); - palf::SCN scn; + SCN base_scn = SCN::base_scn(); + SCN scn; bool is_external_consistent = false; if (OB_ISNULL(buffer = static_cast(ob_malloc(buffer_size)))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -924,13 +924,13 @@ int ObDDLSSTableRedoWriter::init(const ObLSID &ls_id, const ObTabletID &tablet_i return ret; } -int ObDDLSSTableRedoWriter::start_ddl_redo(const ObITable::TableKey &table_key) +int ObDDLSSTableRedoWriter::start_ddl_redo(const ObITable::TableKey &table_key, ObDDLKvMgrHandle &ddl_kv_mgr_handle) { int ret = OB_SUCCESS; ObLS *ls = nullptr; ObDDLStartLog log; - ObDDLKvMgrHandle ddl_kv_mgr_handle; - palf::SCN tmp_scn; + ddl_kv_mgr_handle.reset(); + SCN tmp_scn; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObDDLSSTableRedoWriter has not been inited", K(ret)); @@ -1017,7 +1017,7 @@ int ObDDLSSTableRedoWriter::write_prepare_log(const ObITable::TableKey &table_ke const int64_t table_id, const int64_t execution_id, const int64_t ddl_task_id, - palf::SCN &prepare_scn) + SCN &prepare_scn) { int ret = OB_SUCCESS; @@ -1069,9 +1069,7 @@ int ObDDLSSTableRedoWriter::write_prepare_log(const ObITable::TableKey &table_ke ret = OB_ERR_SYS; LOG_WARN("srv rpc proxy or location service is null", K(ret), KP(srv_rpc_proxy)); } else if (OB_FAIL(srv_rpc_proxy->to(leader_addr_).remote_write_ddl_prepare_log(arg, log_ns))) { - if (OB_TASK_EXPIRED == ret) { - ret = OB_SUCCESS; - } else { + if (OB_TASK_EXPIRED != ret) { LOG_WARN("fail to remote write ddl redo log", K(ret), K(arg)); } } else if (OB_FAIL(prepare_scn.convert_for_tx(log_ns))) { @@ -1082,7 +1080,7 @@ int ObDDLSSTableRedoWriter::write_prepare_log(const ObITable::TableKey &table_ke } int ObDDLSSTableRedoWriter::write_commit_log(const ObITable::TableKey &table_key, - const palf::SCN &prepare_scn) + const SCN &prepare_scn) { int ret = OB_SUCCESS; ObLS *ls = nullptr; @@ -1091,7 +1089,7 @@ int ObDDLSSTableRedoWriter::write_commit_log(const ObITable::TableKey &table_key if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObDDLSSTableRedoWriter has not been inited", K(ret)); - } else if (OB_UNLIKELY(!table_key.is_valid() || palf::SCN::min_scn() == start_scn_ || !prepare_scn.is_valid())) { + } else if (OB_UNLIKELY(!table_key.is_valid() || SCN::min_scn() == start_scn_ || !prepare_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(table_key), K(start_scn_), K(prepare_scn)); } else if (OB_FAIL(log.init(table_key, get_start_scn(), prepare_scn))) { diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.h b/src/storage/ddl/ob_ddl_redo_log_writer.h index 164f44495..7e78774db 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.h +++ b/src/storage/ddl/ob_ddl_redo_log_writer.h @@ -13,7 +13,7 @@ #ifndef OCEANBASE_STORAGE_OB_DDL_REDO_LOG_WRITER_H #define OCEANBASE_STORAGE_OB_DDL_REDO_LOG_WRITER_H #include "common/ob_tablet_id.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_ls_id.h" #include "storage/ddl/ob_ddl_clog.h" #include "storage/ddl/ob_ddl_struct.h" @@ -166,7 +166,7 @@ public: bool is_valid() const { return nullptr != cb_ && scn_.is_valid(); } public: ObDDLMacroBlockClogCb *cb_; - palf::SCN scn_; + share::SCN scn_; }; class ObDDLCommitLogHandle final @@ -176,10 +176,10 @@ public: ~ObDDLCommitLogHandle(); int wait(const int64_t timeout = ObDDLRedoLogHandle::DDL_REDO_LOG_TIMEOUT); void reset(); - palf::SCN get_commit_scn() const { return commit_scn_; } + share::SCN get_commit_scn() const { return commit_scn_; } public: ObDDLClogCb *cb_; - palf::SCN commit_scn_; + share::SCN commit_scn_; }; class ObDDLRedoLogWriter final @@ -193,7 +193,7 @@ public: const blocksstable::MacroBlockId ¯o_block_id, char *buffer, ObDDLRedoLogHandle &handle); - int write_ddl_start_log(const ObDDLStartLog &log, logservice::ObLogHandler *log_handler, palf::SCN &start_scn); + int write_ddl_start_log(const ObDDLStartLog &log, logservice::ObLogHandler *log_handler, share::SCN &start_scn); template int write_ddl_finish_log(const T &log, const ObDDLClogType clog_type, @@ -237,7 +237,7 @@ public: ObDDLSSTableRedoWriter(); ~ObDDLSSTableRedoWriter(); int init(const share::ObLSID &ls_id, const ObTabletID &tablet_id); - int start_ddl_redo(const ObITable::TableKey &table_key); + int start_ddl_redo(const ObITable::TableKey &table_key, ObDDLKvMgrHandle &ddl_kv_mgr_handle); int write_redo_log(const blocksstable::ObDDLMacroBlockRedoInfo &redo_info, const blocksstable::MacroBlockId ¯o_block_id); int wait_redo_log_finish(const blocksstable::ObDDLMacroBlockRedoInfo &redo_info, @@ -246,18 +246,18 @@ public: const int64_t table_id, const int64_t execution_id, const int64_t ddl_task_id, - palf::SCN &prepare_scn); + share::SCN &prepare_scn); int write_commit_log(const ObITable::TableKey &table_key, - const palf::SCN &prepare_scn); - OB_INLINE void set_start_scn(const palf::SCN &start_scn) { start_scn_.atomic_set(start_scn); } - OB_INLINE palf::SCN get_start_scn() const { return start_scn_.atomic_get(); } + const share::SCN &prepare_scn); + OB_INLINE void set_start_scn(const share::SCN &start_scn) { start_scn_.atomic_set(start_scn); } + OB_INLINE share::SCN get_start_scn() const { return start_scn_.atomic_get(); } private: bool need_remote_write(int ret_code); int switch_to_remote_write(); private: bool is_inited_; bool remote_write_; - palf::SCN start_scn_; + share::SCN start_scn_; ObLSHandle ls_handle_; ObTabletHandle tablet_handle_; ObDDLRedoLogHandle ddl_redo_handle_; diff --git a/src/storage/ddl/ob_ddl_struct.cpp b/src/storage/ddl/ob_ddl_struct.cpp index 6b7d1535c..5653af6be 100644 --- a/src/storage/ddl/ob_ddl_struct.cpp +++ b/src/storage/ddl/ob_ddl_struct.cpp @@ -13,7 +13,7 @@ #define USING_LOG_PREFIX STORAGE #include "ob_ddl_struct.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/blocksstable/ob_block_manager.h" #include "storage/blocksstable/ob_block_sstable_struct.h" #include "storage/blocksstable/ob_index_block_builder.h" @@ -136,8 +136,8 @@ ObDDLKV::ObDDLKV() max_scn_(), pending_cnt_(0), cluster_version_(0), ref_cnt_(0), sstable_index_builder_(nullptr), index_block_rebuilder_(nullptr), is_rebuilder_closed_(false) { - min_scn_ = palf::SCN::max_scn(); - freeze_scn_ = palf::SCN::max_scn(); + min_scn_ = SCN::max_scn(); + freeze_scn_ = SCN::max_scn(); } ObDDLKV::~ObDDLKV() @@ -147,9 +147,9 @@ ObDDLKV::~ObDDLKV() int ObDDLKV::init(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, - const palf::SCN &ddl_start_scn, + const SCN &ddl_start_scn, const int64_t snapshot_version, - const palf::SCN &last_freezed_scn, + const SCN &last_freezed_scn, const int64_t cluster_version) { @@ -244,6 +244,7 @@ int ObDDLKV::set_macro_block(const ObDDLMacroBlock ¯o_block) ObDDLTableMergeDagParam param; param.ls_id_ = ls_id_; param.tablet_id_ = tablet_id_; + param.start_scn_ = ddl_start_scn_; int tmp_ret = OB_SUCCESS; if (OB_TMP_FAIL(compaction::ObScheduleDagFunc::schedule_ddl_table_merge_dag(param))) { LOG_WARN("try schedule ddl merge dag failed when ddl kv is full ", @@ -273,15 +274,15 @@ int ObDDLKV::set_macro_block(const ObDDLMacroBlock ¯o_block) } else if (OB_FAIL(ddl_blocks_.push_back(macro_block.block_handle_))) { LOG_WARN("push back block handle failed", K(ret), K(macro_block.block_handle_)); } else { - min_scn_ = palf::SCN::min(min_scn_, macro_block.scn_); - max_scn_ = palf::SCN::max(max_scn_, macro_block.scn_); + min_scn_ = SCN::min(min_scn_, macro_block.scn_); + max_scn_ = SCN::max(max_scn_, macro_block.scn_); LOG_INFO("succeed to set macro block into ddl kv", K(macro_block)); } } return ret; } -int ObDDLKV::freeze(const palf::SCN &freeze_scn) +int ObDDLKV::freeze(const SCN &freeze_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_inited_)) { @@ -373,7 +374,7 @@ int ObDDLKV::wait_pending() } else if (OB_FAIL(ls_service->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { LOG_WARN("get ls handle failed", K(ret), K(ls_id_)); } else { - palf::SCN max_decided_scn; + SCN max_decided_scn; bool wait_ls_ts = true; bool wait_ddl_redo = true; const int64_t abs_timeout_ts = ObTimeUtility::fast_current_time() + 1000L * 1000L * 10L; @@ -383,7 +384,7 @@ int ObDDLKV::wait_pending() LOG_WARN("get max decided log ts failed", K(ret), K(ls_id_)); } else { // max_decided_scn is the left border scn - 1 - wait_ls_ts = palf::SCN::plus(max_decided_scn, 1) < freeze_scn_; + wait_ls_ts = SCN::plus(max_decided_scn, 1) < freeze_scn_; } } if (OB_SUCC(ret)) { @@ -509,7 +510,7 @@ int ObDDLKVsHandle::get_ddl_kv(const int64_t idx, ObDDLKV *&kv) return ret; } -ObDDLKVPendingGuard::ObDDLKVPendingGuard(ObTablet *tablet, const palf::SCN &scn) +ObDDLKVPendingGuard::ObDDLKVPendingGuard(ObTablet *tablet, const SCN &scn) : tablet_(tablet), kv_handle_(), ret_(OB_SUCCESS) { int ret = OB_SUCCESS; diff --git a/src/storage/ddl/ob_ddl_struct.h b/src/storage/ddl/ob_ddl_struct.h index 8c3136e80..d0191b1f4 100644 --- a/src/storage/ddl/ob_ddl_struct.h +++ b/src/storage/ddl/ob_ddl_struct.h @@ -15,7 +15,7 @@ #include "lib/hash/ob_hashmap.h" #include "lib/lock/ob_mutex.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/ob_i_table.h" #include "storage/tablet/ob_tablet.h" #include "storage/blocksstable/ob_block_sstable_struct.h" @@ -62,8 +62,8 @@ public: ObDDLMacroHandle block_handle_; blocksstable::ObLogicMacroBlockId logic_id_; blocksstable::ObDDLMacroBlockType block_type_; - palf::SCN ddl_start_scn_; - palf::SCN scn_; + share::SCN ddl_start_scn_; + share::SCN scn_; const char *buf_; int64_t size_; }; @@ -76,20 +76,20 @@ public: ~ObDDLKV(); int init(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, - const palf::SCN &ddl_start_log_ts, + const share::SCN &ddl_start_log_ts, const int64_t snapshot_version, - const palf::SCN &last_freezed_scn, + const share::SCN &last_freezed_scn, const int64_t cluster_version); void destroy(); int set_macro_block(const ObDDLMacroBlock ¯o_block); - int freeze(const palf::SCN &freeze_scn); + int freeze(const share::SCN &freeze_scn); bool is_freezed() const { return ATOMIC_LOAD(&is_freezed_); } int close(); bool is_closed() const { return is_closed_; } - palf::SCN get_min_scn() const { return min_scn_; } - palf::SCN get_freeze_scn() const { return freeze_scn_; } - palf::SCN get_ddl_start_scn() const { return ddl_start_scn_; } + share::SCN get_min_scn() const { return min_scn_; } + share::SCN get_freeze_scn() const { return freeze_scn_; } + share::SCN get_ddl_start_scn() const { return ddl_start_scn_; } int64_t get_macro_block_cnt() const { return ddl_blocks_.count(); } void inc_pending_cnt(); // used by ddl kv pending guard void dec_pending_cnt(); @@ -110,16 +110,16 @@ private: bool is_inited_; share::ObLSID ls_id_; common::ObTabletID tablet_id_; - palf::SCN ddl_start_scn_; // the log ts of ddl start log + share::SCN ddl_start_scn_; // the log ts of ddl start log int64_t snapshot_version_; // the snapshot version for major sstable which is completed by ddl common::TCRWLock lock_; // lock for ddl_blocks_ and freeze_log_ts_ common::ObArenaAllocator allocator_; bool is_freezed_; bool is_closed_; - palf::SCN last_freezed_scn_; // the freezed log ts of last ddl kv. the log ts range of this ddl kv is (last_freezed_log_ts_, freeze_log_ts_] - palf::SCN min_scn_; // the min log ts of macro blocks - palf::SCN max_scn_; // the max log ts of macro blocks - palf::SCN freeze_scn_; // ddl kv refuse data larger than freeze log ts, freeze_log_ts >= max_log_ts + share::SCN last_freezed_scn_; // the freezed log ts of last ddl kv. the log ts range of this ddl kv is (last_freezed_log_ts_, freeze_log_ts_] + share::SCN min_scn_; // the min log ts of macro blocks + share::SCN max_scn_; // the max log ts of macro blocks + share::SCN freeze_scn_; // ddl kv refuse data larger than freeze log ts, freeze_log_ts >= max_log_ts int64_t pending_cnt_; // the amount of kvs that are replaying int64_t cluster_version_; int64_t ref_cnt_; @@ -161,13 +161,13 @@ class ObDDLKVPendingGuard final public: static int set_macro_block(ObTablet *tablet, const ObDDLMacroBlock ¯o_block); public: - ObDDLKVPendingGuard(ObTablet *tablet, const palf::SCN &scn); + ObDDLKVPendingGuard(ObTablet *tablet, const share::SCN &scn); ~ObDDLKVPendingGuard(); int get_ret() const { return ret_; } int get_ddl_kv(ObDDLKV *&kv); private: ObTablet *tablet_; - palf::SCN scn_; + share::SCN scn_; ObDDLKVHandle kv_handle_; int ret_; }; diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp index 5fb32a09a..2f93dfb18 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx.cpp @@ -204,6 +204,7 @@ ObSSTableInsertTabletContext::~ObSSTableInsertTabletContext() allocator_.free(index_builder_); index_builder_ = nullptr; } + ddl_kv_mgr_handle_.reset(); allocator_.reset(); } @@ -262,7 +263,7 @@ int ObSSTableInsertTabletContext::update(const int64_t snapshot_version) LOG_WARN("invalid argument", K(ret), K(table_key)); } else if (data_sstable_redo_writer_.get_start_scn().is_valid()) { // ddl start log is already written, do nothing - } else if (OB_FAIL(data_sstable_redo_writer_.start_ddl_redo(table_key))) { + } else if (OB_FAIL(data_sstable_redo_writer_.start_ddl_redo(table_key, ddl_kv_mgr_handle_))) { LOG_WARN("fail write start log", K(ret), K(table_key), K(build_param_)); } } @@ -326,7 +327,7 @@ int ObSSTableInsertTabletContext::build_sstable_slice( ObDataStoreDesc data_desc; const int64_t tenant_id = MTL_ID(); int64_t snapshot_version = 0; - palf::SCN snapshot_scn; + SCN snapshot_scn; { lib::ObMutexGuard guard(mutex_); snapshot_version = build_param_.snapshot_version_; @@ -356,7 +357,7 @@ int ObSSTableInsertTabletContext::build_sstable_slice( if (OB_ENTRY_NOT_EXIST != ret) { LOG_WARN("get freeze info failed", K(ret), K(tablet_id)); } else { - frozen_status.frozen_scn_ = palf::SCN::base_scn(); + frozen_status.frozen_scn_ = SCN::base_scn(); ret = OB_SUCCESS; } } @@ -655,7 +656,7 @@ int ObSSTableInsertTabletContext::create_sstable_with_clog( share::schema::ObMultiVersionSchemaService *schema_service = nullptr; const share::schema::ObTableSchema *table_schema = nullptr; const uint64_t tenant_id = MTL_ID(); - palf::SCN prepare_scn; + SCN prepare_scn; ObSchemaGetterGuard schema_guard; if (OB_UNLIKELY(!table_key.is_valid())) { ret = OB_INVALID_ARGUMENT; @@ -681,7 +682,12 @@ int ObSSTableInsertTabletContext::create_sstable_with_clog( build_param_.execution_id_, build_param_.ddl_task_id_, prepare_scn))) { - LOG_WARN("fail write ddl prepare log", K(ret), K(table_key)); + if (OB_TASK_EXPIRED == ret) { + LOG_INFO("ddl task expired, but return success", K(ret), K(table_key), K(build_param_)); + ret = OB_SUCCESS; + } else { + LOG_WARN("fail write ddl prepare log", K(ret), K(table_key)); + } } else { DEBUG_SYNC(AFTER_REMOTE_WRITE_DDL_PREPARE_LOG); ObTabletHandle tablet_handle; @@ -689,19 +695,27 @@ int ObSSTableInsertTabletContext::create_sstable_with_clog( ObLS *ls = ls_handle_.get_ls(); const ObLSID &ls_id = ls->get_ls_id(); const ObTabletID &tablet_id = tablet->get_tablet_meta().tablet_id_; - const palf::SCN &ddl_start_scn = data_sstable_redo_writer_.get_start_scn(); + const SCN &ddl_start_scn = data_sstable_redo_writer_.get_start_scn(); if (OB_FAIL(ls->get_tablet(tablet_id, tablet_handle))) { LOG_WARN("get tablet failed", K(ret)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) { - LOG_WARN("get ddl kv manager failed", K(ret)); + LOG_WARN("get ddl kv manager failed", K(ret), K(ls_id), K(tablet_id)); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_prepare(ddl_start_scn, prepare_scn, table_schema->get_table_id(), build_param_.execution_id_, build_param_.ddl_task_id_))) { - LOG_WARN("failed to do ddl kv prepare", K(ret), K(ddl_start_scn), K(prepare_scn), K(build_param_)); + if (OB_TASK_EXPIRED == ret) { + LOG_INFO("ddl task expired, but return success", K(ret), K(ls_id), K(tablet_id), + K(ddl_start_scn), "new_ddl_start_scn", ddl_kv_mgr_handle.get_obj()->get_start_scn()); + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to do ddl kv prepare", K(ret), K(ddl_start_scn), K(prepare_scn), K(build_param_)); + } } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->wait_ddl_commit(ddl_start_scn, prepare_scn))) { if (OB_TASK_EXPIRED == ret) { + LOG_INFO("ddl task expired, but return success", K(ret), K(ls_id), K(tablet_id), + K(ddl_start_scn), "new_ddl_start_scn", ddl_kv_mgr_handle.get_obj()->get_start_scn()); ret = OB_SUCCESS; } else { LOG_WARN("failed to wait ddl kv commit", K(ret), K(ddl_start_scn), K(build_param_)); diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx.h b/src/storage/ddl/ob_direct_insert_sstable_ctx.h index ef3e16ae7..5ac4cbff9 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx.h +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx.h @@ -137,6 +137,7 @@ private: int64_t task_finish_count_; blocksstable::ObSSTableIndexBuilder *index_builder_; int64_t task_id_; + ObDDLKvMgrHandle ddl_kv_mgr_handle_; // for keeping ddl kv mgr alive }; struct ObSSTableInsertTableParam final diff --git a/src/storage/ddl/ob_tablet_barrier_log.cpp b/src/storage/ddl/ob_tablet_barrier_log.cpp index 4fbc05357..6e35896f2 100644 --- a/src/storage/ddl/ob_tablet_barrier_log.cpp +++ b/src/storage/ddl/ob_tablet_barrier_log.cpp @@ -12,10 +12,11 @@ #define USING_LOG_PREFIX STORAGE #include "ob_tablet_barrier_log.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { +using namespace share; namespace storage { ObTabletBarrierLogState::ObTabletBarrierLogState() @@ -51,7 +52,7 @@ void ObTabletBarrierLogState::reset() void ObTabletBarrierLogState::set_log_info( const ObTabletBarrierLogStateEnum state, - const palf::SCN &scn, + const SCN &scn, const int64_t schema_version) { state_ = state; diff --git a/src/storage/ddl/ob_tablet_barrier_log.h b/src/storage/ddl/ob_tablet_barrier_log.h index 8b4f3d2e3..0d5164a46 100644 --- a/src/storage/ddl/ob_tablet_barrier_log.h +++ b/src/storage/ddl/ob_tablet_barrier_log.h @@ -14,7 +14,7 @@ #define SRC_STORAGE_OB_TABLET_BARRIER_LOG_H_ #include "lib/utility/ob_print_utils.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -35,13 +35,13 @@ public: ~ObTabletBarrierLogState() = default; ObTabletBarrierLogStateEnum &get_state() { return state_; } - palf::SCN get_scn() const { return scn_; } + share::SCN get_scn() const { return scn_; } int64_t get_schema_version() const { return schema_version_; } void reset(); void set_log_info( const ObTabletBarrierLogStateEnum state, - const palf::SCN &scn, + const share::SCN &scn, const int64_t schema_version); NEED_SERIALIZE_AND_DESERIALIZE; TO_STRING_KV(K_(state)); @@ -49,7 +49,7 @@ private: ObTabletBarrierLogStateEnum to_persistent_state() const; private: ObTabletBarrierLogStateEnum state_; - palf::SCN scn_; + share::SCN scn_; int64_t schema_version_; }; } diff --git a/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp b/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp index cebed5930..fcb69b9ea 100644 --- a/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp +++ b/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp @@ -13,7 +13,7 @@ #define USING_LOG_PREFIX STORAGE #include "ob_tablet_ddl_kv_mgr.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_force_print_log.h" #include "storage/ddl/ob_ddl_struct.h" #include "storage/ddl/ob_ddl_merge_task.h" @@ -24,7 +24,7 @@ using namespace oceanbase::common; using namespace oceanbase::blocksstable; -using namespace oceanbase::palf; +using namespace oceanbase::share; using namespace oceanbase::storage; ObTabletDDLKvMgr::ObTabletDDLKvMgr() @@ -88,7 +88,7 @@ int ObTabletDDLKvMgr::init(const share::ObLSID &ls_id, const common::ObTabletID // ddl start from checkpoint // keep ddl sstable table -int ObTabletDDLKvMgr::ddl_start(const ObITable::TableKey &table_key, const palf::SCN &start_scn, const int64_t cluster_version, const palf::SCN &checkpoint_scn) +int ObTabletDDLKvMgr::ddl_start(const ObITable::TableKey &table_key, const SCN &start_scn, const int64_t cluster_version, const SCN &checkpoint_scn) { int ret = OB_SUCCESS; bool is_brand_new = false; @@ -120,7 +120,7 @@ int ObTabletDDLKvMgr::ddl_start(const ObITable::TableKey &table_key, const palf: table_key_ = table_key; cluster_version_ = cluster_version; start_scn_ = start_scn; - max_freeze_scn_ = palf::SCN::max(start_scn, checkpoint_scn); + max_freeze_scn_ = SCN::max(start_scn, checkpoint_scn); } if (OB_SUCC(ret) && !checkpoint_scn.is_valid()) { // remove ddl sstable if exists and flush ddl start log ts and snapshot version into tablet meta @@ -132,8 +132,8 @@ int ObTabletDDLKvMgr::ddl_start(const ObITable::TableKey &table_key, const palf: return ret; } -int ObTabletDDLKvMgr::ddl_prepare(const palf::SCN &start_scn, - const palf::SCN &prepare_scn, +int ObTabletDDLKvMgr::ddl_prepare(const SCN &start_scn, + const SCN &prepare_scn, const uint64_t table_id, const int64_t execution_id, const int64_t ddl_task_id) @@ -147,6 +147,7 @@ int ObTabletDDLKvMgr::ddl_prepare(const palf::SCN &start_scn, ret = OB_STATE_NOT_MATCH; LOG_WARN("ddl not started", K(ret)); } else if (start_scn < start_scn_) { + ret = OB_TASK_EXPIRED; LOG_INFO("skip ddl prepare log", K(start_scn), K(start_scn_), K(ls_id_), K(tablet_id_)); } else if (OB_FAIL(freeze_ddl_kv(prepare_scn))) { LOG_WARN("freeze ddl kv failed", K(ret), K(prepare_scn)); @@ -160,6 +161,7 @@ int ObTabletDDLKvMgr::ddl_prepare(const palf::SCN &start_scn, param.tablet_id_ = tablet_id_; param.rec_scn_ = prepare_scn; param.is_commit_ = true; + param.start_scn_ = start_scn; param.table_id_ = table_id; param.execution_id_ = execution_id_; param.ddl_task_id_ = ddl_task_id_; @@ -185,7 +187,7 @@ int ObTabletDDLKvMgr::ddl_prepare(const palf::SCN &start_scn, return ret; } -int ObTabletDDLKvMgr::ddl_commit(const palf::SCN &start_scn, const palf::SCN &prepare_scn, const bool is_replay) +int ObTabletDDLKvMgr::ddl_commit(const SCN &start_scn, const SCN &prepare_scn, const bool is_replay) { int ret = OB_SUCCESS; ObLSHandle ls_handle; @@ -195,6 +197,7 @@ int ObTabletDDLKvMgr::ddl_commit(const palf::SCN &start_scn, const palf::SCN &pr } else if (is_commit_success_) { LOG_INFO("ddl commit already succeed", K(ls_id_), K(tablet_id_), K(table_id_)); } else if (start_scn < start_scn_) { + ret = OB_TASK_EXPIRED; LOG_INFO("skip ddl commit log", K(start_scn), K(start_scn_), K(ls_id_), K(tablet_id_)); } else if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { LOG_WARN("failed to get log stream", K(ret), K(ls_id_)); @@ -204,6 +207,7 @@ int ObTabletDDLKvMgr::ddl_commit(const palf::SCN &start_scn, const palf::SCN &pr param.tablet_id_ = tablet_id_; param.rec_scn_ = prepare_scn; param.is_commit_ = true; + param.start_scn_ = start_scn; param.table_id_ = table_id_; param.execution_id_ = execution_id_; param.ddl_task_id_ = ddl_task_id_; @@ -217,21 +221,21 @@ int ObTabletDDLKvMgr::ddl_commit(const palf::SCN &start_scn, const palf::SCN &pr } else { ret = OB_EAGAIN; // until major sstable is ready } - if (OB_FAIL(ret) && is_replay) { - if (OB_TABLET_NOT_EXIST == ret) { - ret = OB_SUCCESS; // think as succcess for replay - } else { - if (REACH_TIME_INTERVAL(10L * 1000L * 1000L)) { - LOG_INFO("replay ddl commit", K(ret), K(ls_id_), K(tablet_id_), K(start_scn_), K(prepare_scn), K(max_freeze_scn_)); - } - ret = OB_EAGAIN; // retry by replay service + } + if (OB_FAIL(ret) && is_replay) { + if (OB_TABLET_NOT_EXIST == ret || OB_TASK_EXPIRED == ret) { + ret = OB_SUCCESS; // think as succcess for replay + } else { + if (REACH_TIME_INTERVAL(10L * 1000L * 1000L)) { + LOG_INFO("replay ddl commit", K(ret), K(ls_id_), K(tablet_id_), K(start_scn_), K(start_scn), K(prepare_scn), K(max_freeze_scn_)); } + ret = OB_EAGAIN; // retry by replay service } } return ret; } -int ObTabletDDLKvMgr::wait_ddl_commit(const palf::SCN &start_scn, const palf::SCN &prepare_scn) +int ObTabletDDLKvMgr::wait_ddl_commit(const SCN &start_scn, const SCN &prepare_scn) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_inited_)) { @@ -243,9 +247,6 @@ int ObTabletDDLKvMgr::wait_ddl_commit(const palf::SCN &start_scn, const palf::SC } else if (!is_started()) { ret = OB_STATE_NOT_MATCH; LOG_WARN("ddl not started", K(ret)); - } else if (start_scn < start_scn_) { - ret = OB_TASK_EXPIRED; - LOG_INFO("task expired, skip ddl commit log", K(start_scn), K(start_scn_), K(ls_id_), K(tablet_id_)); } else if (start_scn > start_scn_) { ret = OB_ERR_SYS; LOG_WARN("start log ts not match", K(ret), K(start_scn), K(start_scn_), K(ls_id_), K(tablet_id_)); @@ -309,7 +310,7 @@ int ObTabletDDLKvMgr::cleanup() return ret; } -int ObTabletDDLKvMgr::update_tablet(const palf::SCN &start_scn, const int64_t snapshot_version, const palf::SCN &ddl_checkpoint_scn) +int ObTabletDDLKvMgr::update_tablet(const SCN &start_scn, const int64_t snapshot_version, const SCN &ddl_checkpoint_scn) { int ret = OB_SUCCESS; ObLSHandle ls_handle; @@ -368,7 +369,7 @@ int ObTabletDDLKvMgr::get_ddl_param(ObTabletDDLParam &ddl_param) return ret; } -int ObTabletDDLKvMgr::get_freezed_ddl_kv(const palf::SCN &freeze_scn, ObDDLKVHandle &kv_handle) +int ObTabletDDLKvMgr::get_freezed_ddl_kv(const SCN &freeze_scn, ObDDLKVHandle &kv_handle) { int ret = OB_SUCCESS; kv_handle.reset(); @@ -430,7 +431,7 @@ int ObTabletDDLKvMgr::get_active_ddl_kv_impl(ObDDLKVHandle &kv_handle) return ret; } -int ObTabletDDLKvMgr::get_or_create_ddl_kv(const palf::SCN &scn, ObDDLKVHandle &kv_handle) +int ObTabletDDLKvMgr::get_or_create_ddl_kv(const SCN &scn, ObDDLKVHandle &kv_handle) { int ret = OB_SUCCESS; kv_handle.reset(); @@ -469,7 +470,7 @@ int ObTabletDDLKvMgr::get_or_create_ddl_kv(const palf::SCN &scn, ObDDLKVHandle & return ret; } -void ObTabletDDLKvMgr::try_get_ddl_kv_unlock(const palf::SCN &scn, ObDDLKV *&kv) +void ObTabletDDLKvMgr::try_get_ddl_kv_unlock(const SCN &scn, ObDDLKV *&kv) { int ret = OB_SUCCESS; if (get_count() > 0) { @@ -486,7 +487,7 @@ void ObTabletDDLKvMgr::try_get_ddl_kv_unlock(const palf::SCN &scn, ObDDLKV *&kv) } } -int ObTabletDDLKvMgr::freeze_ddl_kv(const palf::SCN &freeze_scn) +int ObTabletDDLKvMgr::freeze_ddl_kv(const SCN &freeze_scn) { int ret = OB_SUCCESS; ObDDLKVHandle kv_handle; @@ -521,14 +522,14 @@ int ObTabletDDLKvMgr::freeze_ddl_kv(const palf::SCN &freeze_scn) ret = OB_SUCCESS; } } else { - max_freeze_scn_ = palf::SCN::max(max_freeze_scn_, kv->get_freeze_scn()); + max_freeze_scn_ = SCN::max(max_freeze_scn_, kv->get_freeze_scn()); LOG_INFO("freeze ddl kv", "kv", *kv); } } return ret; } -int ObTabletDDLKvMgr::release_ddl_kvs(const palf::SCN &end_scn) +int ObTabletDDLKvMgr::release_ddl_kvs(const SCN &end_scn) { int ret = OB_SUCCESS; DEBUG_SYNC(BEFORE_RELEASE_DDL_KV); @@ -554,7 +555,7 @@ int ObTabletDDLKvMgr::release_ddl_kvs(const palf::SCN &end_scn) ret = OB_ERR_UNEXPECTED; LOG_WARN("ddl kv is null", K(ret), K(ls_id_), K(tablet_id_), KP(kv), K(i), K(head_), K(tail_)); } else if (kv->is_closed() && kv->get_freeze_scn() <= end_scn) { - const palf::SCN &freeze_scn = kv->get_freeze_scn(); + const SCN &freeze_scn = kv->get_freeze_scn(); free_ddl_kv(idx); ++head_; LOG_INFO("succeed to release ddl kv", K(ls_id_), K(tablet_id_), K(freeze_scn)); @@ -564,7 +565,7 @@ int ObTabletDDLKvMgr::release_ddl_kvs(const palf::SCN &end_scn) return ret; } -int ObTabletDDLKvMgr::get_ddl_kv_min_scn(palf::SCN &min_scn) +int ObTabletDDLKvMgr::get_ddl_kv_min_scn(SCN &min_scn) { int ret = OB_SUCCESS; TCRLockGuard guard(lock_); @@ -580,7 +581,7 @@ int ObTabletDDLKvMgr::get_ddl_kv_min_scn(palf::SCN &min_scn) ret = OB_ERR_UNEXPECTED; LOG_WARN("ddl kv is null", K(ret), K(ls_id_), K(tablet_id_), KP(kv), K(i), K(head_), K(tail_)); } else { - min_scn = palf::SCN::min(min_scn, kv->get_min_scn()); + min_scn = SCN::min(min_scn, kv->get_min_scn()); } } } diff --git a/src/storage/ddl/ob_tablet_ddl_kv_mgr.h b/src/storage/ddl/ob_tablet_ddl_kv_mgr.h index 743bf0fe3..5ec759b2d 100644 --- a/src/storage/ddl/ob_tablet_ddl_kv_mgr.h +++ b/src/storage/ddl/ob_tablet_ddl_kv_mgr.h @@ -17,7 +17,7 @@ #include "lib/allocator/ob_concurrent_fifo_allocator.h" #include "lib/lock/ob_tc_rwlock.h" #include "lib/ob_define.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_ls_id.h" #include "storage/ob_i_table.h" @@ -37,19 +37,19 @@ public: ObTabletDDLKvMgr(); ~ObTabletDDLKvMgr(); int init(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); // init before memtable mgr - int ddl_start(const ObITable::TableKey &table_key, const palf::SCN &start_scn, const int64_t cluster_version, const palf::SCN &checkpoint_scn = palf::SCN::invalid_scn()); - int ddl_prepare(const palf::SCN &start_scn, const palf::SCN &commit_scn, const uint64_t table_id = 0, const int64_t execution_id = 0, const int64_t ddl_task_id = 0); // schedule build a major sstable - int ddl_commit(const palf::SCN &start_scn, const palf::SCN &prepare_scn, const bool is_replay); // try wait build major sstable - int wait_ddl_commit(const palf::SCN &start_scn, const palf::SCN &prepare_scn); + int ddl_start(const ObITable::TableKey &table_key, const share::SCN &start_scn, const int64_t cluster_version, const share::SCN &checkpoint_scn = share::SCN::invalid_scn()); + int ddl_prepare(const share::SCN &start_scn, const share::SCN &commit_scn, const uint64_t table_id = 0, const int64_t execution_id = 0, const int64_t ddl_task_id = 0); // schedule build a major sstable + int ddl_commit(const share::SCN &start_scn, const share::SCN &prepare_scn, const bool is_replay); // try wait build major sstable + int wait_ddl_commit(const share::SCN &start_scn, const share::SCN &prepare_scn); int get_ddl_param(ObTabletDDLParam &ddl_param); - int get_or_create_ddl_kv(const palf::SCN &scn, ObDDLKVHandle &kv_handle); // used in active ddl kv guard - int get_freezed_ddl_kv(const palf::SCN &freeze_scn, ObDDLKVHandle &kv_handle); // locate ddl kv with exeact freeze log ts + int get_or_create_ddl_kv(const share::SCN &scn, ObDDLKVHandle &kv_handle); // used in active ddl kv guard + int get_freezed_ddl_kv(const share::SCN &freeze_scn, ObDDLKVHandle &kv_handle); // locate ddl kv with exeact freeze log ts int get_ddl_kvs(const bool frozen_only, ObDDLKVsHandle &ddl_kvs_handle); // get all freeze ddl kvs - int freeze_ddl_kv(const palf::SCN &freeze_scn = palf::SCN::invalid_scn()); // freeze the active ddl kv, when memtable freeze or ddl commit - int release_ddl_kvs(const palf::SCN &rec_scn); // release persistent ddl kv, used in ddl merge task for free ddl kv + int freeze_ddl_kv(const share::SCN &freeze_scn = share::SCN::invalid_scn()); // freeze the active ddl kv, when memtable freeze or ddl commit + int release_ddl_kvs(const share::SCN &rec_scn); // release persistent ddl kv, used in ddl merge task for free ddl kv int check_has_effective_ddl_kv(bool &has_ddl_kv); // used in ddl log handler for checkpoint - int get_ddl_kv_min_scn(palf::SCN &min_scn); // for calculate rec_scn of ls - palf::SCN get_start_scn() const { return start_scn_; } + int get_ddl_kv_min_scn(share::SCN &min_scn); // for calculate rec_scn of ls + share::SCN get_start_scn() const { return start_scn_; } bool is_started() const { return start_scn_.is_valid(); } int set_commit_success(); bool is_commit_success() const { return is_commit_success_; } @@ -69,8 +69,8 @@ private: int alloc_ddl_kv(ObDDLKV *&kv); void free_ddl_kv(const int64_t idx); int get_active_ddl_kv_impl(ObDDLKVHandle &kv_handle); - void try_get_ddl_kv_unlock(const palf::SCN &scn, ObDDLKV *&kv); - int update_tablet(const palf::SCN &start_scn, const int64_t snapshot_version, const palf::SCN &ddl_checkpoint_scn); + void try_get_ddl_kv_unlock(const share::SCN &scn, ObDDLKV *&kv); + int update_tablet(const share::SCN &start_scn, const int64_t snapshot_version, const share::SCN &ddl_checkpoint_scn); void destroy(); private: static const int64_t MAX_DDL_KV_CNT_IN_STORAGE = 64; @@ -80,8 +80,8 @@ private: common::ObTabletID tablet_id_; ObITable::TableKey table_key_; int64_t cluster_version_; - palf::SCN start_scn_; - palf::SCN max_freeze_scn_; + share::SCN start_scn_; + share::SCN max_freeze_scn_; uint64_t table_id_; // used for ddl checksum int64_t execution_id_; // used for ddl checksum int64_t ddl_task_id_; // used for ddl checksum diff --git a/src/storage/high_availability/ob_ls_complete_migration.cpp b/src/storage/high_availability/ob_ls_complete_migration.cpp index a139a7978..674359d2d 100644 --- a/src/storage/high_availability/ob_ls_complete_migration.cpp +++ b/src/storage/high_availability/ob_ls_complete_migration.cpp @@ -798,8 +798,8 @@ ObStartCompleteMigrationTask::ObStartCompleteMigrationTask() is_inited_(false), ls_handle_(), ctx_(nullptr), - log_sync_scn_(palf::SCN::min_scn()), - max_minor_end_scn_(palf::SCN::min_scn()) + log_sync_scn_(SCN::min_scn()), + max_minor_end_scn_(SCN::min_scn()) { } @@ -881,8 +881,8 @@ int ObStartCompleteMigrationTask::wait_log_sync_() bool is_log_sync = false; bool is_need_rebuild = false; bool is_cancel = false; - palf::SCN last_end_scn; - palf::SCN current_end_scn; + SCN last_end_scn; + SCN current_end_scn; const int64_t OB_CHECK_LOG_SYNC_INTERVAL = 200 * 1000; // 200ms const int64_t CLOG_IN_SYNC_DELAY_TIMEOUT = 30 * 60 * 1000 * 1000; // 30 min bool need_wait = true; @@ -979,8 +979,8 @@ int ObStartCompleteMigrationTask::wait_log_replay_sync_() logservice::ObLogService *log_service = nullptr; bool wait_log_replay_success = false; bool is_cancel = false; - palf::SCN current_replay_scn; - palf::SCN last_replay_scn; + SCN current_replay_scn; + SCN last_replay_scn; const int64_t OB_CHECK_LOG_REPLAY_INTERVAL = 200 * 1000; // 200ms const int64_t CLOG_IN_REPLAY_DELAY_TIMEOUT = 30 * 60 * 1000 * 1000L; // 30 min bool need_wait = false; @@ -1010,7 +1010,7 @@ int ObStartCompleteMigrationTask::wait_log_replay_sync_() STORAGE_LOG(WARN, "task is cancelled", K(ret), K(*this)); } else if (OB_FAIL(ls->get_max_decided_scn(current_replay_scn))) { LOG_WARN("failed to get current replay log ts", K(ret), KPC(ctx_)); - } else if (current_replay_scn >= log_sync_scn_) { + } else if (current_replay_scn.convert_to_ts() + IS_REPLAY_DONE_THRESHOLD_US >= log_sync_scn_.convert_to_ts()) { wait_log_replay_success = true; const int64_t cost_ts = ObTimeUtility::current_time() - wait_replay_start_ts; LOG_INFO("wait replay log ts ns success, stop wait", "arg", ctx_->arg_, K(cost_ts)); @@ -1352,7 +1352,7 @@ int ObStartCompleteMigrationTask::wait_ls_checkpoint_scn_push_() LOG_WARN("checkpoint executor should not be NULL", K(ret), KPC(ctx_), KP(checkpoint_executor)); } else { const int64_t wait_checkpoint_push_start_ts = ObTimeUtility::current_time(); - palf::SCN checkpoint_scn = palf::SCN::min_scn(); + SCN checkpoint_scn = SCN::min_scn(); while (OB_SUCC(ret)) { if (ctx_->is_failed()) { ret = OB_CANCELED; diff --git a/src/storage/high_availability/ob_ls_complete_migration.h b/src/storage/high_availability/ob_ls_complete_migration.h index 6f4e40c8f..f9aa994b3 100644 --- a/src/storage/high_availability/ob_ls_complete_migration.h +++ b/src/storage/high_availability/ob_ls_complete_migration.h @@ -188,11 +188,12 @@ private: int record_server_event_(); private: + static const int64_t IS_REPLAY_DONE_THRESHOLD_US = 3L * 1000 * 1000L; bool is_inited_; ObLSHandle ls_handle_; ObLSCompleteMigrationCtx *ctx_; - palf::SCN log_sync_scn_; - palf::SCN max_minor_end_scn_; + share::SCN log_sync_scn_; + share::SCN max_minor_end_scn_; DISALLOW_COPY_AND_ASSIGN(ObStartCompleteMigrationTask); }; diff --git a/src/storage/high_availability/ob_ls_migration.cpp b/src/storage/high_availability/ob_ls_migration.cpp index c986953df..b8fdae33e 100644 --- a/src/storage/high_availability/ob_ls_migration.cpp +++ b/src/storage/high_availability/ob_ls_migration.cpp @@ -1081,7 +1081,7 @@ int ObStartMigrationTask::choose_src_() const share::ObLSID &ls_id = ctx_->arg_.ls_id_; ObStorageHASrcInfo src_info; obrpc::ObCopyLSInfo ls_info; - palf::SCN local_clog_checkpoint_scn = palf::SCN::min_scn(); + SCN local_clog_checkpoint_scn = SCN::min_scn(); if (OB_FAIL(get_local_ls_checkpoint_scn_(local_clog_checkpoint_scn))) { LOG_WARN("failed to get local ls checkpoint ts", K(ret)); } else if (OB_FAIL(src_provider.init(tenant_id, ctx_->arg_.type_, storage_rpc_))) { @@ -1134,7 +1134,7 @@ int ObStartMigrationTask::fetch_ls_info_(const uint64_t tenant_id, const share:: return ret; } -int ObStartMigrationTask::get_local_ls_checkpoint_scn_(palf::SCN &local_checkpoint_scn) +int ObStartMigrationTask::get_local_ls_checkpoint_scn_(SCN &local_checkpoint_scn) { int ret = OB_SUCCESS; local_checkpoint_scn.set_min(); diff --git a/src/storage/high_availability/ob_ls_migration.h b/src/storage/high_availability/ob_ls_migration.h index 9f95e8258..154de1f5c 100644 --- a/src/storage/high_availability/ob_ls_migration.h +++ b/src/storage/high_availability/ob_ls_migration.h @@ -20,7 +20,7 @@ #include "share/scheduler/ob_sys_task_stat.h" #include "observer/ob_rpc_processor_simple.h" #include "share/scheduler/ob_dag_scheduler.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/ob_storage_rpc.h" #include "ob_storage_ha_struct.h" #include "ob_physical_copy_task.h" @@ -47,7 +47,7 @@ public: public: uint64_t tenant_id_; ObMigrationOpArg arg_; - palf::SCN local_clog_checkpoint_scn_; + share::SCN local_clog_checkpoint_scn_; int64_t local_rebuild_seq_; int64_t start_ts_; @@ -236,7 +236,7 @@ private: int choose_src_(); int fetch_ls_info_(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObAddr &member_addr, obrpc::ObCopyLSInfo &ls_info); - int get_local_ls_checkpoint_scn_(palf::SCN &local_checkpoint_scn); + int get_local_ls_checkpoint_scn_(share::SCN &local_checkpoint_scn); int try_remove_member_list_(); int get_tablet_id_array_(common::ObIArray &tablet_id_array); int check_ls_need_copy_data_(bool &need_copy); diff --git a/src/storage/high_availability/ob_ls_prepare_migration.cpp b/src/storage/high_availability/ob_ls_prepare_migration.cpp index 605812a92..fb738c481 100644 --- a/src/storage/high_availability/ob_ls_prepare_migration.cpp +++ b/src/storage/high_availability/ob_ls_prepare_migration.cpp @@ -30,7 +30,7 @@ ObLSPrepareMigrationCtx::ObLSPrepareMigrationCtx() task_id_(), start_ts_(0), finish_ts_(0), - log_sync_scn_(palf::SCN::min_scn()) + log_sync_scn_(SCN::min_scn()) { } @@ -812,8 +812,8 @@ int ObStartPrepareMigrationTask::wait_log_replay_sync_() logservice::ObLogService *log_service = nullptr; bool wait_log_replay_success = false; bool is_cancel = false; - palf::SCN current_replay_scn; - palf::SCN last_replay_scn; + SCN current_replay_scn; + SCN last_replay_scn; const int64_t OB_CHECK_LOG_SYNC_INTERVAL = 200 * 1000; // 200ms const int64_t CLOG_IN_SYNC_DELAY_TIMEOUT = 30 * 60 * 1000 * 1000L; // 30 min ObLSSavedInfo saved_info; @@ -1001,7 +1001,7 @@ int ObStartPrepareMigrationTask::wait_ls_checkpoint_scn_push_() ObLSHandle ls_handle; ObLS *ls = nullptr; checkpoint::ObCheckpointExecutor *checkpoint_executor = NULL; - palf::SCN checkpoint_scn; + SCN checkpoint_scn; const int64_t MAX_WAIT_INTERVAL_BY_CHECKPOINT_BY_FLUSH = GCONF._advance_checkpoint_timeout; const int64_t MAX_SLEEP_INTERVAL_MS = 5 * 1000 * 1000; //5s bool is_cancel = false; diff --git a/src/storage/high_availability/ob_ls_prepare_migration.h b/src/storage/high_availability/ob_ls_prepare_migration.h index f1225ec28..ee10d26d7 100644 --- a/src/storage/high_availability/ob_ls_prepare_migration.h +++ b/src/storage/high_availability/ob_ls_prepare_migration.h @@ -42,7 +42,7 @@ public: int64_t start_ts_; int64_t finish_ts_; - palf::SCN log_sync_scn_; + share::SCN log_sync_scn_; INHERIT_TO_STRING_KV( "ObIHADagNetCtx", ObIHADagNetCtx, diff --git a/src/storage/high_availability/ob_storage_ha_dag.cpp b/src/storage/high_availability/ob_storage_ha_dag.cpp index 2af15c0a2..24f5dd30b 100644 --- a/src/storage/high_availability/ob_storage_ha_dag.cpp +++ b/src/storage/high_availability/ob_storage_ha_dag.cpp @@ -14,7 +14,7 @@ #include "ob_storage_ha_dag.h" #include "observer/ob_server.h" #include "share/rc/ob_tenant_base.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "observer/ob_server_event_history_table_operator.h" namespace oceanbase @@ -685,8 +685,8 @@ int ObStorageHATaskUtils::check_ddl_sstable_need_copy_( } else if (OB_FAIL(ddl_sstable_array.get_table(param.table_key_, table_handle))) { LOG_WARN("failed to get table", K(ret), K(param), K(ddl_sstable_array)); } else if (!table_handle.is_valid()) { - const palf::SCN start_scn = ddl_sstable_array.get_table(0)->get_start_scn(); - const palf::SCN end_scn = ddl_sstable_array.get_table(ddl_sstable_array.count() - 1)->get_end_scn(); + const SCN start_scn = ddl_sstable_array.get_table(0)->get_start_scn(); + const SCN end_scn = ddl_sstable_array.get_table(ddl_sstable_array.count() - 1)->get_end_scn(); if (param.table_key_.scn_range_.start_scn_ >= start_scn && param.table_key_.scn_range_.end_scn_ <= end_scn) { need_copy = false; diff --git a/src/storage/high_availability/ob_storage_ha_reader.cpp b/src/storage/high_availability/ob_storage_ha_reader.cpp index a7dd341e3..ab97b49f5 100644 --- a/src/storage/high_availability/ob_storage_ha_reader.cpp +++ b/src/storage/high_availability/ob_storage_ha_reader.cpp @@ -15,7 +15,7 @@ #include "ob_storage_ha_reader.h" #include "storage/tx_storage/ob_ls_service.h" #include "share/rc/ob_tenant_base.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/blocksstable/ob_logic_macro_id.h" namespace oceanbase @@ -555,7 +555,7 @@ int ObCopyMacroBlockObProducer::init( const ObITable::TableKey &table_key, const ObCopyMacroRangeInfo ©_macro_range_info, const int64_t data_version, - const palf::SCN backfill_tx_scn) + const SCN backfill_tx_scn) { int ret = OB_SUCCESS; ObLSHandle ls_handle; @@ -1014,7 +1014,7 @@ int ObCopyTabletInfoObProducer::build_deleted_tablet_info_( tablet_info.param_.multi_version_start_ = 0; tablet_info.param_.snapshot_version_ = 0; tablet_info.param_.tx_data_.tablet_status_ = ObTabletStatus::DELETING; - tablet_info.param_.tx_data_.tx_scn_ = palf::SCN::min_scn(); + tablet_info.param_.tx_data_.tx_scn_ = SCN::min_scn(); if (OB_FAIL(tablet_info.param_.ha_status_.set_restore_status(restore_status))) { LOG_WARN("failed to set restore status", K(ret), K(restore_status)); @@ -1586,8 +1586,8 @@ int ObCopySSTableInfoObProducer::check_need_copy_sstable_( need_copy_sstable = true; } } else if (sstable->is_ddl_sstable()) { - const palf::SCN ddl_sstable_start_scn = tablet_sstable_info_.ddl_sstable_scn_range_.start_scn_; - const palf::SCN ddl_sstable_end_scn = tablet_sstable_info_.ddl_sstable_scn_range_.end_scn_; + const SCN ddl_sstable_start_scn = tablet_sstable_info_.ddl_sstable_scn_range_.start_scn_; + const SCN ddl_sstable_end_scn = tablet_sstable_info_.ddl_sstable_scn_range_.end_scn_; if (tablet_sstable_info_.ddl_sstable_scn_range_.is_empty()) { need_copy_sstable = false; } else if (sstable->get_key().scn_range_.start_scn_ >= ddl_sstable_end_scn) { diff --git a/src/storage/high_availability/ob_storage_ha_reader.h b/src/storage/high_availability/ob_storage_ha_reader.h index f45674d91..add301e41 100644 --- a/src/storage/high_availability/ob_storage_ha_reader.h +++ b/src/storage/high_availability/ob_storage_ha_reader.h @@ -167,7 +167,7 @@ public: const ObITable::TableKey &table_key, const ObCopyMacroRangeInfo ©_macro_range_info, const int64_t data_version, - const palf::SCN backfill_tx_scn); + const share::SCN backfill_tx_scn); int get_next_macro_block( blocksstable::ObBufferReader &data, ObCopyMacroBlockHeader ©_macro_block_header); diff --git a/src/storage/high_availability/ob_storage_ha_src_provider.cpp b/src/storage/high_availability/ob_storage_ha_src_provider.cpp index 7585f7a45..ad74d3ad0 100644 --- a/src/storage/high_availability/ob_storage_ha_src_provider.cpp +++ b/src/storage/high_availability/ob_storage_ha_src_provider.cpp @@ -16,6 +16,7 @@ #include "observer/ob_server_event_history_table_operator.h" namespace oceanbase { +using namespace share; namespace storage { ObStorageHASrcProvider::ObStorageHASrcProvider() @@ -47,7 +48,7 @@ int ObStorageHASrcProvider::init(const uint64_t tenant_id, const ObMigrationOpTy return ret; } -int ObStorageHASrcProvider::choose_ob_src(const share::ObLSID &ls_id, const palf::SCN &local_clog_checkpoint_scn, +int ObStorageHASrcProvider::choose_ob_src(const share::ObLSID &ls_id, const SCN &local_clog_checkpoint_scn, ObStorageHASrcInfo &src_info) { int ret = OB_SUCCESS; @@ -181,13 +182,13 @@ int ObStorageHASrcProvider::fetch_ls_meta_info_(const uint64_t tenant_id, const } int ObStorageHASrcProvider::inner_choose_ob_src_(const uint64_t tenant_id, const share::ObLSID &ls_id, - const palf::SCN &local_clog_checkpoint_scn, const common::ObIArray &addr_list, + const SCN &local_clog_checkpoint_scn, const common::ObIArray &addr_list, common::ObAddr &choosen_src_addr) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; int64_t choose_member_idx = -1; - palf::SCN max_clog_checkpoint_scn; + SCN max_clog_checkpoint_scn; for (int64_t i = 0; OB_SUCC(ret) && i < addr_list.count(); ++i) { const common::ObAddr &addr = addr_list.at(i); obrpc::ObFetchLSMetaInfoResp ls_info; diff --git a/src/storage/high_availability/ob_storage_ha_src_provider.h b/src/storage/high_availability/ob_storage_ha_src_provider.h index 196ddc7a3..49c9c8c03 100644 --- a/src/storage/high_availability/ob_storage_ha_src_provider.h +++ b/src/storage/high_availability/ob_storage_ha_src_provider.h @@ -25,7 +25,7 @@ public: ObStorageHASrcProvider(); virtual ~ObStorageHASrcProvider(); int init(const uint64_t tenant_id, const ObMigrationOpType::TYPE &type, storage::ObStorageRpc *storage_rpc); - int choose_ob_src(const share::ObLSID &ls_id, const palf::SCN &local_clog_checkpoint_scn, + int choose_ob_src(const share::ObLSID &ls_id, const share::SCN &local_clog_checkpoint_scn, ObStorageHASrcInfo &src_info); private: @@ -33,7 +33,7 @@ private: int fetch_ls_member_list_(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObAddr &addr, common::ObIArray &addr_list); int inner_choose_ob_src_(const uint64_t tenant_id, const share::ObLSID &ls_id, - const palf::SCN &local_clog_checkpoint_scn, const common::ObIArray &addr_list, + const share::SCN &local_clog_checkpoint_scn, const common::ObIArray &addr_list, common::ObAddr &choosen_src_addr); int fetch_ls_meta_info_(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObAddr &member_addr, obrpc::ObFetchLSMetaInfoResp &ls_meta_info); diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp index 08d92debf..457465593 100644 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp @@ -16,7 +16,7 @@ #include "share/rc/ob_tenant_base.h" #include "storage/tablet/ob_tablet_common.h" #include "storage/tablet/ob_tablet_create_delete_helper.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -694,8 +694,8 @@ int ObStorageHATabletsBuilder::get_need_copy_ddl_sstable_range_( need_copy_scn_range.start_scn_.set_min(); need_copy_scn_range.end_scn_.set_min(); } else { - const palf::SCN ddl_start_scn = tablet->get_tablet_meta().ddl_start_scn_; - const palf::SCN ddl_checkpoint_scn = tablet->get_tablet_meta().ddl_checkpoint_scn_; + const SCN ddl_start_scn = tablet->get_tablet_meta().ddl_start_scn_; + const SCN ddl_checkpoint_scn = tablet->get_tablet_meta().ddl_checkpoint_scn_; if (ddl_start_scn == ddl_checkpoint_scn) { need_copy_scn_range.start_scn_ = ddl_start_scn; need_copy_scn_range.end_scn_ = ddl_checkpoint_scn; @@ -703,7 +703,7 @@ int ObStorageHATabletsBuilder::get_need_copy_ddl_sstable_range_( bool ddl_checkpoint_pushed = !ddl_sstable_array.empty(); if (ddl_checkpoint_pushed) { need_copy_scn_range.start_scn_ = ddl_start_scn; - palf::SCN max_start_scn = palf::SCN::max_scn(); + SCN max_start_scn = SCN::max_scn(); if (OB_FAIL(get_ddl_sstable_max_start_scn_(ddl_sstable_array, max_start_scn))) { LOG_WARN("failed to get ddl sstable min start log ts", K(ret)); } else { @@ -714,12 +714,12 @@ int ObStorageHATabletsBuilder::get_need_copy_ddl_sstable_range_( need_copy_scn_range.end_scn_ = ddl_checkpoint_scn; } #ifdef ERRSIM - LOG_INFO("ddl checkpoint pushed", K(ddl_checkpoint_pushed), K(ddl_sstable_array), K(ddl_start_log_ts), K(ddl_checkpoint_ts)); + LOG_INFO("ddl checkpoint pushed", K(ddl_checkpoint_pushed), K(ddl_sstable_array), K(ddl_start_scn), K(ddl_checkpoint_scn)); SERVER_EVENT_SYNC_ADD("storage_ha", "get_need_copy_ddl_sstable_range", "tablet_id", tablet->get_tablet_meta().tablet_id_, "dest_ddl_checkpoint_pushed", ddl_checkpoint_pushed, - "start_log_ts", need_copy_log_ts_range.start_log_ts_, - "end_log_ts", need_copy_log_ts_range.end_log_ts_); + "start_scn", need_copy_scn_range.start_scn_, + "end_scn", need_copy_scn_range.end_scn_); #endif } else { ret = OB_ERR_UNEXPECTED; @@ -732,11 +732,11 @@ int ObStorageHATabletsBuilder::get_need_copy_ddl_sstable_range_( int ObStorageHATabletsBuilder::get_ddl_sstable_max_start_scn_( const ObSSTableArray &ddl_sstable_array, - palf::SCN &max_start_scn) + SCN &max_start_scn) { int ret = OB_SUCCESS; ObArray sstables; - max_start_scn = palf::SCN::max_scn(); + max_start_scn = SCN::max_scn(); if (!is_inited_) { ret = OB_NOT_INIT; @@ -757,13 +757,13 @@ int ObStorageHATabletsBuilder::get_ddl_sstable_max_start_scn_( ret = OB_ERR_UNEXPECTED; LOG_WARN("sstable type is unexpected", K(ret), KP(table), K(param_)); } else { - palf::SCN start_scn = table->get_key().scn_range_.start_scn_.is_valid() ? (table->get_key().scn_range_.start_scn_) : palf::SCN::max_scn(); + SCN start_scn = table->get_key().scn_range_.start_scn_.is_valid() ? (table->get_key().scn_range_.start_scn_) : SCN::max_scn(); max_start_scn = std::min(max_start_scn, start_scn); } } if (OB_FAIL(ret)) { - } else if (palf::SCN::max_scn() == max_start_scn) { + } else if (SCN::max_scn() == max_start_scn) { ret = OB_ERR_UNEXPECTED; LOG_WARN("max start scn must not be equal to palf::max_scn", K(ret), K(max_start_scn)); } @@ -898,8 +898,8 @@ int ObStorageHATabletsBuilder::create_tablet_remote_logical_sstable_( int ret = OB_SUCCESS; ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; - palf::SCN start_scn; - palf::SCN end_scn; + SCN start_scn; + SCN end_scn; ObArray minor_tables; ObTableHandleV2 table_handle; @@ -940,8 +940,8 @@ int ObStorageHATabletsBuilder::create_tablet_remote_logical_sstable_( int ObStorageHATabletsBuilder::create_remote_logical_sstable_( const common::ObTabletID &tablet_id, - const palf::SCN start_scn, - const palf::SCN end_scn, + const SCN start_scn, + const SCN end_scn, ObTablet *tablet, ObTableHandleV2 &table_handle) { @@ -966,8 +966,8 @@ int ObStorageHATabletsBuilder::create_remote_logical_sstable_( } int ObStorageHATabletsBuilder::build_remote_logical_sstable_param_( - const palf::SCN start_scn, - const palf::SCN end_scn, + const SCN start_scn, + const SCN end_scn, const ObStorageSchema &table_schema, const common::ObTabletID &tablet_id, ObTabletCreateSSTableParam ¶m) diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.h b/src/storage/high_availability/ob_storage_ha_tablet_builder.h index 2e056bccb..f58a03c1b 100644 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.h +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.h @@ -22,7 +22,7 @@ namespace oceanbase { -namespace palf +namespace share { class SCN; } @@ -104,7 +104,7 @@ private: share::ObScnRange &scn_range); int get_ddl_sstable_max_start_scn_( const ObSSTableArray &ddl_sstable_array, - palf::SCN &max_start_scn); + share::SCN &max_start_scn); int hold_local_reuse_sstable_( const common::ObTabletID &tablet_id, ObTablesHandleArray &tables_handle); @@ -118,13 +118,13 @@ private: ObTablesHandleArray &tables_handle); int create_remote_logical_sstable_( const common::ObTabletID &tablet_id, - const palf::SCN start_scn, - const palf::SCN end_scn, + const share::SCN start_scn, + const share::SCN end_scn, ObTablet *tablet, ObTableHandleV2 &table_handle); int build_remote_logical_sstable_param_( - const palf::SCN start_scn, - const palf::SCN end_scn, + const share::SCN start_scn, + const share::SCN end_scn, const ObStorageSchema &table_schema, const common::ObTabletID &tablet_id, ObTabletCreateSSTableParam ¶m); diff --git a/src/storage/high_availability/ob_storage_ha_utils.cpp b/src/storage/high_availability/ob_storage_ha_utils.cpp index 598cf5d39..11a59de47 100644 --- a/src/storage/high_availability/ob_storage_ha_utils.cpp +++ b/src/storage/high_availability/ob_storage_ha_utils.cpp @@ -15,7 +15,7 @@ #include "share/tablet/ob_tablet_table_operator.h" #include "share/ob_global_merge_table_operator.h" #include "share/ob_tablet_replica_checksum_operator.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" using namespace oceanbase::share; @@ -28,7 +28,7 @@ int ObStorageHAUtils::check_tablet_replica_validity(const uint64_t tenant_id, co const common::ObAddr &src_addr, const common::ObTabletID &tablet_id, common::ObISQLClient &sql_client) { int ret = OB_SUCCESS; - palf::SCN compaction_scn; + SCN compaction_scn; if (tablet_id.is_ls_inner_tablet()) { // do nothing } else if (OB_INVALID_ID == tenant_id || !ls_id.is_valid() || !src_addr.is_valid() || !tablet_id.is_valid()) { @@ -63,7 +63,7 @@ int ObStorageHAUtils::check_merge_error_(const uint64_t tenant_id, common::ObISQ } int ObStorageHAUtils::fetch_src_tablet_meta_info_(const uint64_t tenant_id, const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, const common::ObAddr &src_addr, common::ObISQLClient &sql_client, palf::SCN &compaction_scn) + const share::ObLSID &ls_id, const common::ObAddr &src_addr, common::ObISQLClient &sql_client, SCN &compaction_scn) { int ret = OB_SUCCESS; ObTabletTableOperator op; @@ -79,7 +79,7 @@ int ObStorageHAUtils::fetch_src_tablet_meta_info_(const uint64_t tenant_id, cons } int ObStorageHAUtils::check_tablet_replica_checksum_(const uint64_t tenant_id, const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, const palf::SCN &compaction_scn, common::ObISQLClient &sql_client) + const share::ObLSID &ls_id, const SCN &compaction_scn, common::ObISQLClient &sql_client) { int ret = OB_SUCCESS; ObArray items; diff --git a/src/storage/high_availability/ob_storage_ha_utils.h b/src/storage/high_availability/ob_storage_ha_utils.h index bb64695af..125efebd9 100644 --- a/src/storage/high_availability/ob_storage_ha_utils.h +++ b/src/storage/high_availability/ob_storage_ha_utils.h @@ -16,7 +16,7 @@ namespace oceanbase { -namespace palf +namespace share { class SCN; } @@ -33,9 +33,9 @@ private: static int check_merge_error_(const uint64_t tenant_id, common::ObISQLClient &sql_client); static int fetch_src_tablet_meta_info_(const uint64_t tenant_id, const common::ObTabletID &tablet_id, const share::ObLSID &ls_id, const common::ObAddr &src_addr, common::ObISQLClient &sql_client, - palf::SCN &compaction_scn); + share::SCN &compaction_scn); static int check_tablet_replica_checksum_(const uint64_t tenant_id, const common::ObTabletID &tablet_id, - const share::ObLSID &ls_id, const palf::SCN &compaction_scn, common::ObISQLClient &sql_client); + const share::ObLSID &ls_id, const share::SCN &compaction_scn, common::ObISQLClient &sql_client); }; } // end namespace storage diff --git a/src/storage/high_availability/ob_storage_restore_struct.h b/src/storage/high_availability/ob_storage_restore_struct.h index e0d599cff..e971a9b64 100644 --- a/src/storage/high_availability/ob_storage_restore_struct.h +++ b/src/storage/high_availability/ob_storage_restore_struct.h @@ -47,7 +47,7 @@ struct ObRestoreBaseInfo K_(backup_dest), K_(backup_set_list)); - palf::SCN restore_scn_; + share::SCN restore_scn_; int64_t backup_cluster_version_; share::ObBackupDest backup_dest_; common::ObArray backup_set_list_; diff --git a/src/storage/high_availability/ob_tablet_backfill_tx.cpp b/src/storage/high_availability/ob_tablet_backfill_tx.cpp index b1e199215..b9b26f320 100644 --- a/src/storage/high_availability/ob_tablet_backfill_tx.cpp +++ b/src/storage/high_availability/ob_tablet_backfill_tx.cpp @@ -26,7 +26,7 @@ namespace storage ObBackfillTXCtx::ObBackfillTXCtx() : task_id_(), ls_id_(), - log_sync_scn_(palf::SCN::min_scn()), + log_sync_scn_(SCN::min_scn()), lock_(), tablet_id_index_(0), tablet_id_array_() @@ -85,7 +85,7 @@ int ObBackfillTXCtx::get_tablet_id(ObTabletID &tablet_id) int ObBackfillTXCtx::build_backfill_tx_ctx( const share::ObTaskId &task_id, const share::ObLSID &ls_id, - const palf::SCN log_sync_scn, + const SCN log_sync_scn, const common::ObIArray &tablet_id_array) { int ret = OB_SUCCESS; @@ -945,7 +945,7 @@ int64_t ObFinishBackfillTXDag::hash() const int ObFinishBackfillTXDag::init( const share::ObTaskId &task_id, const share::ObLSID &ls_id, - const palf::SCN log_sync_scn, + const SCN log_sync_scn, ObIHADagNetCtx *ha_dag_net_ctx) { int ret = OB_SUCCESS; @@ -968,7 +968,7 @@ int ObFinishBackfillTXDag::init( int ObFinishBackfillTXDag::prepare_backfill_tx_ctx_( const share::ObTaskId &task_id, const share::ObLSID &ls_id, - const palf::SCN log_sync_scn) + const SCN log_sync_scn) { int ret = OB_SUCCESS; ObLS *ls = nullptr; diff --git a/src/storage/high_availability/ob_tablet_backfill_tx.h b/src/storage/high_availability/ob_tablet_backfill_tx.h index d3c79bdd5..0f1589dda 100644 --- a/src/storage/high_availability/ob_tablet_backfill_tx.h +++ b/src/storage/high_availability/ob_tablet_backfill_tx.h @@ -36,7 +36,7 @@ public: int build_backfill_tx_ctx( const share::ObTaskId &task_id, const share::ObLSID &ls_id, - const palf::SCN log_sync_scn, + const share::SCN log_sync_scn, const common::ObIArray &tablet_id_array); bool is_empty() const; int check_is_same( @@ -54,7 +54,7 @@ public: public: share::ObTaskId task_id_; share::ObLSID ls_id_; - palf::SCN log_sync_scn_; + share::SCN log_sync_scn_; private: bool inner_is_valid_() const; private: @@ -189,7 +189,7 @@ public: int init( const share::ObTaskId &task_id, const share::ObLSID &ls_id, - const palf::SCN log_sync_scn, + const share::SCN log_sync_scn, ObIHADagNetCtx *ha_dag_net_ctx); ObBackfillTXCtx *get_backfill_tx_ctx() { return &backfill_tx_ctx_; } INHERIT_TO_STRING_KV("ObStorageHADag", ObStorageHADag, KP(this)); @@ -197,7 +197,7 @@ protected: int prepare_backfill_tx_ctx_( const share::ObTaskId &task_id, const share::ObLSID &ls_id, - const palf::SCN log_sync_scn); + const share::SCN log_sync_scn); protected: bool is_inited_; diff --git a/src/storage/lob/ob_lob_util.h b/src/storage/lob/ob_lob_util.h index fc44aa62f..a37133792 100644 --- a/src/storage/lob/ob_lob_util.h +++ b/src/storage/lob/ob_lob_util.h @@ -58,7 +58,7 @@ struct ObLobAccessParam { int64_t byte_size_; int64_t handle_size_; int64_t timeout_; - palf::SCN fb_snapshot_; + share::SCN fb_snapshot_; bool scan_backward_; bool asscess_ptable_; uint64_t offset_; // is_char为true时 offset代表字符长度 diff --git a/src/storage/ls/ob_freezer.cpp b/src/storage/ls/ob_freezer.cpp index cd8fdb3e6..75fa29cfa 100644 --- a/src/storage/ls/ob_freezer.cpp +++ b/src/storage/ls/ob_freezer.cpp @@ -26,6 +26,7 @@ namespace oceanbase { using namespace logservice; +using namespace share; using namespace compaction; namespace storage { @@ -40,8 +41,8 @@ ObFrozenMemtableInfo::ObFrozenMemtableInfo() {} ObFrozenMemtableInfo::ObFrozenMemtableInfo(const ObTabletID &tablet_id, - const palf::SCN &start_scn, - const palf::SCN &end_scn, + const SCN &start_scn, + const SCN &end_scn, const int64_t write_ref_cnt, const int64_t unsubmitted_cnt, const int64_t unsynced_cnt, @@ -72,8 +73,8 @@ void ObFrozenMemtableInfo::reset() } void ObFrozenMemtableInfo::set(const ObTabletID &tablet_id, - const palf::SCN &start_scn, - const palf::SCN &end_scn, + const SCN &start_scn, + const SCN &end_scn, const int64_t write_ref_cnt, const int64_t unsubmitted_cnt, const int64_t unsynced_cnt, @@ -131,8 +132,8 @@ bool ObFreezerStat::is_valid() } int ObFreezerStat::add_memtable_info(const ObTabletID &tablet_id, - const palf::SCN &start_scn, - const palf::SCN &end_scn, + const SCN &start_scn, + const SCN &end_scn, const int64_t write_ref_cnt, const int64_t unsubmitted_cnt, const int64_t unsynced_cnt, @@ -301,8 +302,8 @@ int ObFreezer::init(ObLSWRSHandler *ls_loop_worker, int ObFreezer::logstream_freeze() { int ret = OB_SUCCESS; - palf::SCN freeze_snapshot_version; - palf::SCN max_decided_scn; + SCN freeze_snapshot_version; + SCN max_decided_scn; FLOG_INFO("[Freezer] logstream_freeze start", K(ret), K_(ls_id)); stat_.reset(); stat_.start_time_ = ObTimeUtility::current_time(); @@ -350,7 +351,7 @@ int ObFreezer::inner_logstream_freeze() uint32_t freeze_clock = get_freeze_clock(); TRANS_LOG(INFO, "[Freezer] freeze_clock", K(ret), K_(ls_id), K(freeze_clock)); - if (OB_FAIL(data_checkpoint_->ls_freeze(palf::SCN::max_scn()))) { + if (OB_FAIL(data_checkpoint_->ls_freeze(SCN::max_scn()))) { // move memtables from active_list to frozen_list TRANS_LOG(WARN, "[Freezer] data_checkpoint freeze failed", K(ret), K_(ls_id)); stat_.add_diagnose_info("data_checkpoint freeze failed"); @@ -382,7 +383,7 @@ int ObFreezer::tablet_freeze(const ObTabletID &tablet_id) ObTablet *tablet = nullptr; ObTabletMemtableMgr *memtable_mgr = nullptr; memtable::ObIMemtable *imemtable = nullptr; - palf::SCN freeze_snapshot_version; + SCN freeze_snapshot_version; FLOG_INFO("[Freezer] tablet_freeze start", K(ret), K_(ls_id), K(tablet_id)); stat_.reset(); stat_.start_time_ = ObTimeUtility::current_time(); @@ -456,7 +457,7 @@ int ObFreezer::force_tablet_freeze(const ObTabletID &tablet_id) ObTablet *tablet = nullptr; ObTabletMemtableMgr *memtable_mgr = nullptr; memtable::ObIMemtable *imemtable = nullptr; - palf::SCN freeze_snapshot_version; + SCN freeze_snapshot_version; FLOG_INFO("[Freezer] force_tablet_freeze start", K(ret), K_(ls_id), K(tablet_id)); stat_.reset(); stat_.start_time_ = ObTimeUtility::current_time(); @@ -774,7 +775,7 @@ bool ObFreezer::is_freeze(uint32_t freeze_flag) const } /* other public functions */ -int ObFreezer::decide_max_decided_scn(palf::SCN &max_decided_scn) +int ObFreezer::decide_max_decided_scn(SCN &max_decided_scn) { int ret = OB_SUCCESS; @@ -797,7 +798,7 @@ int ObFreezer::decide_max_decided_scn(palf::SCN &max_decided_scn) return ret; } -int ObFreezer::get_max_consequent_callbacked_scn(palf::SCN &max_consequent_callbacked_scn) +int ObFreezer::get_max_consequent_callbacked_scn(SCN &max_consequent_callbacked_scn) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -816,7 +817,7 @@ int ObFreezer::get_max_consequent_callbacked_scn(palf::SCN &max_consequent_callb return ret; } -int ObFreezer::get_ls_weak_read_scn(palf::SCN &weak_read_scn) +int ObFreezer::get_ls_weak_read_scn(SCN &weak_read_scn) { int ret = OB_SUCCESS; weak_read_scn.reset(); @@ -833,7 +834,7 @@ int ObFreezer::get_ls_weak_read_scn(palf::SCN &weak_read_scn) } int ObFreezer::get_newest_clog_checkpoint_scn(const ObTabletID &tablet_id, - palf::SCN &clog_checkpoint_scn) + SCN &clog_checkpoint_scn) { int ret = OB_SUCCESS; ObTabletHandle handle; @@ -855,7 +856,7 @@ int ObFreezer::get_newest_clog_checkpoint_scn(const ObTabletID &tablet_id, } int ObFreezer::get_newest_snapshot_version(const ObTabletID &tablet_id, - palf::SCN &snapshot_version) + SCN &snapshot_version) { int ret = OB_SUCCESS; ObTabletHandle handle; diff --git a/src/storage/ls/ob_freezer.h b/src/storage/ls/ob_freezer.h index 6116c94db..f86e44a23 100644 --- a/src/storage/ls/ob_freezer.h +++ b/src/storage/ls/ob_freezer.h @@ -107,8 +107,8 @@ class ObFrozenMemtableInfo public: ObFrozenMemtableInfo(); ObFrozenMemtableInfo(const ObTabletID &tablet_id, - const palf::SCN &start_scn_, - const palf::SCN &end_scn, + const share::SCN &start_scn_, + const share::SCN &end_scn, const int64_t write_ref_cnt, const int64_t unsubmitted_cnt, const int64_t unsynced_cnt, @@ -117,8 +117,8 @@ public: void reset(); void set(const ObTabletID &tablet_id, - const palf::SCN &start_scn, - const palf::SCN &end_scn, + const share::SCN &start_scn, + const share::SCN &end_scn, const int64_t write_ref_cnt, const int64_t unsubmitted_cnt, const int64_t unsynced_cnt, @@ -127,8 +127,8 @@ public: public: ObTabletID tablet_id_; - palf::SCN start_scn_; - palf::SCN end_scn_; + share::SCN start_scn_; + share::SCN end_scn_; int64_t write_ref_cnt_; int64_t unsubmitted_cnt_; int64_t unsynced_cnt_; @@ -151,8 +151,8 @@ public: public: int add_memtable_info(const ObTabletID &tablet_id, - const palf::SCN &start_scn, - const palf::SCN &end_scn, + const share::SCN &start_scn, + const share::SCN &end_scn, const int64_t write_ref_cnt, const int64_t unsubmitted_cnt, const int64_t unsynced_cnt, @@ -224,10 +224,10 @@ public: ObLSTabletService *get_ls_tablet_svr() { return ls_tablet_svr_; } /* freeze_snapshot_version */ - palf::SCN get_freeze_snapshot_version() { return freeze_snapshot_version_; } + share::SCN get_freeze_snapshot_version() { return freeze_snapshot_version_; } /* max_decided_scn */ - palf::SCN get_max_decided_scn() { return max_decided_scn_; } + share::SCN get_max_decided_scn() { return max_decided_scn_; } /* statistics*/ void inc_empty_memtable_cnt(); @@ -237,15 +237,15 @@ public: /* others */ // get consequent callbacked log_ts right boundary - virtual int get_max_consequent_callbacked_scn(palf::SCN &max_consequent_callbacked_scn); + virtual int get_max_consequent_callbacked_scn(share::SCN &max_consequent_callbacked_scn); // to set snapshot version when memtables meet ready_for_flush - int get_ls_weak_read_scn(palf::SCN &weak_read_scn); - int decide_max_decided_scn(palf::SCN &max_decided_scn); + int get_ls_weak_read_scn(share::SCN &weak_read_scn); + int decide_max_decided_scn(share::SCN &max_decided_scn); // to resolve concurrency problems about multi-version tablet int get_newest_clog_checkpoint_scn(const ObTabletID &tablet_id, - palf::SCN &clog_checkpoint_scn); + share::SCN &clog_checkpoint_scn); int get_newest_snapshot_version(const ObTabletID &tablet_id, - palf::SCN &snapshot_version); + share::SCN &snapshot_version); ObFreezerStat& get_stat() { return stat_; } private: @@ -296,10 +296,10 @@ private: // weak read timestamp saved for memtable, which means the version before // which all transaction has been saved into the memtable and the memtables // before it. - palf::SCN freeze_snapshot_version_; + share::SCN freeze_snapshot_version_; // max_decided_scn saved for memtable when freeze happen, which means the // log ts before which will be smaller than the log ts in the latter memtables - palf::SCN max_decided_scn_; + share::SCN max_decided_scn_; ObLSWRSHandler *ls_wrs_handler_; ObLSTxService *ls_tx_svr_; diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index f220637e0..983051817 100644 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -76,7 +76,7 @@ int ObLS::init(const share::ObLSID &ls_id, const ObReplicaType replica_type, const ObMigrationStatus &migration_status, const ObLSRestoreStatus &restore_status, - const palf::SCN &create_scn, + const SCN &create_scn, observer::ObIMetaReport *reporter) { int ret = OB_SUCCESS; @@ -206,7 +206,7 @@ int ObLS::init(const share::ObLSID &ls_id, } int ObLS::create_ls_inner_tablet(const lib::Worker::CompatMode compat_mode, - const palf::SCN &create_scn) + const SCN &create_scn) { int ret = OB_SUCCESS; if (OB_FAIL(tx_table_.create_tablet(compat_mode, create_scn))) { @@ -944,12 +944,12 @@ int ObLS::finish_slog_replay() } int ObLS::replay_get_tablet(const common::ObTabletID &tablet_id, - const palf::SCN &scn, + const SCN &scn, ObTabletHandle &handle) const { int ret = OB_SUCCESS; const ObTabletMapKey key(ls_meta_.ls_id_, tablet_id); - const palf::SCN tablet_change_checkpoint_scn = ls_meta_.get_tablet_change_checkpoint_scn(); + const SCN tablet_change_checkpoint_scn = ls_meta_.get_tablet_change_checkpoint_scn(); ObTabletHandle tablet_handle; if (IS_NOT_INIT) { @@ -1205,8 +1205,8 @@ int ObLS::flush_if_need(const bool need_flush) int ObLS::flush_if_need_(const bool need_flush) { int ret = OB_SUCCESS; - palf::SCN clog_checkpoint_scn = get_clog_checkpoint_scn(); - palf::SCN invalid_scn; + SCN clog_checkpoint_scn = get_clog_checkpoint_scn(); + SCN invalid_scn; if ((!need_flush && !checkpoint_executor_.need_flush()) || checkpoint_executor_.is_wait_advance_checkpoint()) { STORAGE_LOG(INFO, "the ls no need flush to advance_checkpoint", K(get_ls_id())); } else if (OB_FAIL(checkpoint_executor_.advance_checkpoint_by_flush(invalid_scn))) { @@ -1267,7 +1267,7 @@ int ObLS::try_update_uppder_trans_version() return ret; } -int ObLS::set_tablet_change_checkpoint_scn(const palf::SCN &scn) +int ObLS::set_tablet_change_checkpoint_scn(const SCN &scn) { int ret = OB_SUCCESS; int64_t read_lock = LSLOCKLS; @@ -1285,7 +1285,7 @@ int ObLS::set_tablet_change_checkpoint_scn(const palf::SCN &scn) int ObLS::update_id_meta_with_writing_slog(const int64_t service_type, const int64_t limited_id, - const palf::SCN &latest_scn) + const SCN &latest_scn) { int ret = OB_SUCCESS; int64_t read_lock = LSLOCKLS; @@ -1312,7 +1312,7 @@ int ObLS::update_id_meta_with_writing_slog(const int64_t service_type, int ObLS::update_id_meta_without_writing_slog(const int64_t service_type, const int64_t limited_id, - const palf::SCN &latest_scn) + const SCN &latest_scn) { int ret = OB_SUCCESS; diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index e9c15b55b..8996b8f47 100644 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -60,7 +60,7 @@ namespace observer { class ObIMetaReport; } -namespace palf +namespace share { class SCN; } @@ -117,9 +117,9 @@ struct ObLSVTInfo common::ObRole ls_state_; ObMigrationStatus migrate_status_; int64_t tablet_count_; - palf::SCN weak_read_scn_; + share::SCN weak_read_scn_; bool need_rebuild_; - palf::SCN checkpoint_scn_; + share::SCN checkpoint_scn_; //TODO SCN int64_t checkpoint_lsn_; }; @@ -155,7 +155,7 @@ public: const ObReplicaType replica_type, const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, - const palf::SCN &create_scn, + const share::SCN &create_scn, observer::ObIMetaReport *reporter); // I am ready to work now. int start(); @@ -164,6 +164,7 @@ public: void destroy(); int offline(); int online(); + bool is_offline() const { return false; } // mock function, TODO(@yanyuan) ObLSTxService *get_tx_svr() { return &ls_tx_svr_; } ObLockTable *get_lock_table() { return &lock_table_; } @@ -234,7 +235,7 @@ public: int remove_ls(); // create all the inner tablet. int create_ls_inner_tablet(const lib::Worker::CompatMode compat_mode, - const palf::SCN &create_scn); + const share::SCN &create_scn); // load all the inner tablet. int load_ls_inner_tablet(); @@ -271,7 +272,7 @@ public: // get tablet while replaying clog int replay_get_tablet(const common::ObTabletID &tablet_id, - const palf::SCN &scn, + const share::SCN &scn, ObTabletHandle &handle) const; int flush_if_need(const bool need_flush); @@ -289,21 +290,21 @@ public: // ObLSMeta interface: int update_id_meta_with_writing_slog(const int64_t service_type, const int64_t limited_id, - const palf::SCN &latest_scn); + const share::SCN &latest_scn); int update_id_meta_without_writing_slog(const int64_t service_type, const int64_t limited_id, - const palf::SCN &latest_scn); + const share::SCN &latest_scn); // int set_ls_rebuild(); UPDATE_LSMETA_WITH_LOCK(ls_meta_, set_ls_rebuild); // protect in ls lock // int set_gc_state(const logservice::LSGCState &gc_state); UPDATE_LSMETA_WITH_LOCK(ls_meta_, set_gc_state); // int set_clog_checkpoint(const palf::LSN &clog_checkpoint_lsn, - // const palf::SCN &clog_checkpoint_scn, + // const share::SCN &clog_checkpoint_scn, // const bool write_slog = true); UPDATE_LSMETA_WITH_LOCK(ls_meta_, set_clog_checkpoint); UPDATE_LSMETA_WITHOUT_LOCK(ls_meta_, set_clog_checkpoint); - CONST_DELEGATE_WITH_RET(ls_meta_, get_clog_checkpoint_scn, palf::SCN); + CONST_DELEGATE_WITH_RET(ls_meta_, get_clog_checkpoint_scn, share::SCN); DELEGATE_WITH_RET(ls_meta_, get_clog_base_lsn, palf::LSN &); DELEGATE_WITH_RET(ls_meta_, get_saved_info, int); // int build_saved_info(); @@ -317,7 +318,7 @@ public: // const ObLSMeta &src_ls_meta); UPDATE_LSMETA_WITHOUT_LOCK(ls_meta_, update_ls_meta); CONST_DELEGATE_WITH_RET(ls_meta_, get_rebuild_seq, int64_t); - CONST_DELEGATE_WITH_RET(ls_meta_, get_tablet_change_checkpoint_scn, palf::SCN); + CONST_DELEGATE_WITH_RET(ls_meta_, get_tablet_change_checkpoint_scn, share::SCN); // set restore status // @param [in] restore status. // int set_restore_status(const share::ObLSRestoreStatus &status); @@ -350,7 +351,7 @@ public: UPDATE_LSMETA_WITH_LOCK(ls_meta_, set_offline_scn); // get offline ts // @param [in] offline ts. - // int get_offline_scn(const SCN &offline_scn); + // int get_offline_scn(const share::SCN &offline_scn); DELEGATE_WITH_RET(ls_meta_, get_offline_scn, int); // update replayable point // @param [in] replayable point. @@ -362,7 +363,7 @@ public: DELEGATE_WITH_RET(ls_meta_, get_ls_replayable_point, int); // set tablet_change_checkpoint_scn, add write lock of LSLOCKLOGMETA. // @param [in] scn - int set_tablet_change_checkpoint_scn(const palf::SCN &scn); + int set_tablet_change_checkpoint_scn(const share::SCN &scn); // get ls_meta_package and unsorted tablet_ids, add read lock of LSLOCKLOGMETA. // @param [out] meta_package // @param [out] tablet_ids @@ -479,7 +480,7 @@ public: // WARNING: must has ls read lock and log write lock. int disable_replay_without_lock(); // @brief, get max decided log scn considering both apply and replay. - // @param[out] palf::SCN&, max decided log scn. + // @param[out] share::SCN&, max decided log scn. DELEGATE_WITH_RET(log_handler_, get_max_decided_scn, int); // @breif, check request server is in self member list // @param[in] const common::ObAddr, request server. @@ -599,8 +600,6 @@ public: // ObFreezer interface: // logstream freeze // @param [in] null - // int logstream_freeze(); - // DELEGATE_WITH_RET(ls_freezer_, logstream_freeze, int); int logstream_freeze(); // tablet freeze // @param [in] tablet_id diff --git a/src/storage/ls/ob_ls_ddl_log_handler.cpp b/src/storage/ls/ob_ls_ddl_log_handler.cpp index 87ee3527a..3b27c7471 100644 --- a/src/storage/ls/ob_ls_ddl_log_handler.cpp +++ b/src/storage/ls/ob_ls_ddl_log_handler.cpp @@ -20,7 +20,7 @@ #include "storage/tablet/ob_tablet_iterator.h" #include "storage/ddl/ob_tablet_ddl_kv_mgr.h" #include "logservice/ob_log_base_header.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -112,7 +112,7 @@ int ObLSDDLLogHandler::online() int ObLSDDLLogHandler::replay(const void *buffer, const int64_t buf_size, const palf::LSN &lsn, - const palf::SCN &log_scn) + const SCN &log_scn) { int ret = OB_SUCCESS; logservice::ObLogBaseHeader base_header; @@ -205,7 +205,7 @@ int ObLSDDLLogHandler::resume_leader() return ret; } -int ObLSDDLLogHandler::flush(palf::SCN &rec_scn) +int ObLSDDLLogHandler::flush(SCN &rec_scn) { int ret = OB_SUCCESS; ObLSTabletIterator tablet_iter(ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US); @@ -236,10 +236,13 @@ int ObLSDDLLogHandler::flush(palf::SCN &rec_scn) ObDDLTableMergeDagParam param; param.ls_id_ = ls_->get_ls_id(); param.tablet_id_ = ddl_kv_mgr_handle.get_obj()->get_tablet_id(); + param.start_scn_ = ddl_kv_mgr_handle.get_obj()->get_start_scn(); param.rec_scn_ = rec_scn; if (OB_FAIL(compaction::ObScheduleDagFunc::schedule_ddl_table_merge_dag(param))) { if (OB_EAGAIN != ret && OB_SIZE_OVERFLOW != ret) { LOG_WARN("failed to schedule ddl kv merge dag", K(ret)); + } else { + ret = OB_SUCCESS; } } } @@ -249,18 +252,18 @@ int ObLSDDLLogHandler::flush(palf::SCN &rec_scn) return OB_SUCCESS; } -palf::SCN ObLSDDLLogHandler::get_rec_scn() +SCN ObLSDDLLogHandler::get_rec_scn() { int ret = OB_SUCCESS; ObLSTabletIterator tablet_iter(ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US); - palf::SCN rec_scn = palf::SCN::max_scn(); + SCN rec_scn = SCN::max_scn(); bool has_ddl_kv = false; if (OB_FAIL(ls_->get_tablet_svr()->build_tablet_iter(tablet_iter))) { LOG_WARN("failed to build ls tablet iter", K(ret), K(ls_)); } else { while (OB_SUCC(ret)) { ObDDLKvMgrHandle ddl_kv_mgr_handle; - palf::SCN min_scn = palf::SCN::max_scn(); + SCN min_scn = SCN::max_scn(); if (OB_FAIL(tablet_iter.get_next_ddl_kv_mgr(ddl_kv_mgr_handle))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; @@ -277,18 +280,18 @@ palf::SCN ObLSDDLLogHandler::get_rec_scn() if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_ddl_kv_min_scn(min_scn))) { LOG_WARN("fail to get ddl kv min log ts", K(ret)); } else { - rec_scn = palf::SCN::min(rec_scn, min_scn); + rec_scn = SCN::min(rec_scn, min_scn); } } } } - return OB_SUCC(ret) ? rec_scn : palf::SCN::max_scn(); + return OB_SUCC(ret) ? rec_scn : SCN::max_scn(); } int ObLSDDLLogHandler::replay_ddl_redo_log_(const char *log_buf, const int64_t buf_size, int64_t pos, - const palf::SCN &log_scn) + const SCN &log_scn) { int ret = OB_SUCCESS; ObDDLRedoLog log; @@ -306,7 +309,7 @@ int ObLSDDLLogHandler::replay_ddl_redo_log_(const char *log_buf, int ObLSDDLLogHandler::replay_ddl_prepare_log_(const char *log_buf, const int64_t buf_size, int64_t pos, - const palf::SCN &log_scn) + const SCN &log_scn) { int ret = OB_SUCCESS; ObDDLPrepareLog log; @@ -324,7 +327,7 @@ int ObLSDDLLogHandler::replay_ddl_prepare_log_(const char *log_buf, int ObLSDDLLogHandler::replay_ddl_commit_log_(const char *log_buf, const int64_t buf_size, int64_t pos, - const palf::SCN &log_scn) + const SCN &log_scn) { int ret = OB_SUCCESS; ObDDLCommitLog log; @@ -342,7 +345,7 @@ int ObLSDDLLogHandler::replay_ddl_commit_log_(const char *log_buf, int ObLSDDLLogHandler::replay_ddl_tablet_schema_version_change_log_(const char *log_buf, const int64_t buf_size, int64_t pos, - const palf::SCN &log_scn) + const SCN &log_scn) { int ret = OB_SUCCESS; ObTabletSchemaVersionChangeLog log; @@ -362,7 +365,7 @@ int ObLSDDLLogHandler::replay_ddl_tablet_schema_version_change_log_(const char * int ObLSDDLLogHandler::replay_ddl_start_log_(const char *log_buf, const int64_t buf_size, int64_t pos, - const palf::SCN &log_scn) + const SCN &log_scn) { int ret = OB_SUCCESS; ObDDLStartLog log; diff --git a/src/storage/ls/ob_ls_ddl_log_handler.h b/src/storage/ls/ob_ls_ddl_log_handler.h index 1495d2eda..564854e0a 100644 --- a/src/storage/ls/ob_ls_ddl_log_handler.h +++ b/src/storage/ls/ob_ls_ddl_log_handler.h @@ -44,7 +44,7 @@ public: int replay(const void *buffer, const int64_t buf_size, const palf::LSN &lsn, - const palf::SCN &log_ts) override final; + const share::SCN &log_ts) override final; // for role change void switch_to_follower_forcedly() override final; @@ -53,14 +53,14 @@ public: int resume_leader() override final; // for checkpoint - int flush(palf::SCN &rec_scn) override final; - palf::SCN get_rec_scn() override final; + int flush(share::SCN &rec_scn) override final; + share::SCN get_rec_scn() override final; private: - int replay_ddl_redo_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const palf::SCN &scn); - int replay_ddl_prepare_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const palf::SCN &scn); - int replay_ddl_commit_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const palf::SCN &scn); - int replay_ddl_tablet_schema_version_change_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const palf::SCN &scn); - int replay_ddl_start_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const palf::SCN &scn); + int replay_ddl_redo_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn); + int replay_ddl_prepare_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn); + int replay_ddl_commit_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn); + int replay_ddl_tablet_schema_version_change_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn); + int replay_ddl_start_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn); private: bool is_inited_; bool is_online_; diff --git a/src/storage/ls/ob_ls_meta.cpp b/src/storage/ls/ob_ls_meta.cpp index 1533c8650..a42413915 100644 --- a/src/storage/ls/ob_ls_meta.cpp +++ b/src/storage/ls/ob_ls_meta.cpp @@ -57,7 +57,7 @@ ObLSMeta::ObLSMeta() offline_scn_(), restore_status_(ObLSRestoreStatus::LS_RESTORE_STATUS_MAX), replayable_point_(), - tablet_change_checkpoint_scn_(palf::SCN::min_scn()), + tablet_change_checkpoint_scn_(SCN::min_scn()), all_id_meta_(), saved_info_() { @@ -120,7 +120,7 @@ void ObLSMeta::reset() offline_scn_.reset(); restore_status_ = ObLSRestoreStatus::LS_RESTORE_STATUS_MAX; replayable_point_.reset(); - tablet_change_checkpoint_scn_ = palf::SCN::min_scn(); + tablet_change_checkpoint_scn_ = SCN::min_scn(); saved_info_.reset(); } @@ -158,13 +158,13 @@ int ObLSMeta::set_clog_checkpoint(const LSN &clog_checkpoint_lsn, return ret; } -palf::SCN ObLSMeta::get_tablet_change_checkpoint_scn() const +SCN ObLSMeta::get_tablet_change_checkpoint_scn() const { ObSpinLockTimeGuard guard(lock_); return tablet_change_checkpoint_scn_; } -int ObLSMeta::set_tablet_change_checkpoint_scn(const palf::SCN &tablet_change_checkpoint_scn) +int ObLSMeta::set_tablet_change_checkpoint_scn(const SCN &tablet_change_checkpoint_scn) { ObSpinLockTimeGuard guard(lock_); int ret = OB_SUCCESS; @@ -286,7 +286,7 @@ int ObLSMeta::get_gc_state(logservice::LSGCState &gc_state) return ret; } -int ObLSMeta::set_offline_scn(const palf::SCN &offline_scn) +int ObLSMeta::set_offline_scn(const SCN &offline_scn) { // 不主动写slog int ret = OB_SUCCESS; @@ -295,7 +295,7 @@ int ObLSMeta::set_offline_scn(const palf::SCN &offline_scn) return ret; } -int ObLSMeta::get_offline_scn(palf::SCN &offline_scn) +int ObLSMeta::get_offline_scn(SCN &offline_scn) { int ret = OB_SUCCESS; ObSpinLockTimeGuard guard(lock_); @@ -344,7 +344,7 @@ int ObLSMeta::get_restore_status(ObLSRestoreStatus &restore_status) const return ret; } -int ObLSMeta::update_ls_replayable_point(const palf::SCN &replayable_point) +int ObLSMeta::update_ls_replayable_point(const SCN &replayable_point) { int ret = OB_SUCCESS; ObSpinLockTimeGuard guard(lock_); @@ -578,7 +578,7 @@ int ObLSMeta::init( const ObReplicaType &replica_type, const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, - const palf::SCN &create_scn) + const SCN &create_scn) { int ret = OB_SUCCESS; if (OB_INVALID_ID == tenant_id || !ls_id.is_valid() @@ -610,7 +610,7 @@ void ObLSMeta::set_write_slog_func_(WriteSlog write_slog) int ObLSMeta::update_id_meta(const int64_t service_type, const int64_t limited_id, - const palf::SCN &latest_scn, + const SCN &latest_scn, const bool write_slog) { int ret = OB_SUCCESS; diff --git a/src/storage/ls/ob_ls_meta.h b/src/storage/ls/ob_ls_meta.h index 47ff55237..4c1ca97b5 100644 --- a/src/storage/ls/ob_ls_meta.h +++ b/src/storage/ls/ob_ls_meta.h @@ -27,14 +27,10 @@ #include "share/restore/ob_ls_restore_status.h" #include "storage/tx/ob_id_service.h" #include "storage/ls/ob_ls_saved_info.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { -namespace palf -{ -class SCN; -} namespace storage { @@ -46,10 +42,10 @@ public: ObLSMeta(const ObLSMeta &ls_meta); ~ObLSMeta() {} ObLSMeta &operator=(const ObLSMeta &other); - palf::SCN get_clog_checkpoint_scn() const; + share::SCN get_clog_checkpoint_scn() const; palf::LSN &get_clog_base_lsn(); int set_clog_checkpoint(const palf::LSN &clog_checkpoint_lsn, - const palf::SCN &clog_checkpoint_scn, + const share::SCN &clog_checkpoint_scn, const bool write_slog = true); void reset(); bool is_valid() const; @@ -60,13 +56,13 @@ public: int get_migration_status (ObMigrationStatus &migration_status) const; int set_gc_state(const logservice::LSGCState &gc_state); int get_gc_state(logservice::LSGCState &gc_state); - int set_offline_scn(const palf::SCN &offline_scn); - int get_offline_scn(palf::SCN &offline_scn); + int set_offline_scn(const share::SCN &offline_scn); + int get_offline_scn(share::SCN &offline_scn); int set_restore_status(const share::ObLSRestoreStatus &restore_status); int get_restore_status(share::ObLSRestoreStatus &restore_status) const; - int update_ls_replayable_point(const palf::SCN &replayable_point); - int get_ls_replayable_point(palf::SCN &replayable_point); + int update_ls_replayable_point(const share::SCN &replayable_point); + int get_ls_replayable_point(share::SCN &replayable_point); //for ha batch update ls meta element int update_ls_meta( @@ -75,11 +71,11 @@ public: //for ha rebuild update ls meta int set_ls_rebuild(); int check_valid_for_backup(); - palf::SCN get_tablet_change_checkpoint_scn() const; - int set_tablet_change_checkpoint_scn(const palf::SCN &tablet_change_checkpoint_scn); + share::SCN get_tablet_change_checkpoint_scn() const; + int set_tablet_change_checkpoint_scn(const share::SCN &tablet_change_checkpoint_scn); int update_id_meta(const int64_t service_type, const int64_t limited_id, - const palf::SCN &latest_scn, + const share::SCN &latest_scn, const bool write_slog); int update_id_service() { @@ -95,7 +91,7 @@ public: const ObReplicaType &replica_type, const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, - const palf::SCN &create_scn); + const share::SCN &create_scn); class ObSpinLockTimeGuard { public: @@ -128,7 +124,7 @@ private: // clog_checkpoint_scn_, meaning: // 1. dump points of all modules have exceeded clog_checkpoint_scn_ // 2. all clog entries which log_scn are smaller than clog_checkpoint_scn_ can be recycled - palf::SCN clog_checkpoint_scn_; + share::SCN clog_checkpoint_scn_; // clog_base_lsn_, meaning: // 1. all clog entries which lsn are smaller than clog_base_lsn_ have been recycled // 2. log_scn of log entry that clog_base_lsn_ points to is smaller than/equal to clog_checkpoint_scn_ @@ -137,11 +133,11 @@ private: int64_t rebuild_seq_; ObMigrationStatus migration_status_; logservice::LSGCState gc_state_; - palf::SCN offline_scn_; + share::SCN offline_scn_; share::ObLSRestoreStatus restore_status_; - palf::SCN replayable_point_; + share::SCN replayable_point_; //TODO(yaoying.yyy):modify this - palf::SCN tablet_change_checkpoint_scn_; + share::SCN tablet_change_checkpoint_scn_; transaction::ObAllIDMeta all_id_meta_; ObLSSavedInfo saved_info_; }; diff --git a/src/storage/ls/ob_ls_saved_info.cpp b/src/storage/ls/ob_ls_saved_info.cpp index 4053254bc..fae1abbc7 100644 --- a/src/storage/ls/ob_ls_saved_info.cpp +++ b/src/storage/ls/ob_ls_saved_info.cpp @@ -17,6 +17,7 @@ namespace oceanbase { +using namespace share; namespace storage { @@ -24,7 +25,7 @@ ObLSSavedInfo::ObLSSavedInfo() : clog_checkpoint_scn_(share::ObScnRange::MIN_SCN), clog_base_lsn_(palf::PALF_INITIAL_LSN_VAL), replayable_point_(0), - tablet_change_checkpoint_scn_(palf::SCN::min_scn()) + tablet_change_checkpoint_scn_(SCN::min_scn()) { } @@ -33,7 +34,7 @@ void ObLSSavedInfo::reset() clog_checkpoint_scn_ = share::ObScnRange::MIN_SCN; clog_base_lsn_ = palf::PALF_INITIAL_LSN_VAL; replayable_point_ = 0; - tablet_change_checkpoint_scn_ = palf::SCN::min_scn(); + tablet_change_checkpoint_scn_ = SCN::min_scn(); } bool ObLSSavedInfo::is_valid() const diff --git a/src/storage/ls/ob_ls_saved_info.h b/src/storage/ls/ob_ls_saved_info.h index 2612ca3a5..fdad8746c 100644 --- a/src/storage/ls/ob_ls_saved_info.h +++ b/src/storage/ls/ob_ls_saved_info.h @@ -15,7 +15,7 @@ #include "lib/utility/ob_print_utils.h" #include "logservice/palf/lsn.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -34,10 +34,10 @@ public: TO_STRING_KV(K_(clog_checkpoint_scn), K_(clog_base_lsn), K_(replayable_point), K_(tablet_change_checkpoint_scn)); - palf::SCN clog_checkpoint_scn_; + share::SCN clog_checkpoint_scn_; palf::LSN clog_base_lsn_; int64_t replayable_point_; - palf::SCN tablet_change_checkpoint_scn_; + share::SCN tablet_change_checkpoint_scn_; }; diff --git a/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp b/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp index e1885e206..4349e5fc9 100644 --- a/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp +++ b/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp @@ -15,7 +15,7 @@ #include "storage/ls/ob_ls.h" #include "storage/ob_sync_tablet_seq_clog.h" #include "logservice/ob_log_base_header.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "lib/oblog/ob_log_module.h" #include "share/ob_tablet_autoincrement_service.h" @@ -107,16 +107,16 @@ int ObLSSyncTabletSeqHandler::resume_leader() return ret; } -int ObLSSyncTabletSeqHandler::flush(palf::SCN &scn) +int ObLSSyncTabletSeqHandler::flush(SCN &scn) { // TODO UNUSED(scn); return OB_SUCCESS; } -palf::SCN ObLSSyncTabletSeqHandler::get_rec_scn() +SCN ObLSSyncTabletSeqHandler::get_rec_scn() { - return palf::SCN::max_scn(); + return SCN::max_scn(); } } diff --git a/src/storage/ls/ob_ls_sync_tablet_seq_handler.h b/src/storage/ls/ob_ls_sync_tablet_seq_handler.h index b6d693b72..1f72160d5 100644 --- a/src/storage/ls/ob_ls_sync_tablet_seq_handler.h +++ b/src/storage/ls/ob_ls_sync_tablet_seq_handler.h @@ -17,7 +17,7 @@ namespace oceanbase { -namespace palf +namespace share { class SCN; } @@ -40,7 +40,7 @@ public: int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, - const palf::SCN &scn) override final; + const share::SCN &scn) override final; // for role change void switch_to_follower_forcedly() override final; @@ -49,8 +49,8 @@ public: int resume_leader() override final; // for checkpoint - int flush(palf::SCN &scn) override final; - palf::SCN get_rec_scn() override final; + int flush(share::SCN &scn) override final; + share::SCN get_rec_scn() override final; private: bool is_inited_; diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 828b6e63a..bada697d9 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -21,7 +21,7 @@ #include "logservice/ob_log_base_header.h" #include "logservice/ob_log_base_type.h" #include "logservice/ob_log_service.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "observer/report/ob_i_meta_report.h" #include "share/ob_disk_usage_table_operator.h" #include "share/ob_rpc_struct.h" @@ -155,7 +155,7 @@ int ObLSTabletService::replay( const void *buffer, const int64_t nbytes, const palf::LSN &lsn, - const palf::SCN &scn) + const SCN &scn) { int ret = OB_SUCCESS; int64_t pos = 0; @@ -198,15 +198,15 @@ int ObLSTabletService::resume_leader() return ret; } -int ObLSTabletService::flush(palf::SCN &recycle_scn) +int ObLSTabletService::flush(SCN &recycle_scn) { UNUSED(recycle_scn); return OB_SUCCESS; } -palf::SCN ObLSTabletService::get_rec_scn() +SCN ObLSTabletService::get_rec_scn() { - return palf::SCN::max_scn(); + return SCN::max_scn(); } int ObLSTabletService::prepare_for_safe_destroy() @@ -259,7 +259,7 @@ int ObLSTabletService::safe_to_destroy(bool &is_safe) int ObLSTabletService::batch_create_tablets( const obrpc::ObBatchCreateTabletArg &arg, - const palf::SCN &create_scn, + const SCN &create_scn, const bool is_replay_clog) { ALLOW_NEXT_LOG(); @@ -322,7 +322,7 @@ int ObLSTabletService::batch_create_tablets( int ObLSTabletService::do_batch_create_tablets( const obrpc::ObBatchCreateTabletArg &arg, - const palf::SCN &create_scn, + const SCN &create_scn, const bool is_replay_clog, common::ObTimeGuard &time_guard, NonLockedHashSet &data_tablet_id_set) @@ -1023,19 +1023,15 @@ int ObLSTabletService::update_tablet_table_store( time_guard.click("GetOld"); ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*); ObTablet *old_tablet = old_tablet_handle.get_obj(); - ObTabletTxMultiSourceDataUnit tx_data; - ObTabletBindingInfo ddl_data; + const ObTabletTxMultiSourceDataUnit *tx_data = nullptr; + const ObTabletBindingInfo *binding_info = nullptr; + const ObTabletAutoincSeq *auto_inc_seq = nullptr; ObMetaDiskAddr disk_addr; - ObTabletAutoincSeq autoinc_seq; - if (OB_FAIL(old_tablet->get_tx_data(tx_data))) { - LOG_WARN("failed to get tx data from old tablet", K(ret), KPC(old_tablet)); - } else if (OB_FAIL(old_tablet->get_ddl_data(ddl_data))) { - LOG_WARN("failed to get ddl data from old tablet", K(ret), KPC(old_tablet)); - } else if (OB_FAIL(old_tablet->get_latest_autoinc_seq(autoinc_seq))) { - LOG_WARN("failed to get autoinc seq from old tablet", K(ret)); - } else if (OB_FAIL(new_tablet->init(param, *old_tablet, tx_data, ddl_data, autoinc_seq))) { - LOG_WARN("failed to init tablet", K(ret), K(param), KPC(old_tablet), K(tx_data), K(ddl_data), K(autoinc_seq)); + if (OB_FAIL(choose_msd(param, *old_tablet, tx_data, binding_info, auto_inc_seq))) { + LOG_WARN("failed to choose msd", K(ret), K(param), KPC(old_tablet)); + } else if (OB_FAIL(new_tablet->init(param, *old_tablet, *tx_data, *binding_info, *auto_inc_seq))) { + LOG_WARN("failed to init tablet", K(ret), K(param), KPC(old_tablet), KPC(tx_data), KPC(binding_info), KPC(auto_inc_seq)); } else if (FALSE_IT(time_guard.click("InitNew"))) { } else if (OB_FAIL(ObTabletSlogHelper::write_create_tablet_slog(new_tablet_handle, disk_addr))) { LOG_WARN("fail to write update tablet slog", K(ret), K(new_tablet_handle), K(disk_addr)); @@ -1054,6 +1050,32 @@ int ObLSTabletService::update_tablet_table_store( return ret; } +int ObLSTabletService::choose_msd( + const ObUpdateTableStoreParam ¶m, + const ObTablet &old_tablet, + const ObTabletTxMultiSourceDataUnit *&tx_data, + const ObTabletBindingInfo *&binding_info, + const share::ObTabletAutoincSeq *&auto_inc_seq) +{ + int ret = OB_SUCCESS; + const ObTabletMeta &old_tablet_meta = old_tablet.get_tablet_meta(); + tx_data = ¶m.tx_data_; + binding_info = ¶m.binding_info_; + auto_inc_seq = ¶m.auto_inc_seq_; + + if (!tx_data->is_valid()) { + tx_data = &old_tablet_meta.tx_data_; + } + if (!binding_info->is_valid()) { + binding_info = &old_tablet_meta.ddl_data_; + } + if (!auto_inc_seq->is_valid()) { + auto_inc_seq = &old_tablet_meta.autoinc_seq_; + } + + return ret; +} + int ObLSTabletService::update_tablet_report_status(const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -1357,12 +1379,15 @@ int ObLSTabletService::try_pin_tablet_if_needed(const ObTabletHandle &tablet_han ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*); ObTablet *tablet = tablet_handle.get_obj(); ObTabletTxMultiSourceDataUnit tx_data; + bool exist_on_memtable = false; if (OB_ISNULL(tablet)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error, tablet is null", K(ret), K(tablet_handle)); - } else if (OB_FAIL(tablet->get_tx_data(tx_data))) { + } else if (OB_FAIL(tablet->inner_get_tx_data(tx_data, exist_on_memtable))) { LOG_WARN("failed to get tx data", K(ret), KPC(tablet)); + } else if (!tx_data.is_valid()) { + // tablet is not valid, do nothing } else if (!tx_data.is_in_tx()) { // tablet not in tx, do nothing } else { @@ -1500,7 +1525,7 @@ int ObLSTabletService::has_tablet( int ObLSTabletService::create_tablet( const share::ObLSID &ls_id, const obrpc::ObBatchCreateTabletArg &arg, - const palf::SCN &create_scn, + const SCN &create_scn, const obrpc::ObCreateTabletInfo &info, common::ObIArray &tablet_handle_array, NonLockedHashSet &data_tablet_id_set) @@ -1530,8 +1555,8 @@ int ObLSTabletService::do_create_tablet( const common::ObTabletID &tablet_id, const common::ObTabletID &data_tablet_id, const common::ObIArray &index_tablet_array, - const palf::SCN &create_scn, - const palf::SCN &snapshot_version, + const SCN &create_scn, + const SCN &snapshot_version, const share::schema::ObTableSchema &table_schema, const lib::Worker::CompatMode &compat_mode, const common::ObTabletID &lob_meta_tablet_id, @@ -1587,7 +1612,7 @@ int ObLSTabletService::do_create_tablet( int ObLSTabletService::build_single_data_tablet( const share::ObLSID &ls_id, const obrpc::ObBatchCreateTabletArg &arg, - const palf::SCN &create_scn, + const SCN &create_scn, const obrpc::ObCreateTabletInfo &info, common::ObIArray &tablet_handle_array) { @@ -4755,7 +4780,7 @@ int ObLSTabletService::prepare_dml_running_ctx( return ret; } -int ObLSTabletService::get_ls_min_end_scn_in_old_tablets(palf::SCN &end_scn) +int ObLSTabletService::get_ls_min_end_scn_in_old_tablets(SCN &end_scn) { int ret = OB_SUCCESS; const ObLSID &ls_id = ls_->get_ls_id(); @@ -5140,10 +5165,10 @@ int ObLSTabletService::create_ls_inner_tablet( const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t memstore_version, - const palf::SCN &major_frozen_scn, + const SCN &major_frozen_scn, const share::schema::ObTableSchema &table_schema, const lib::Worker::CompatMode &compat_mode, - const palf::SCN &create_scn) + const SCN &create_scn) { int ret = OB_SUCCESS; const int64_t data_version = ObVersion(memstore_version - 1, 0); diff --git a/src/storage/ls/ob_ls_tablet_service.h b/src/storage/ls/ob_ls_tablet_service.h index d747130ac..e1401f9d1 100644 --- a/src/storage/ls/ob_ls_tablet_service.h +++ b/src/storage/ls/ob_ls_tablet_service.h @@ -113,7 +113,7 @@ private: const void *buffer, const int64_t nbytes, const palf::LSN &lsn, - const palf::SCN &scn) override; + const share::SCN &scn) override; // for role change virtual void switch_to_follower_forcedly() override; @@ -122,8 +122,8 @@ private: virtual int resume_leader() override; // for checkpoint - virtual int flush(palf::SCN &recycle_scn) override; - virtual palf::SCN get_rec_scn() override; + virtual int flush(share::SCN &recycle_scn) override; + virtual share::SCN get_rec_scn() override; public: int prepare_for_safe_destroy(); @@ -131,7 +131,7 @@ public: // tablet operation int batch_create_tablets( const obrpc::ObBatchCreateTabletArg &arg, - const palf::SCN &create_scn, + const share::SCN &create_scn, const bool is_replay_clog = false); int batch_remove_tablets( const obrpc::ObBatchRemoveTabletArg &arg, @@ -141,10 +141,10 @@ public: const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, const int64_t memstore_version, - const palf::SCN &frozen_timestamp, + const share::SCN &frozen_timestamp, const share::schema::ObTableSchema &table_schema, const lib::Worker::CompatMode &compat_mode, - const palf::SCN &create_scn); + const share::SCN &create_scn); int remove_ls_inner_tablet( const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); @@ -187,7 +187,7 @@ public: ObTabletHandle &handle, const int64_t timeout_us = ObTabletCommon::DEFAULT_GET_TABLET_TIMEOUT_US); int remove_tablets(const common::ObIArray &tablet_id_array); - int get_ls_min_end_scn_in_old_tablets(palf::SCN &end_scn); + int get_ls_min_end_scn_in_old_tablets(share::SCN &end_scn); int get_tx_data_memtable_mgr(ObMemtableMgrHandle &mgr_handle); int get_tx_ctx_memtable_mgr(ObMemtableMgrHandle &mgr_handle); int get_lock_memtable_mgr(ObMemtableMgrHandle &mgr_handle); @@ -436,7 +436,7 @@ private: int create_tablet( const share::ObLSID &ls_id, const obrpc::ObBatchCreateTabletArg &arg, - const palf::SCN &create_scn, + const share::SCN &create_scn, const obrpc::ObCreateTabletInfo &info, common::ObIArray &tablet_handle_array, NonLockedHashSet &data_tablet_id_set); @@ -445,8 +445,8 @@ private: const common::ObTabletID &tablet_id, const common::ObTabletID &data_tablet_id, const common::ObIArray &index_tablet_array, - const palf::SCN &create_scn, - const palf::SCN &snapshot_version, + const share::SCN &create_scn, + const share::SCN &snapshot_version, const share::schema::ObTableSchema &table_schema, const lib::Worker::CompatMode &compat_mode, const common::ObTabletID &lob_meta_tablet_id, @@ -455,7 +455,7 @@ private: int build_single_data_tablet( const share::ObLSID &ls_id, const obrpc::ObBatchCreateTabletArg &arg, - const palf::SCN &create_scn, + const share::SCN &create_scn, const obrpc::ObCreateTabletInfo &info, common::ObIArray &tablet_handle_array); static int build_batch_create_tablet_arg( @@ -468,7 +468,7 @@ private: common::ObIArray &tablet_handle_array); int do_batch_create_tablets( const obrpc::ObBatchCreateTabletArg &arg, - const palf::SCN &create_scn, + const share::SCN &create_scn, const bool is_replay_clog, common::ObTimeGuard &time_guard, NonLockedHashSet &data_tablet_id_set); @@ -498,6 +498,12 @@ private: const ObMigrationTabletParam &mig_tablet_param, ObTabletHandle &handle); int delete_all_tablets(); + int choose_msd( + const ObUpdateTableStoreParam ¶m, + const ObTablet &old_tablet, + const ObTabletTxMultiSourceDataUnit *&tx_data, + const ObTabletBindingInfo *&binding_info, + const share::ObTabletAutoincSeq *&auto_inc_seq); static int build_create_sstable_param_for_migration( const blocksstable::ObMigrationSSTableParam &migrate_sstable_param, ObTabletCreateSSTableParam &create_sstable_param); diff --git a/src/storage/ls/ob_ls_tx_service.cpp b/src/storage/ls/ob_ls_tx_service.cpp index aaa9b1f59..af6f1574f 100644 --- a/src/storage/ls/ob_ls_tx_service.cpp +++ b/src/storage/ls/ob_ls_tx_service.cpp @@ -23,7 +23,7 @@ #include "storage/tx/ob_trans_part_ctx.h" #include "storage/tx/ob_tx_retain_ctx_mgr.h" #include "logservice/ob_log_base_header.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -137,7 +137,7 @@ int ObLSTxService::get_read_store_ctx(const ObTxReadSnapshot &snapshot, return ret; } -int ObLSTxService::get_read_store_ctx(const palf::SCN &snapshot, +int ObLSTxService::get_read_store_ctx(const SCN &snapshot, const int64_t lock_timeout, ObStoreCtx &store_ctx) const { @@ -304,7 +304,7 @@ int ObLSTxService::iterate_tx_obj_lock_op(ObLockOpIterator &iter) const int ObLSTxService::replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, - const palf::SCN &scn) + const SCN &scn) { int ret = OB_SUCCESS; logservice::ObLogBaseHeader base_header; @@ -334,7 +334,7 @@ ObTxLSLogWriter *ObLSTxService::get_tx_ls_log_writer() { return mgr_->get_ls_log ObITxLogAdapter *ObLSTxService::get_tx_ls_log_adapter() { return mgr_->get_ls_log_adapter(); } -int ObLSTxService::replay_start_working_log(const ObTxStartWorkingLog &log, palf::SCN &log_ts_ns) +int ObLSTxService::replay_start_working_log(const ObTxStartWorkingLog &log, SCN &log_ts_ns) { int ret = OB_SUCCESS; if (OB_ISNULL(mgr_)) { @@ -415,14 +415,14 @@ void get_min_rec_scn_common_checkpoint_type_by_index_(int index, } } -palf::SCN ObLSTxService::get_rec_scn() +SCN ObLSTxService::get_rec_scn() { - palf::SCN min_rec_scn = palf::SCN::max_scn(); + SCN min_rec_scn = SCN::max_scn(); int min_rec_scn_common_checkpoint_type_index = 0; char common_checkpoint_type[common::MAX_CHECKPOINT_TYPE_BUF_LENGTH]; for (int i = 1; i < ObCommonCheckpointType::MAX_BASE_TYPE; i++) { if (OB_NOT_NULL(common_checkpoints_[i])) { - palf::SCN rec_scn = common_checkpoints_[i]->get_rec_scn(); + SCN rec_scn = common_checkpoints_[i]->get_rec_scn(); if (rec_scn.is_valid() && rec_scn < min_rec_scn) { min_rec_scn = rec_scn; min_rec_scn_common_checkpoint_type_index = i; @@ -440,7 +440,7 @@ palf::SCN ObLSTxService::get_rec_scn() return min_rec_scn; } -int ObLSTxService::flush(palf::SCN &recycle_scn) +int ObLSTxService::flush(SCN &recycle_scn) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -540,7 +540,7 @@ int ObLSTxService::traversal_flush() int tmp_ret = OB_SUCCESS; for (int i = 1; i < ObCommonCheckpointType::MAX_BASE_TYPE; i++) { if (OB_NOT_NULL(common_checkpoints_[i]) && - OB_SUCCESS != (tmp_ret = common_checkpoints_[i]->flush(palf::SCN::max_scn(), false))) { + OB_SUCCESS != (tmp_ret = common_checkpoints_[i]->flush(SCN::max_scn(), false))) { TRANS_LOG(WARN, "obCommonCheckpoint flush failed", K(tmp_ret), KP(common_checkpoints_[i])); } } @@ -553,7 +553,7 @@ void ObLSTxService::reset_() { } } -palf::SCN ObLSTxService::get_ls_weak_read_ts() { +SCN ObLSTxService::get_ls_weak_read_ts() { return parent_->get_ls_wrs_handler()->get_ls_weak_read_ts(); } diff --git a/src/storage/ls/ob_ls_tx_service.h b/src/storage/ls/ob_ls_tx_service.h index 0d6b14a23..ca61b90ff 100644 --- a/src/storage/ls/ob_ls_tx_service.h +++ b/src/storage/ls/ob_ls_tx_service.h @@ -24,7 +24,7 @@ namespace oceanbase { -namespace palf +namespace share { class SCN; } @@ -82,7 +82,7 @@ public: const bool read_latest, const int64_t lock_timeout, ObStoreCtx &store_ctx) const; - int get_read_store_ctx(const palf::SCN &snapshot_version, + int get_read_store_ctx(const share::SCN &snapshot_version, const int64_t lock_timeout, ObStoreCtx &store_ctx) const; int get_write_store_ctx(transaction::ObTxDesc &tx, @@ -128,16 +128,16 @@ public: // get the obj lock op iterator from tx of this ls. int iterate_tx_obj_lock_op(transaction::tablelock::ObLockOpIterator &iter) const; public: - int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const palf::SCN &scn); + int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn); - int replay_start_working_log(const transaction::ObTxStartWorkingLog &log, palf::SCN &log_ts_ns); + int replay_start_working_log(const transaction::ObTxStartWorkingLog &log, share::SCN &log_ts_ns); void switch_to_follower_forcedly(); int switch_to_leader(); int switch_to_follower_gracefully(); int resume_leader(); - palf::SCN get_rec_scn() override; - int flush(palf::SCN &recycle_scn) override; + share::SCN get_rec_scn() override; + int flush(share::SCN &recycle_scn) override; int get_common_checkpoint_info( ObIArray &common_checkpoint_array); @@ -154,7 +154,7 @@ public: const checkpoint::ObCommonCheckpoint* common_checkpoint); // undertake dump int traversal_flush(); - virtual palf::SCN get_ls_weak_read_ts(); + virtual share::SCN get_ls_weak_read_ts(); int check_in_leader_serving_state(bool& bool_ret); transaction::ObTxRetainCtxMgr *get_retain_ctx_mgr(); diff --git a/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp b/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp index 89effa010..37e359dbb 100644 --- a/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp +++ b/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp @@ -27,6 +27,7 @@ namespace oceanbase { using namespace storage; +using namespace share; using namespace transaction; using namespace common; using namespace blocksstable; @@ -43,10 +44,10 @@ ObMultiVersionValueIterator::ObMultiVersionValueIterator() version_iter_(NULL), multi_version_iter_(NULL), max_committed_trans_version_(-1), - cur_trans_version_(palf::SCN::min_scn()), + cur_trans_version_(SCN::min_scn()), is_node_compacted_(false), has_multi_commit_trans_(false), - merge_scn_(palf::SCN::max_scn()) + merge_scn_(SCN::max_scn()) { } @@ -131,7 +132,7 @@ void ObMultiVersionValueIterator::print_cur_status() int ObMultiVersionValueIterator::get_next_uncommitted_node( const void *&tnode, transaction::ObTransID &trans_id, - palf::SCN &trans_version, + SCN &trans_version, int64_t &sql_sequence) { int ret = OB_SUCCESS; @@ -145,7 +146,7 @@ int ObMultiVersionValueIterator::get_next_uncommitted_node( while (OB_SUCC(ret) && OB_ISNULL(tnode)) { if (OB_ISNULL(version_iter_)) { ret = OB_ITER_END; - } else if (palf::SCN::max_scn() == version_iter_->scn_) { + } else if (SCN::max_scn() == version_iter_->scn_) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "current trans node has not submit clog yet", K(ret), KPC_(version_iter)); } else if (NDT_COMPACT == version_iter_->type_) { // ignore compact node @@ -166,7 +167,7 @@ int ObMultiVersionValueIterator::get_next_uncommitted_node( TRANS_LOG(WARN, "trans_id is invalid", K(ret), K(trans_id), KPC(version_iter_), K(merge_scn_)); } else { tnode = static_cast(version_iter_); - trans_version = palf::SCN::max_scn(); + trans_version = SCN::max_scn(); sql_sequence = version_iter_->seq_no_; version_iter_ = version_iter_->prev_; } @@ -355,7 +356,7 @@ int ObMultiVersionValueIterator::get_next_multi_version_node(const void *&tnode) } else if (OB_ISNULL(multi_version_iter_)) { ret = OB_ITER_END; } else { - const palf::SCN cur_trans_version = multi_version_iter_->trans_version_; + const SCN cur_trans_version = multi_version_iter_->trans_version_; ObMvccTransNode *record_node = nullptr; while (OB_SUCC(ret) && OB_NOT_NULL(multi_version_iter_) && OB_ISNULL(record_node)) { if (multi_version_iter_->trans_version_.get_val_for_tx() <= version_range_.base_version_) { @@ -434,7 +435,7 @@ void ObMultiVersionValueIterator::reset() version_iter_ = NULL; multi_version_iter_ = NULL; max_committed_trans_version_ = -1; - cur_trans_version_ = palf::SCN::min_scn(); + cur_trans_version_ = SCN::min_scn(); is_node_compacted_ = false; has_multi_commit_trans_ = false; ctx_ = NULL; diff --git a/src/storage/memtable/mvcc/ob_multi_version_iterator.h b/src/storage/memtable/mvcc/ob_multi_version_iterator.h index 54c35222e..ef99aedc6 100644 --- a/src/storage/memtable/mvcc/ob_multi_version_iterator.h +++ b/src/storage/memtable/mvcc/ob_multi_version_iterator.h @@ -44,7 +44,7 @@ public: int get_next_uncommitted_node( const void *&tnode, transaction::ObTransID &trans_id, - palf::SCN &trans_version, + share::SCN &trans_version, int64_t &sql_sequence); int check_next_sql_sequence( const transaction::ObTransID &input_trans_id, @@ -59,8 +59,8 @@ public: bool is_trans_node_iter_null() const; bool is_compact_iter_end() const; int init_multi_version_iter(); - void set_merge_scn(const palf::SCN merge_scn) { merge_scn_ = merge_scn; } - palf::SCN get_merge_scn() const { return merge_scn_; } + void set_merge_scn(const share::SCN merge_scn) { merge_scn_ = merge_scn; } + share::SCN get_merge_scn() const { return merge_scn_; } void print_cur_status(); blocksstable::ObDmlFlag get_row_first_dml_flag() const { @@ -72,10 +72,10 @@ public: private: int get_trans_status_with_scn( - const palf::SCN scn, + const share::SCN scn, ObMvccTransNode *trans_node, int64_t &status, - palf::SCN &trans_version_at_merge_scn); + share::SCN &trans_version_at_merge_scn); int get_state_of_curr_trans_node( transaction::ObTransID &trans_id, int64_t &state, @@ -93,10 +93,10 @@ private: ObMvccTransNode *version_iter_; ObMvccTransNode *multi_version_iter_; int64_t max_committed_trans_version_; - palf::SCN cur_trans_version_; + share::SCN cur_trans_version_; bool is_node_compacted_; bool has_multi_commit_trans_; - palf::SCN merge_scn_; + share::SCN merge_scn_; }; //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/storage/memtable/mvcc/ob_mvcc.h b/src/storage/memtable/mvcc/ob_mvcc.h index c4118fe7d..c7b45777c 100644 --- a/src/storage/memtable/mvcc/ob_mvcc.h +++ b/src/storage/memtable/mvcc/ob_mvcc.h @@ -32,13 +32,13 @@ public: ObITransCallback() : need_fill_redo_(true), need_submit_log_(true), - scn_(palf::SCN::max_scn()), + scn_(share::SCN::max_scn()), prev_(NULL), next_(NULL) {} ObITransCallback(const bool need_fill_redo, const bool need_submit_log) : need_fill_redo_(need_fill_redo), need_submit_log_(need_submit_log), - scn_(palf::SCN::max_scn()), + scn_(share::SCN::max_scn()), prev_(NULL), next_(NULL) {} virtual ~ObITransCallback() {} @@ -53,8 +53,8 @@ public: virtual int del() { return remove(); } virtual bool is_need_free() const { return true; } virtual bool log_synced() const { return false; } - void set_scn(const palf::SCN scn); - palf::SCN get_scn() const; + void set_scn(const share::SCN scn); + share::SCN get_scn() const; int before_append_cb(const bool is_replay); int after_append_cb(const bool is_replay, const int ret_code); // interface for redo log generator @@ -63,14 +63,14 @@ public: virtual bool is_logging_blocked() const { return false; } int log_submitted_cb(); int undo_log_submitted_cb(); - int log_sync_cb(const palf::SCN scn); + int log_sync_cb(const share::SCN scn); int log_sync_fail_cb(); // interface should be implement by subclasses virtual int before_append(const bool is_replay) { return common::OB_SUCCESS; } virtual int after_append(const bool is_replay, const int ret_code) { return common::OB_SUCCESS; } virtual int log_submitted() { return common::OB_SUCCESS; } virtual int undo_log_submitted() { return common::OB_SUCCESS; } - virtual int log_sync(const palf::SCN scn) + virtual int log_sync(const share::SCN scn) { UNUSED(scn); return common::OB_SUCCESS; } virtual int log_sync_fail() { return common::OB_SUCCESS; } @@ -103,7 +103,7 @@ public: // the checksum system, you need be care of checksum_scn, you should only // execution the checksum if checksum_scn is smaller or equal than your // scn. - virtual int calc_checksum(const palf::SCN checksum_scn, + virtual int calc_checksum(const share::SCN checksum_scn, ObBatchChecksum *checksumer) { UNUSED(checksum_scn); @@ -155,7 +155,7 @@ protected: bool need_fill_redo_ : 1; // Identifies whether log is needed bool need_submit_log_ : 1; // Identifies whether log has been submitted }; - palf::SCN scn_; + share::SCN scn_; private: ObITransCallback *prev_; ObITransCallback *next_; diff --git a/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h index 8589c090d..8b2dac26d 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_acc_ctx.h @@ -79,7 +79,8 @@ public: } bool is_valid() const { switch(type_) { - case T::READ: return is_read_valid__(); + case T::STRONG_READ: return is_read_valid__(); + case T::WEAK_READ: return is_read_valid__(); case T::WRITE: return is_write_valid__(); case T::REPLAY: return is_replay_valid__(); default: return false; @@ -110,10 +111,11 @@ public: const storage::ObTxTableGuard &tx_table_guard, const transaction::ObTxSnapshot &snapshot, const int64_t abs_lock_timeout, - const int64_t tx_lock_timeout) + const int64_t tx_lock_timeout, + const bool is_weak_read) { reset(); - type_ = T::READ; + type_ = is_weak_read ? T::WEAK_READ : T::STRONG_READ; tx_ctx_ = tx_ctx; mem_ctx_ = mem_ctx; tx_table_guard_ = tx_table_guard; @@ -123,13 +125,13 @@ public: } // light read, used by storage background merge/compaction routine void init_read(const storage::ObTxTableGuard &tx_table_guard, - const palf::SCN snapshot_version, + const share::SCN snapshot_version, const int64_t timeout, const int64_t tx_lock_timeout) { transaction::ObTxSnapshot snapshot; snapshot.version_ = snapshot_version; - init_read(NULL, NULL, tx_table_guard, snapshot, timeout, tx_lock_timeout); + init_read(NULL, NULL, tx_table_guard, snapshot, timeout, tx_lock_timeout, false); } void init_write(transaction::ObPartTransCtx &tx_ctx, ObMemtableCtx &mem_ctx, @@ -166,7 +168,7 @@ public: const transaction::ObTransID &get_tx_id() const { return tx_id_; } - palf::SCN get_snapshot_version() const { + share::SCN get_snapshot_version() const { return snapshot_.version_; } storage::ObTxTable *get_tx_table() const { @@ -178,7 +180,8 @@ public: ObMemtableCtx *get_mem_ctx() const { return mem_ctx_; } - bool is_read() const { return type_ == T::READ; } + bool is_read() const { return type_ == T::STRONG_READ || type_ == T::WEAK_READ; } + bool is_weak_read() const { return type_ == T::WEAK_READ; } bool is_write() const { return type_ == T::WRITE; } bool is_replay() const { return type_ == T::REPLAY; } int64_t eval_lock_expire_ts(int64_t lock_wait_start_ts = 0) const { @@ -213,7 +216,7 @@ public: private: void warn_tx_ctx_leaky_(); public: // NOTE: those field should only be accessed by txn relative routine - enum class T { INVL, READ, WRITE, REPLAY } type_; + enum class T { INVL, STRONG_READ, WEAK_READ, WRITE, REPLAY } type_; // abs_lock_timeout is calculated from the minimum of the wait time of the // select_for_update and timeout in dml_param / scan_param int64_t abs_lock_timeout_; diff --git a/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp index 07eaee8e1..a1ef77ac5 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_ctx.cpp @@ -21,13 +21,14 @@ namespace oceanbase { using namespace common; +using namespace share; using namespace storage; using namespace transaction::tablelock; using namespace blocksstable; namespace memtable { -void ObIMvccCtx::before_prepare(const palf::SCN version) +void ObIMvccCtx::before_prepare(const SCN version) { #ifdef TRANS_ERROR const int random = (int)ObRandom::rand(1, 1000); @@ -38,8 +39,8 @@ void ObIMvccCtx::before_prepare(const palf::SCN version) bool ObIMvccCtx::is_prepared() const { - const palf::SCN prepare_version = trans_version_.atomic_get(); - return (prepare_version >= palf::SCN::min_scn() && palf::SCN::max_scn() != prepare_version); + const SCN prepare_version = trans_version_.atomic_get(); + return (prepare_version >= SCN::min_scn() && SCN::max_scn() != prepare_version); } int ObIMvccCtx::inc_pending_log_size(const int64_t size) @@ -100,7 +101,7 @@ int ObIMvccCtx::register_row_replay_cb( const int64_t data_size, ObMemtable *memtable, const int64_t seq_no, - const palf::SCN scn) + const SCN scn) { int ret = OB_SUCCESS; const bool is_replay = true; @@ -193,7 +194,7 @@ int ObIMvccCtx::register_table_lock_cb( int ObIMvccCtx::register_table_lock_replay_cb( ObLockMemtable *memtable, ObMemCtxLockOpLinkNode *lock_op, - const palf::SCN scn) + const SCN scn) { int ret = OB_SUCCESS; ObOBJLockCallback *cb = nullptr; diff --git a/src/storage/memtable/mvcc/ob_mvcc_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_ctx.h index 1c725dc4e..db2850bfe 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_ctx.h @@ -62,13 +62,13 @@ public: min_table_version_(0), //记录一个事务内存最大的一次table version max_table_version_(0), - trans_version_(palf::SCN::max_scn()), - commit_version_(palf::SCN::min_scn()), + trans_version_(share::SCN::max_scn()), + commit_version_(share::SCN::min_scn()), lock_start_time_(0), - redo_scn_(palf::SCN::min_scn()), + redo_scn_(share::SCN::min_scn()), redo_log_id_(0), lock_wait_start_ts_(0), - replay_compact_version_(palf::SCN::min_scn()) + replay_compact_version_(share::SCN::min_scn()) { } virtual ~ObIMvccCtx() {} @@ -91,8 +91,8 @@ public: // for mvcc engine invoke virtual void add_lock_for_read_elapse(const int64_t n) = 0; virtual int64_t get_lock_for_read_elapse() const = 0; virtual void on_tsc_retry(const ObMemtableKey& key, - const palf::SCN snapshot_version, - const palf::SCN max_trans_version, + const share::SCN snapshot_version, + const share::SCN max_trans_version, const transaction::ObTransID &conflict_tx_id) = 0; virtual void on_wlock_retry(const ObMemtableKey& key, const transaction::ObTransID &conflict_tx_id) = 0; virtual bool is_can_elr() const = 0; @@ -106,19 +106,19 @@ public: // for mvcc engine invoke virtual void dec_unsubmitted_cnt() = 0; virtual void inc_unsynced_cnt() = 0; virtual void dec_unsynced_cnt() = 0; - virtual palf::SCN get_tx_end_scn() const { return palf::SCN::max_scn(); }; + virtual share::SCN get_tx_end_scn() const { return share::SCN::max_scn(); }; public: inline int get_alloc_type() const { return alloc_type_; } - inline palf::SCN get_trans_version() const { return trans_version_.atomic_get(); } - inline palf::SCN get_commit_version() const { return commit_version_.atomic_get(); } + inline share::SCN get_trans_version() const { return trans_version_.atomic_get(); } + inline share::SCN get_commit_version() const { return commit_version_.atomic_get(); } inline int64_t get_min_table_version() const { return min_table_version_; } inline int64_t get_max_table_version() const { return max_table_version_; } inline void set_alloc_type(const int alloc_type) { alloc_type_ = alloc_type; } - void before_prepare(const palf::SCN version = palf::SCN::min_scn()); + void before_prepare(const share::SCN version = share::SCN::min_scn()); bool is_prepared() const; - inline void set_prepare_version(const palf::SCN version) { set_trans_version(version); } - inline void set_trans_version(const palf::SCN trans_version) { trans_version_.atomic_set(trans_version); } - inline void set_commit_version(const palf::SCN trans_version) { commit_version_.atomic_set(trans_version); } + inline void set_prepare_version(const share::SCN version) { set_trans_version(version); } + inline void set_trans_version(const share::SCN trans_version) { trans_version_.atomic_set(trans_version); } + inline void set_commit_version(const share::SCN trans_version) { commit_version_.atomic_set(trans_version); } inline void set_table_version(const int64_t table_version) { if (INT64_MAX == min_table_version_) { @@ -139,19 +139,19 @@ public: max_table_version_ = table_version; } } - inline bool is_commit_version_valid() const { return commit_version_ != palf::SCN::min_scn() && commit_version_ != palf::SCN::max_scn(); } + inline bool is_commit_version_valid() const { return commit_version_ != share::SCN::min_scn() && commit_version_ != share::SCN::max_scn(); } inline void set_lock_start_time(const int64_t start_time) { lock_start_time_ = start_time; } inline int64_t get_lock_start_time() { return lock_start_time_; } inline void set_for_replay(const bool for_replay) { trans_mgr_.set_for_replay(for_replay); } inline bool is_for_replay() const { return trans_mgr_.is_for_replay(); } - inline void set_redo_scn(const palf::SCN redo_scn) { redo_scn_ = redo_scn; } + inline void set_redo_scn(const share::SCN redo_scn) { redo_scn_ = redo_scn; } inline void set_redo_log_id(const int64_t redo_log_id) { redo_log_id_ = redo_log_id; } - inline palf::SCN get_redo_scn() const { return redo_scn_; } + inline share::SCN get_redo_scn() const { return redo_scn_; } inline int64_t get_redo_log_id() const { return redo_log_id_; } inline void set_lock_wait_start_ts(const int64_t lock_wait_start_ts) { lock_wait_start_ts_ = lock_wait_start_ts; } - palf::SCN get_replay_compact_version() const { return replay_compact_version_; } - void set_replay_compact_version(const palf::SCN v) { replay_compact_version_ = v; } + share::SCN get_replay_compact_version() const { return replay_compact_version_; } + void set_replay_compact_version(const share::SCN v) { replay_compact_version_ = v; } inline int64_t get_lock_wait_start_ts() const { return lock_wait_start_ts_; } void acquire_callback_list() { trans_mgr_.acquire_callback_list(); } void revert_callback_list() { trans_mgr_.revert_callback_list(); } @@ -171,14 +171,14 @@ public: const int64_t data_size, ObMemtable *memtable, const int64_t seq_no, - const palf::SCN scn); + const share::SCN scn); int register_table_lock_cb( transaction::tablelock::ObLockMemtable *memtable, ObMemCtxLockOpLinkNode *lock_op); int register_table_lock_replay_cb( ObLockMemtable *memtable, ObMemCtxLockOpLinkNode *lock_op, - const palf::SCN scn); + const share::SCN scn); int inc_pending_log_size(const int64_t size); public: virtual void reset() @@ -187,13 +187,13 @@ public: trans_mgr_.reset(); min_table_version_ = INT64_MAX; max_table_version_ = 0; - trans_version_ = palf::SCN::max_scn(); - commit_version_ = palf::SCN::min_scn(); + trans_version_ = share::SCN::max_scn(); + commit_version_ = share::SCN::min_scn(); lock_start_time_ = 0; - redo_scn_ = palf::SCN::min_scn(); + redo_scn_ = share::SCN::min_scn(); redo_log_id_ = 0; lock_wait_start_ts_ = 0; - replay_compact_version_ = palf::SCN::min_scn(); + replay_compact_version_ = share::SCN::min_scn(); } virtual int64_t to_string(char *buf, const int64_t buf_len) const { @@ -240,13 +240,13 @@ protected: ObTransCallbackMgr trans_mgr_; int64_t min_table_version_; int64_t max_table_version_; - palf::SCN trans_version_; - palf::SCN commit_version_; + share::SCN trans_version_; + share::SCN commit_version_; int64_t lock_start_time_; - palf::SCN redo_scn_; + share::SCN redo_scn_; int64_t redo_log_id_; int64_t lock_wait_start_ts_; - palf::SCN replay_compact_version_; + share::SCN replay_compact_version_; }; class ObMemtableCtx; diff --git a/src/storage/memtable/mvcc/ob_mvcc_define.h b/src/storage/memtable/mvcc/ob_mvcc_define.h index 67150b34a..c3e25d7b5 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_define.h +++ b/src/storage/memtable/mvcc/ob_mvcc_define.h @@ -43,7 +43,7 @@ struct ObTxNodeArg // seq_no_ is the sequence no of the executing sql int64_t seq_no_; // scn_ is thee log ts of the redo log - palf::SCN scn_; + share::SCN scn_; TO_STRING_KV(KP_(data), KP_(old_row), @@ -64,7 +64,7 @@ struct ObTxNodeArg acc_checksum_(0), memstore_version_(memstore_version), seq_no_(seq_no), - scn_(palf::SCN::max_scn()) {} + scn_(share::SCN::max_scn()) {} // Constructor for follower ObTxNodeArg(const ObMemtableData *data, @@ -73,7 +73,7 @@ struct ObTxNodeArg const int64_t seq_no, const uint32_t modify_count, const uint32_t acc_checksum, - const palf::SCN scn) + const share::SCN scn) : data_(data), old_row_(old_row), modify_count_(modify_count), @@ -89,7 +89,7 @@ struct ObTxNodeArg acc_checksum_ = 0; memstore_version_ = 0; seq_no_ = 0; - scn_ = palf::SCN::min_scn(); + scn_ = share::SCN::min_scn(); } }; diff --git a/src/storage/memtable/mvcc/ob_mvcc_engine.cpp b/src/storage/memtable/mvcc/ob_mvcc_engine.cpp index b361a69b8..9410c646c 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_engine.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_engine.cpp @@ -21,6 +21,7 @@ namespace oceanbase { using namespace common; +using namespace share; using namespace storage; using namespace transaction; namespace memtable @@ -78,16 +79,16 @@ void ObMvccEngine::destroy() memtable_ = NULL; } -int ObMvccEngine::try_compact_row_when_mvcc_read_(const palf::SCN &snapshot_version, +int ObMvccEngine::try_compact_row_when_mvcc_read_(const SCN &snapshot_version, ObMvccRow &row) { int ret = OB_SUCCESS; const int64_t latest_compact_ts = row.latest_compact_ts_; const int64_t WEAK_READ_COMPACT_THRESHOLD = 3 * 1000 * 1000; - if (palf::SCN::min_scn() >= snapshot_version) { + if (SCN::min_scn() >= snapshot_version) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "invalid snapshot version", K(ret), K(snapshot_version)); - } else if (palf::SCN::max_scn() == snapshot_version + } else if (SCN::max_scn() == snapshot_version || ObTimeUtility::current_time() < latest_compact_ts + WEAK_READ_COMPACT_THRESHOLD) { // do not compact row when merging } else { @@ -306,7 +307,7 @@ int ObMvccEngine::create_kv( } int ObMvccEngine::mvcc_write(ObIMemtableCtx &ctx, - const palf::SCN snapshot_version, + const SCN snapshot_version, ObMvccRow &value, const ObTxNodeArg &arg, ObMvccWriteResult &res) @@ -359,7 +360,7 @@ int ObMvccEngine::build_tx_node_(ObIMemtableCtx &ctx, TRANS_LOG(WARN, "MvccTranNode dup fail", K(ret), "node", node); } else { node->tx_id_ = ctx.get_tx_id(); - node->trans_version_ = palf::SCN::max_scn(); + node->trans_version_ = SCN::max_scn(); node->modify_count_ = arg.modify_count_; node->acc_checksum_ = arg.acc_checksum_; node->version_ = arg.memstore_version_; diff --git a/src/storage/memtable/mvcc/ob_mvcc_engine.h b/src/storage/memtable/mvcc/ob_mvcc_engine.h index 1fc7164b9..3813247dc 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_engine.h +++ b/src/storage/memtable/mvcc/ob_mvcc_engine.h @@ -68,7 +68,7 @@ public: // OB_TRANSACTION_SET_VIOLATION if encountering lost update. The interesting // implementation about mvcc_write is located in ob_mvcc_row.cpp/.h int mvcc_write(ObIMemtableCtx &ctx, - const palf::SCN snapshot_version, + const share::SCN snapshot_version, ObMvccRow &value, const ObTxNodeArg &arg, ObMvccWriteResult &res); @@ -115,7 +115,7 @@ public: int estimate_scan_row_count(const ObMvccScanRange &range, storage::ObPartitionEst &part_est) const; private: - int try_compact_row_when_mvcc_read_(const palf::SCN &snapshot_version, + int try_compact_row_when_mvcc_read_(const share::SCN &snapshot_version, ObMvccRow &row); int build_tx_node_(ObIMemtableCtx &ctx, diff --git a/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp b/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp index 93b0e4b35..52520d674 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp @@ -23,6 +23,7 @@ namespace oceanbase { using namespace storage; +using namespace share; using namespace transaction; using namespace common; namespace memtable @@ -90,6 +91,17 @@ int ObMvccValueIterator::lock_for_read_(const ObQueryFlag &flag) } } + // add barrier snapshot version for defensive check + if (NULL != version_iter_) { + if (ctx_->is_weak_read()) { + version_iter_->set_safe_read_barrier(true); + } + if (!flag.is_prewarm() + && !version_iter_->is_elr()) { + version_iter_->set_snapshot_version_barrier(ctx_->snapshot_.version_); + } + } + lock_for_read_end(lock_start_time, ret); return ret; } @@ -137,7 +149,7 @@ int ObMvccValueIterator::lock_for_read_inner_(const ObQueryFlag &flag, const int64_t data_seq_no = iter->get_seq_no(); const int64_t snapshot_seq_no = ctx_->snapshot_.scn_; - const palf::SCN snapshot_version = ctx_->get_snapshot_version(); + const SCN snapshot_version = ctx_->get_snapshot_version(); const int64_t read_epoch = ctx_->get_tx_table_guard().epoch(); const bool read_latest = flag.is_read_latest(); ObTxTable *tx_table = ctx_->get_tx_table_guard().get_tx_table(); @@ -154,7 +166,7 @@ int ObMvccValueIterator::lock_for_read_inner_(const ObQueryFlag &flag, // because inner tx read only care whether tx node rollbacked if (iter->is_committed() || iter->is_elr()) { // Case 2: Data is committed, so the state is decided - const palf::SCN data_version = iter->trans_version_; + const SCN data_version = iter->trans_version_; if (snapshot_version >= data_version) { // Case 2.1 Read the version if it is smaller than read version version_iter_ = iter; @@ -198,7 +210,7 @@ int ObMvccValueIterator::lock_for_read_inner_(const ObQueryFlag &flag, // is_delay_cleanout() to check the state and we only cleanout it // when data is delay cleanout bool can_read = false; - palf::SCN data_version = palf::SCN::max_scn(); + SCN data_version = SCN::max_scn(); bool is_determined_state = false; // Opt3: we only cleanout tx node who is delay cleanout ObCleanoutOp cleanout_op; diff --git a/src/storage/memtable/mvcc/ob_mvcc_iterator.h b/src/storage/memtable/mvcc/ob_mvcc_iterator.h index 439056c24..5c72a39f0 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_iterator.h +++ b/src/storage/memtable/mvcc/ob_mvcc_iterator.h @@ -104,7 +104,7 @@ public: ctx_(NULL), value_(NULL), version_iter_(NULL), - last_trans_version_(palf::SCN::max_scn()), + last_trans_version_(share::SCN::max_scn()), skip_compact_(false) { } @@ -126,7 +126,7 @@ public: ctx_ = NULL; value_ = NULL; version_iter_ = NULL; - last_trans_version_ = palf::SCN::max_scn(); + last_trans_version_ = share::SCN::max_scn(); } const ObMvccRow *get_mvcc_row() const { return value_; } const ObMvccTransNode *get_trans_node() const { return version_iter_; } @@ -146,7 +146,7 @@ private: ObMvccAccessCtx *ctx_; ObMvccRow *value_; ObMvccTransNode *version_iter_; - palf::SCN last_trans_version_; + share::SCN last_trans_version_; bool skip_compact_; }; diff --git a/src/storage/memtable/mvcc/ob_mvcc_row.cpp b/src/storage/memtable/mvcc/ob_mvcc_row.cpp index c29fd8516..1d830c4fa 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_row.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_row.cpp @@ -33,6 +33,7 @@ namespace oceanbase { using namespace storage; +using namespace share; using namespace transaction; using namespace common; using namespace share; @@ -129,7 +130,7 @@ bool ObMvccTransNode::is_safe_read_barrier() const || (flag & F_STRONG_CONSISTENT_READ_BARRIER)); } -void ObMvccTransNode::set_snapshot_version_barrier(const palf::SCN version) +void ObMvccTransNode::set_snapshot_version_barrier(const SCN version) { snapshot_version_barrier_ = version; } @@ -204,19 +205,19 @@ bool ObMvccTransNode::is_delayed_cleanout() const return ATOMIC_LOAD(&flag_) & F_DELAYED_CLEANOUT; } -int ObMvccTransNode::fill_trans_version(const palf::SCN version) +int ObMvccTransNode::fill_trans_version(const SCN version) { trans_version_ = version; return OB_SUCCESS; } -int ObMvccTransNode::fill_scn(const palf::SCN scn) +int ObMvccTransNode::fill_scn(const SCN scn) { scn_ = scn; return OB_SUCCESS; } -void ObMvccTransNode::trans_commit(const palf::SCN commit_version, const palf::SCN tx_end_scn) +void ObMvccTransNode::trans_commit(const SCN commit_version, const SCN tx_end_scn) { // NB: we need set commit version before set committed fill_trans_version(commit_version); @@ -224,7 +225,7 @@ void ObMvccTransNode::trans_commit(const palf::SCN commit_version, const palf::S set_tx_end_scn(tx_end_scn); } -void ObMvccTransNode::trans_abort(const palf::SCN tx_end_scn) +void ObMvccTransNode::trans_abort(const SCN tx_end_scn) { set_aborted(); set_tx_end_scn(tx_end_scn); @@ -323,8 +324,8 @@ void ObMvccRow::reset() first_dml_flag_ = ObDmlFlag::DF_NOT_EXIST; last_dml_flag_ = ObDmlFlag::DF_NOT_EXIST; list_head_ = NULL; - max_trans_version_ = palf::SCN::min_scn(); - max_elr_trans_version_ = palf::SCN::min_scn(); + max_trans_version_ = SCN::min_scn(); + max_elr_trans_version_ = SCN::min_scn(); latest_compact_node_ = NULL; latest_compact_ts_ = 0; index_ = NULL; @@ -549,7 +550,7 @@ bool ObMvccRow::need_compact(const bool for_read, const bool for_replay) int ObMvccRow::row_compact(ObMemtable *memtable, const bool for_replay, - const palf::SCN snapshot_version, + const SCN snapshot_version, ObIAllocator *node_alloc) { int ret = OB_SUCCESS; @@ -692,14 +693,14 @@ int ObMvccRow::insert_trans_node(ObIMvccCtx &ctx, return ret; } -bool ObMvccRow::is_transaction_set_violation(const palf::SCN snapshot_version) +bool ObMvccRow::is_transaction_set_violation(const SCN snapshot_version) { return max_trans_version_ > snapshot_version || max_elr_trans_version_ > snapshot_version; } int ObMvccRow::elr(const ObTransID &tx_id, - const palf::SCN elr_commit_version, + const SCN elr_commit_version, const ObTabletID &tablet_id, const ObMemtableKey* key) { @@ -712,7 +713,7 @@ int ObMvccRow::elr(const ObTransID &tx_id, while (NULL != iter && OB_SUCC(ret)) { if (tx_id != iter->tx_id_) { break; - } else if (palf::SCN::max_scn() != iter->trans_version_ && iter->trans_version_ > elr_commit_version) { + } else if (SCN::max_scn() != iter->trans_version_ && iter->trans_version_ > elr_commit_version) { // leader revoke ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "unexected transaction version", K(*iter), K(elr_commit_version)); @@ -762,28 +763,28 @@ void ObMvccRow::mvcc_write_end(ObIMemtableCtx &ctx, int64_t ret) const } } -palf::SCN ObMvccRow::get_max_trans_version() const +SCN ObMvccRow::get_max_trans_version() const { - const palf::SCN max_elr_commit_version = max_elr_trans_version_.atomic_get(); - const palf::SCN max_trans_version = max_trans_version_.atomic_get(); + const SCN max_elr_commit_version = max_elr_trans_version_.atomic_get(); + const SCN max_trans_version = max_trans_version_.atomic_get(); return MAX(max_elr_commit_version, max_trans_version); } -void ObMvccRow::update_max_trans_version(const palf::SCN max_trans_version, +void ObMvccRow::update_max_trans_version(const SCN max_trans_version, const transaction::ObTransID &tx_id) { - palf::SCN v = max_trans_version_.inc_update(max_trans_version); + SCN v = max_trans_version_.inc_update(max_trans_version); if (v == max_trans_version) { max_trans_id_ = tx_id; } } -void ObMvccRow::update_max_elr_trans_version(const palf::SCN max_trans_version, +void ObMvccRow::update_max_elr_trans_version(const SCN max_trans_version, const transaction::ObTransID &tx_id) { - palf::SCN v = max_elr_trans_version_.inc_update(max_trans_version); + SCN v = max_elr_trans_version_.inc_update(max_trans_version); if (v == max_trans_version) { max_elr_trans_id_ = tx_id; } } -int ObMvccRow::trans_commit(const palf::SCN commit_version, ObMvccTransNode &node) +int ObMvccRow::trans_commit(const SCN commit_version, ObMvccTransNode &node) { int ret = OB_SUCCESS; @@ -878,7 +879,7 @@ int ObMvccRow::wakeup_waiter(const ObTabletID &tablet_id, int ObMvccRow::mvcc_write_(ObIMemtableCtx &ctx, ObMvccTransNode &writer_node, - const palf::SCN snapshot_version, + const SCN snapshot_version, ObMvccWriteResult &res) { int ret = OB_SUCCESS; @@ -1007,7 +1008,7 @@ int ObMvccRow::mvcc_write_(ObIMemtableCtx &ctx, return ret; } -int ObMvccRow::check_double_insert_(const palf::SCN snapshot_version, +int ObMvccRow::check_double_insert_(const SCN snapshot_version, ObMvccTransNode &node, ObMvccTransNode *prev) { @@ -1044,7 +1045,7 @@ void ObMvccRow::mvcc_undo() } int ObMvccRow::mvcc_write(ObIMemtableCtx &ctx, - const palf::SCN snapshot_version, + const SCN snapshot_version, ObMvccTransNode &node, ObMvccWriteResult &res) { diff --git a/src/storage/memtable/mvcc/ob_mvcc_row.h b/src/storage/memtable/mvcc/ob_mvcc_row.h index 22a9a5468..fea68e61b 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_row.h +++ b/src/storage/memtable/mvcc/ob_mvcc_row.h @@ -45,32 +45,32 @@ struct ObMvccTransNode public: ObMvccTransNode() : tx_id_(), - trans_version_(palf::SCN::min_scn()), - scn_(palf::SCN::max_scn()), + trans_version_(share::SCN::min_scn()), + scn_(share::SCN::max_scn()), seq_no_(0), - tx_end_scn_(palf::SCN::max_scn()), + tx_end_scn_(share::SCN::max_scn()), prev_(NULL), next_(NULL), modify_count_(0), acc_checksum_(0), version_(0), - snapshot_version_barrier_(palf::SCN::min_scn()), + snapshot_version_barrier_(share::SCN::min_scn()), type_(NDT_NORMAL), flag_(0) {} ~ObMvccTransNode() {} transaction::ObTransID tx_id_; - palf::SCN trans_version_; - palf::SCN scn_; + share::SCN trans_version_; + share::SCN scn_; int64_t seq_no_; - palf::SCN tx_end_scn_; + share::SCN tx_end_scn_; ObMvccTransNode *prev_; ObMvccTransNode *next_; uint32_t modify_count_; uint32_t acc_checksum_; int64_t version_; - palf::SCN snapshot_version_barrier_; + share::SCN snapshot_version_barrier_; uint8_t type_; uint8_t flag_; char buf_[0]; @@ -86,9 +86,9 @@ public: // trans_commit/abort commit/abort the tx node // fill in the version and set committed flag - void trans_commit(const palf::SCN commit_version, const palf::SCN tx_end_scn); + void trans_commit(const share::SCN commit_version, const share::SCN tx_end_scn); // set aborted flag - void trans_abort(const palf::SCN tx_end_scn); + void trans_abort(const share::SCN tx_end_scn); // remove the callback void remove_callback(); @@ -101,7 +101,7 @@ public: void set_safe_read_barrier(const bool is_weak_consistent_read); void clear_safe_read_barrier(); bool is_safe_read_barrier() const; - void set_snapshot_version_barrier(const palf::SCN version); + void set_snapshot_version_barrier(const share::SCN version); // ===================== ObMvccTransNode Flag Interface ===================== void set_committed(); @@ -117,21 +117,21 @@ public: // ===================== ObMvccTransNode Setter/Getter ===================== blocksstable::ObDmlFlag get_dml_flag() const; - int fill_trans_version(const palf::SCN version); - int fill_scn(const palf::SCN scn); + int fill_trans_version(const share::SCN version); + int fill_scn(const share::SCN scn); void get_trans_id_and_seq_no(transaction::ObTransID &trans_id, int64_t &seq_no); int64_t get_seq_no() const { return seq_no_; } transaction::ObTransID get_tx_id() const { return tx_id_; } void set_seq_no(const int64_t seq_no) { seq_no_ = seq_no; } int is_lock_node(bool &is_lock) const; int64_t to_string(char *buf, const int64_t buf_len) const; - void set_tx_end_scn(const palf::SCN tx_end_scn) + void set_tx_end_scn(const share::SCN tx_end_scn) { - if (palf::SCN::max_scn() != tx_end_scn) { + if (share::SCN::max_scn() != tx_end_scn) { tx_end_scn_ = tx_end_scn; } } - palf::SCN get_tx_end_scn() { return tx_end_scn_; } + share::SCN get_tx_end_scn() { return tx_end_scn_; } private: static const uint8_t F_INIT; @@ -192,9 +192,9 @@ struct ObMvccRow blocksstable::ObDmlFlag last_dml_flag_; ObMvccTransNode *list_head_; transaction::ObTransID max_trans_id_; - palf::SCN max_trans_version_; + share::SCN max_trans_version_; transaction::ObTransID max_elr_trans_id_; - palf::SCN max_elr_trans_version_; + share::SCN max_elr_trans_version_; ObMvccTransNode *latest_compact_node_; // using for optimizing inserting trans node when replaying ObMvccRowIndex *index_; @@ -216,7 +216,7 @@ struct ObMvccRow // is_new_locked returns whether node represents the first lock for the operation // conflict_tx_id if write failed this field indicate the txn-id which hold the lock of current row int mvcc_write(ObIMemtableCtx &ctx, - const palf::SCN snapshot_version, + const share::SCN snapshot_version, ObMvccTransNode &node, ObMvccWriteResult &res); @@ -255,11 +255,11 @@ struct ObMvccRow // node_alloc is the allocator for compact node allocation int row_compact(ObMemtable *memtable, const bool for_replay, - const palf::SCN snapshot_version, + const share::SCN snapshot_version, common::ObIAllocator *node_alloc); int elr(const transaction::ObTransID &tx_id, - const palf::SCN elr_commit_version, + const share::SCN elr_commit_version, const ObTabletID &tablet_id, const ObMemtableKey* key); @@ -269,7 +269,7 @@ struct ObMvccRow // - max_elr_trans_version // - first_dml // - last_dml - int trans_commit(const palf::SCN commit_version, + int trans_commit(const share::SCN commit_version, ObMvccTransNode &node); // remove_callback remove the tx node in the row @@ -285,7 +285,7 @@ struct ObMvccRow // TODO(handora.qc): handle it properly bool is_del(const int64_t version) const; // is_transaction_set_violation check the tsc problem for the row - bool is_transaction_set_violation(const palf::SCN snapshot_version); + bool is_transaction_set_violation(const share::SCN snapshot_version); // ===================== ObMvccRow Getter Interface ===================== // need_compact checks whether the compaction is necessary @@ -303,11 +303,11 @@ struct ObMvccRow blocksstable::ObDmlFlag get_first_dml_flag() const { return first_dml_flag_; } // max_trans_version/max_elr_trans_version is the max (elr) version on the row - palf::SCN get_max_trans_version() const; + share::SCN get_max_trans_version() const; int64_t get_max_trans_id() const { return max_trans_id_; } - void update_max_trans_version(const palf::SCN max_trans_version, + void update_max_trans_version(const share::SCN max_trans_version, const transaction::ObTransID &tx_id); - void update_max_elr_trans_version(const palf::SCN max_trans_version, + void update_max_elr_trans_version(const share::SCN max_trans_version, const transaction::ObTransID &tx_id); int64_t get_total_trans_node_cnt() const { return total_trans_node_cnt_; } int64_t get_last_compact_cnt() const { return last_compact_cnt_; } @@ -334,12 +334,12 @@ struct ObMvccRow // ===================== ObMvccRow Private Function ===================== int mvcc_write_(ObIMemtableCtx &ctx, ObMvccTransNode &node, - const palf::SCN snapshot_version, + const share::SCN snapshot_version, ObMvccWriteResult &res); // ===================== ObMvccRow Protection Code ===================== // check double insert - int check_double_insert_(const palf::SCN snapshot_version, + int check_double_insert_(const share::SCN snapshot_version, ObMvccTransNode &node, ObMvccTransNode *prev); }; diff --git a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp index 0293d9d6b..c906cf77b 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp @@ -28,6 +28,7 @@ namespace oceanbase { using namespace common; +using namespace share; using namespace transaction; namespace memtable { @@ -74,14 +75,14 @@ void TableLockRedoDataNode::set( create_schema_version_ = lock_op.create_schema_version_; } -void ObITransCallback::set_scn(const palf::SCN scn) +void ObITransCallback::set_scn(const SCN scn) { - if (palf::SCN::max_scn() == scn_) { + if (SCN::max_scn() == scn_) { scn_ = scn; } } -palf::SCN ObITransCallback::get_scn() const +SCN ObITransCallback::get_scn() const { return scn_; } @@ -126,11 +127,11 @@ int ObITransCallback::undo_log_submitted_cb() return ret; } -int ObITransCallback::log_sync_cb(const palf::SCN scn) +int ObITransCallback::log_sync_cb(const SCN scn) { int ret = OB_SUCCESS; if (!need_fill_redo_) { - } else if (OB_UNLIKELY(palf::SCN::max_scn() == scn)) { + } else if (OB_UNLIKELY(SCN::max_scn() == scn)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "log ts should not be invalid", K(ret), K(scn), K(*this)); } else if (OB_SUCC(log_sync(scn))) { @@ -414,13 +415,13 @@ int ObTransCallbackMgr::clean_unlog_callbacks(int64_t &removed_cnt) return ret; } -int ObTransCallbackMgr::calc_checksum_before_scn(const palf::SCN scn, +int ObTransCallbackMgr::calc_checksum_before_scn(const SCN scn, uint64_t &checksum, - palf::SCN &checksum_scn) + SCN &checksum_scn) { int ret = OB_SUCCESS; - if (palf::SCN::max_scn() == scn) { + if (SCN::max_scn() == scn) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "log ts is invalid", K(scn)); } else if (OB_FAIL(callback_list_.tx_calc_checksum_before_scn(scn))) { @@ -433,7 +434,7 @@ int ObTransCallbackMgr::calc_checksum_before_scn(const palf::SCN scn, } void ObTransCallbackMgr::update_checksum(const uint64_t checksum, - const palf::SCN checksum_scn) + const SCN checksum_scn) { callback_list_.update_checksum(checksum, checksum_scn); } @@ -557,12 +558,12 @@ void ObTransCallbackMgr::set_for_replay(const bool for_replay) } } -int ObTransCallbackMgr::replay_fail(const palf::SCN scn) +int ObTransCallbackMgr::replay_fail(const SCN scn) { return callback_list_.replay_fail(scn); } -int ObTransCallbackMgr::replay_succ(const palf::SCN scn) +int ObTransCallbackMgr::replay_succ(const SCN scn) { return OB_SUCCESS; } @@ -854,7 +855,7 @@ ObTransCtx *ObMvccRowCallback::get_trans_ctx() const return trans_ctx; } -int ObMvccRowCallback::calc_checksum(const palf::SCN checksum_scn, +int ObMvccRowCallback::calc_checksum(const SCN checksum_scn, ObBatchChecksum *checksumer) { ObRowLatchGuard guard(value_.latch_); @@ -975,11 +976,11 @@ int ObMvccRowCallback::trans_commit() (void)ATOMIC_FAA(&value_.update_since_compact_, 1); if (value_.need_compact(for_read, ctx_.is_for_replay())) { if (ctx_.is_for_replay()) { - if (palf::SCN::min_scn() != ctx_.get_replay_compact_version() && palf::SCN::max_scn() != ctx_.get_replay_compact_version()) { + if (SCN::min_scn() != ctx_.get_replay_compact_version() && SCN::max_scn() != ctx_.get_replay_compact_version()) { memtable_->row_compact(&value_, ctx_.is_for_replay(), ctx_.get_replay_compact_version()); } } else { - palf::SCN snapshot_version_for_compact = palf::SCN::minus(palf::SCN::max_scn(), 100); + SCN snapshot_version_for_compact = SCN::minus(SCN::max_scn(), 100); memtable_->row_compact(&value_, ctx_.is_for_replay(), snapshot_version_for_compact); } } @@ -1054,7 +1055,7 @@ int ObMvccRowCallback::rollback_callback() if (need_submit_log_ && need_fill_redo_ - && palf::SCN::max_scn() == scn_) { + && SCN::max_scn() == scn_) { ctx_.inc_pending_log_size(-1 * data_size_); } @@ -1166,7 +1167,7 @@ int64_t ObMvccRowCallback::to_string(char *buf, const int64_t buf_len) const return pos; } -int ObMvccRowCallback::log_sync(const palf::SCN scn) +int ObMvccRowCallback::log_sync(const SCN scn) { int ret = OB_SUCCESS; diff --git a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h index eedb8406b..6aa1820fe 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h @@ -202,8 +202,8 @@ public: void print_callbacks(); void elr_trans_preparing(); int trans_end(const bool commit); - int replay_fail(const palf::SCN scn); - int replay_succ(const palf::SCN scn); + int replay_fail(const share::SCN scn); + int replay_succ(const share::SCN scn); int rollback_to(const int64_t seq_no, const int64_t from_seq_no); void set_for_replay(const bool for_replay); @@ -220,11 +220,11 @@ public: private: void wakeup_waiting_txns_(); public: - int calc_checksum_before_scn(const palf::SCN scn, + int calc_checksum_before_scn(const share::SCN scn, uint64_t &checksum, - palf::SCN &checksum_scn); + share::SCN &checksum_scn); void update_checksum(const uint64_t checksum, - const palf::SCN checksum_scn); + const share::SCN checksum_scn); int clean_unlog_callbacks(int64_t &removed_cnt); // when not inc, return -1 int64_t inc_pending_log_size(const int64_t size); @@ -271,7 +271,7 @@ public: { ATOMIC_AAF(&callback_remove_for_rollback_to_count_, cnt); } int64_t get_checksum() const { return callback_list_.get_checksum(); } int64_t get_tmp_checksum() const { return callback_list_.get_tmp_checksum(); } - palf::SCN get_checksum_scn() const { return callback_list_.get_checksum_scn(); } + share::SCN get_checksum_scn() const { return callback_list_.get_checksum_scn(); } transaction::ObPartTransCtx *get_trans_ctx() const; private: bool is_all_redo_submitted(ObMvccRowCallback *generate_cursor) @@ -388,7 +388,7 @@ public: int get_cluster_version(uint64_t &cluster_version) const override; transaction::ObTransCtx *get_trans_ctx() const; int64_t to_string(char *buf, const int64_t buf_len) const; - bool log_synced() const override { return palf::SCN::max_scn() != scn_; } + bool log_synced() const override { return share::SCN::max_scn() != scn_; } virtual int before_append(const bool is_replay) override; virtual int after_append(const bool is_replay, const int ret_code) override; virtual int log_submitted() override; @@ -400,7 +400,7 @@ public: virtual int clean(); virtual int del(); virtual int checkpoint_callback(); - virtual int log_sync(const palf::SCN scn) override; + virtual int log_sync(const share::SCN scn) override; virtual int log_sync_fail() override; virtual int print_callback() override; virtual blocksstable::ObDmlFlag get_dml_flag() const override; @@ -414,7 +414,7 @@ private: virtual int trans_commit() override; virtual int trans_abort() override; virtual int rollback_callback() override; - virtual int calc_checksum(const palf::SCN checksum_scn, + virtual int calc_checksum(const share::SCN checksum_scn, ObBatchChecksum *checksumer) override; virtual int elr_trans_preparing() override; private: diff --git a/src/storage/memtable/mvcc/ob_query_engine.cpp b/src/storage/memtable/mvcc/ob_query_engine.cpp index f9bd61f26..ae33a9434 100644 --- a/src/storage/memtable/mvcc/ob_query_engine.cpp +++ b/src/storage/memtable/mvcc/ob_query_engine.cpp @@ -176,7 +176,7 @@ void ObQueryEngine::destroy() { if (IS_NOT_INIT) { // do nothing - } else if (OB_NOT_NULL(index_)) { + } else if (OB_NOT_NULL(index_) && NOT_PLACE_HOLDER(index_)) { index_->destroy(); memstore_allocator_.free(index_); index_ = nullptr; @@ -704,8 +704,9 @@ int ObQueryEngine::estimate_row_count(const ObMemtableKey *start_key, const int void ObQueryEngine::dump2text(FILE *fd) { - if (OB_NOT_NULL(index_)) { - index_->dump2text(fd); + TableIndex *index = ATOMIC_LOAD(&index_); + if (OB_NOT_NULL(index) && NOT_PLACE_HOLDER(index)) { + index->dump2text(fd); } } diff --git a/src/storage/memtable/mvcc/ob_query_engine.h b/src/storage/memtable/mvcc/ob_query_engine.h index 456184583..9b325a8cd 100644 --- a/src/storage/memtable/mvcc/ob_query_engine.h +++ b/src/storage/memtable/mvcc/ob_query_engine.h @@ -49,6 +49,8 @@ public: class ObQueryEngine { +#define NOT_PLACE_HOLDER(ptr) OB_UNLIKELY(PLACE_HOLDER != ptr) + public: enum { INIT_TABLE_INDEX_COUNT = (1 << 10), @@ -198,15 +200,37 @@ public: int estimate_row_count(const ObMemtableKey *start_key, const int start_exclude, const ObMemtableKey *end_key, const int end_exclude, int64_t &logical_row_count, int64_t &physical_row_count); - int dump_keyhash(FILE *fd) const { return OB_NOT_NULL(index_) ? index_->dump_keyhash(fd) : OB_SUCCESS; } - int64_t hash_size() const { return OB_NOT_NULL(index_) ? index_->hash_size() : 0; } - int64_t hash_alloc_memory() const { return OB_NOT_NULL(index_) ? index_->hash_alloc_memory() : 0; } - int dump_keybtree(FILE* fd) { return OB_NOT_NULL(index_) ? index_->dump_keybtree(fd) : OB_SUCCESS; } - int64_t btree_size() const { return OB_NOT_NULL(index_) ? index_->btree_size() : 0; } + int dump_keyhash(FILE *fd) const + { + TableIndex *index = ATOMIC_LOAD(&index_); + return OB_NOT_NULL(index) && NOT_PLACE_HOLDER(index) ? index->dump_keyhash(fd) : OB_SUCCESS; + } + int64_t hash_size() const + { + TableIndex *index = ATOMIC_LOAD(&index_); + return OB_NOT_NULL(index) && NOT_PLACE_HOLDER(index) ? index->hash_size() : 0; + } + int64_t hash_alloc_memory() const + { + TableIndex *index = ATOMIC_LOAD(&index_); + return OB_NOT_NULL(index) && NOT_PLACE_HOLDER(index) ? index->hash_alloc_memory() : 0; + } + int dump_keybtree(FILE *fd) + { + TableIndex *index = ATOMIC_LOAD(&index_); + return OB_NOT_NULL(index) && NOT_PLACE_HOLDER(index) ? index->dump_keybtree(fd) : OB_SUCCESS; + } + int64_t btree_size() const + { + TableIndex *index = ATOMIC_LOAD(&index_); + return OB_NOT_NULL(index) && NOT_PLACE_HOLDER(index) ? index->btree_size() : 0; + } int64_t btree_alloc_memory() const { - return OB_NOT_NULL(index_) ? - index_->btree_alloc_memory() + btree_allocator_.get_allocated() : 0; + TableIndex *index = ATOMIC_LOAD(&index_); + return OB_NOT_NULL(index) && NOT_PLACE_HOLDER(index) + ? index->btree_alloc_memory() + btree_allocator_.get_allocated() + : 0; } void dump2text(FILE *fd); int get_table_index(TableIndex *&return_ptr) const; diff --git a/src/storage/memtable/mvcc/ob_tx_callback_functor.h b/src/storage/memtable/mvcc/ob_tx_callback_functor.h index 366f4f842..257cd2733 100644 --- a/src/storage/memtable/mvcc/ob_tx_callback_functor.h +++ b/src/storage/memtable/mvcc/ob_tx_callback_functor.h @@ -74,8 +74,8 @@ class ObRemoveCallbacksForFastCommitFunctor : public ObITxCallbackFunctor public: ObRemoveCallbacksForFastCommitFunctor(const int64_t need_remove_count) : need_remove_count_(need_remove_count), - last_scn_for_remove_(palf::SCN::min_scn()), - checksum_scn_(palf::SCN::min_scn()), + last_scn_for_remove_(share::SCN::min_scn()), + checksum_scn_(share::SCN::min_scn()), checksumer_(NULL) {} virtual bool is_iter_end(ObITransCallback *callback) const override @@ -88,15 +88,15 @@ public: } else if (callback->need_fill_redo() || callback->need_submit_log()) { // case2: the callback has not been sync successfully is_iter_end = true; - } else if (palf::SCN::max_scn() == callback->get_scn()) { + } else if (share::SCN::max_scn() == callback->get_scn()) { // case3: the callback has not been sync successfully is_iter_end = true; - } else if (palf::SCN::min_scn() != last_scn_for_remove_ + } else if (share::SCN::min_scn() != last_scn_for_remove_ && callback->get_scn() != last_scn_for_remove_) { // case4: the callback has exceeded the last log whose log ts need to be // removed is_iter_end = true; - } else if (palf::SCN::min_scn() == last_scn_for_remove_ + } else if (share::SCN::min_scn() == last_scn_for_remove_ && 0 >= need_remove_count_) { // case6: the callback has not reached the last log whose log ts need to // be removed while having no logs need to be removed @@ -108,14 +108,14 @@ public: return is_iter_end; } - void set_checksumer(const palf::SCN checksum_scn, + void set_checksumer(const share::SCN checksum_scn, ObBatchChecksum *checksumer) { checksum_scn_ = checksum_scn; checksumer_ = checksumer; } - palf::SCN get_checksum_last_scn() const + share::SCN get_checksum_last_scn() const { return last_scn_for_remove_; } @@ -141,13 +141,13 @@ public: // callbacks until all callbacks with the same log ts has been removed in // order to satisfy the checksum calculation ObITransCallback *next = callback->get_next(); - if (palf::SCN::min_scn() == last_scn_for_remove_) { + if (share::SCN::min_scn() == last_scn_for_remove_) { if (0 == need_remove_count_) { last_scn_for_remove_ = callback->get_scn(); } else if (NULL == next || next->need_submit_log() || next->need_fill_redo() - || palf::SCN::max_scn() == next->get_scn()) { + || share::SCN::max_scn() == next->get_scn()) { last_scn_for_remove_ = callback->get_scn(); } } @@ -162,8 +162,8 @@ public: private: int64_t need_remove_count_; - palf::SCN last_scn_for_remove_; - palf::SCN checksum_scn_; + share::SCN last_scn_for_remove_; + share::SCN checksum_scn_; ObBatchChecksum *checksumer_; }; @@ -190,9 +190,9 @@ public: cond_for_stop_(stop_func), need_checksum_(false), need_remove_data_(need_remove_data), - checksum_scn_(palf::SCN::min_scn()), + checksum_scn_(share::SCN::min_scn()), checksumer_(NULL), - checksum_last_scn_(palf::SCN::min_scn()) + checksum_last_scn_(share::SCN::min_scn()) { is_reverse_ = is_reverse; } @@ -209,7 +209,7 @@ public: return is_valid; } - void set_checksumer(const palf::SCN checksum_scn, + void set_checksumer(const share::SCN checksum_scn, ObBatchChecksum *checksumer) { need_checksum_ = true; @@ -217,13 +217,13 @@ public: checksumer_ = checksumer; } - palf::SCN get_checksum_last_scn() const + share::SCN get_checksum_last_scn() const { if (need_checksum_) { return checksum_last_scn_; } else { TRANS_LOG(ERROR, "we donot go here if we donot checksum", KPC(this)); - return palf::SCN::min_scn(); + return share::SCN::min_scn(); } } @@ -237,7 +237,7 @@ public: } else if (callback->need_fill_redo() || callback->need_submit_log()) { // case2: the callback has not been sync successfully is_iter_end = true; - } else if (palf::SCN::max_scn() == callback->get_scn()) { + } else if (share::SCN::max_scn() == callback->get_scn()) { // case3: the callback has not been sync successfully is_iter_end = true; } else if (cond_for_stop_(callback)) { @@ -288,9 +288,9 @@ private: ObFunction cond_for_stop_; bool need_checksum_; bool need_remove_data_; - palf::SCN checksum_scn_; + share::SCN checksum_scn_; ObBatchChecksum *checksumer_; - palf::SCN checksum_last_scn_; + share::SCN checksum_last_scn_; }; class ObRemoveCallbacksWCondFunctor : public ObITxCallbackFunctor @@ -300,24 +300,24 @@ public: const bool need_remove_data = true) : cond_for_remove_(func), need_remove_data_(need_remove_data), - checksum_scn_(palf::SCN::min_scn()), + checksum_scn_(share::SCN::min_scn()), checksumer_(NULL), - checksum_last_scn_(palf::SCN::min_scn()) {} + checksum_last_scn_(share::SCN::min_scn()) {} - void set_checksumer(const palf::SCN checksum_scn, + void set_checksumer(const share::SCN checksum_scn, ObBatchChecksum *checksumer) { checksum_scn_ = checksum_scn; checksumer_ = checksumer; } - palf::SCN get_checksum_last_scn() const + share::SCN get_checksum_last_scn() const { if (NULL != checksumer_) { return checksum_last_scn_; } else { TRANS_LOG(ERROR, "we donot go here if we donot checksum", KPC(this)); - return palf::SCN::min_scn(); + return share::SCN::min_scn(); } } @@ -387,9 +387,9 @@ public: private: ObFunction cond_for_remove_; bool need_remove_data_; - palf::SCN checksum_scn_; + share::SCN checksum_scn_; ObBatchChecksum *checksumer_; - palf::SCN checksum_last_scn_; + share::SCN checksum_last_scn_; }; class ObTxForAllFunctor : public ObITxCallbackFunctor @@ -545,11 +545,11 @@ private: class ObCalcChecksumFunctor : public ObITxCallbackFunctor { public: - ObCalcChecksumFunctor(const palf::SCN target_scn = palf::SCN::max_scn()) + ObCalcChecksumFunctor(const share::SCN target_scn = share::SCN::max_scn()) : target_scn_(target_scn), - checksum_scn_(palf::SCN::min_scn()), + checksum_scn_(share::SCN::min_scn()), checksumer_(NULL), - checksum_last_scn_(palf::SCN::min_scn()) {} + checksum_last_scn_(share::SCN::min_scn()) {} virtual bool is_iter_end(ObITransCallback *callback) const override { @@ -569,14 +569,14 @@ public: } - void set_checksumer(const palf::SCN checksum_scn, + void set_checksumer(const share::SCN checksum_scn, ObBatchChecksum *checksumer) { checksum_scn_ = checksum_scn; checksumer_ = checksumer; } - palf::SCN get_checksum_last_scn() const + share::SCN get_checksum_last_scn() const { return checksum_last_scn_; } @@ -605,10 +605,10 @@ public: K_(checksum_scn), K_(checksum_last_scn)); private: - palf::SCN target_scn_; - palf::SCN checksum_scn_; + share::SCN target_scn_; + share::SCN checksum_scn_; ObBatchChecksum *checksumer_; - palf::SCN checksum_last_scn_; + share::SCN checksum_last_scn_; }; } // memtable diff --git a/src/storage/memtable/mvcc/ob_tx_callback_list.cpp b/src/storage/memtable/mvcc/ob_tx_callback_list.cpp index 4bf2aa64f..bfb4fbe2c 100644 --- a/src/storage/memtable/mvcc/ob_tx_callback_list.cpp +++ b/src/storage/memtable/mvcc/ob_tx_callback_list.cpp @@ -19,6 +19,7 @@ namespace oceanbase { +using namespace share; namespace memtable { @@ -26,7 +27,7 @@ ObTxCallbackList::ObTxCallbackList(ObTransCallbackMgr &callback_mgr) : head_(), length_(0), batch_checksum_(), - checksum_scn_(palf::SCN::min_scn()), + checksum_scn_(SCN::min_scn()), checksum_(0), tmp_checksum_(0), callback_mgr_(callback_mgr), @@ -41,7 +42,7 @@ void ObTxCallbackList::reset() head_.set_next(&head_); checksum_ = 0; tmp_checksum_ = 0; - checksum_scn_ = palf::SCN::min_scn(); + checksum_scn_ = SCN::min_scn(); batch_checksum_.reset(); length_ = 0; } @@ -156,7 +157,7 @@ int ObTxCallbackList::remove_callbacks_for_fast_commit(bool &has_remove) } else { callback_mgr_.add_fast_commit_callback_remove_cnt(functor.get_remove_cnt()); ensure_checksum_(functor.get_checksum_last_scn()); - has_remove = palf::SCN::min_scn() != functor.get_checksum_last_scn(); + has_remove = SCN::min_scn() != functor.get_checksum_last_scn(); } return ret; @@ -282,7 +283,7 @@ int ObTxCallbackList::get_memtable_key_arr_w_timeout(transaction::ObMemtableKeyA return ret; } -int ObTxCallbackList::tx_calc_checksum_before_scn(const palf::SCN scn) +int ObTxCallbackList::tx_calc_checksum_before_scn(const SCN scn) { int ret = OB_SUCCESS; SpinLockGuard guard(latch_); @@ -312,7 +313,7 @@ int ObTxCallbackList::tx_calc_checksum_all() if (OB_FAIL(callback_(functor))) { TRANS_LOG(ERROR, "calc checksum wont report error", K(ret), K(functor)); } else { - ensure_checksum_(palf::SCN::max_scn()); + ensure_checksum_(SCN::max_scn()); } return ret; @@ -384,7 +385,7 @@ int ObTxCallbackList::tx_print_callback() return ret; } -int ObTxCallbackList::replay_fail(const palf::SCN scn) +int ObTxCallbackList::replay_fail(const SCN scn) { int ret = OB_SUCCESS; ObRemoveSyncCallbacksWCondFunctor functor( @@ -418,7 +419,7 @@ int ObTxCallbackList::replay_fail(const palf::SCN scn) return OB_SUCCESS; } -void ObTxCallbackList::get_checksum_and_scn(uint64_t &checksum, palf::SCN &checksum_scn) +void ObTxCallbackList::get_checksum_and_scn(uint64_t &checksum, SCN &checksum_scn) { SpinLockGuard guard(latch_); checksum = batch_checksum_.calc(); @@ -426,7 +427,7 @@ void ObTxCallbackList::get_checksum_and_scn(uint64_t &checksum, palf::SCN &check TRANS_LOG(INFO, "get checksum and checksum_scn", KPC(this), K(checksum), K(checksum_scn)); } -void ObTxCallbackList::update_checksum(const uint64_t checksum, const palf::SCN checksum_scn) +void ObTxCallbackList::update_checksum(const uint64_t checksum, const SCN checksum_scn) { SpinLockGuard guard(latch_); batch_checksum_.set_base(checksum); @@ -434,17 +435,17 @@ void ObTxCallbackList::update_checksum(const uint64_t checksum, const palf::SCN TRANS_LOG(INFO, "update checksum and checksum_scn", KPC(this), K(checksum), K(checksum_scn)); } -void ObTxCallbackList::ensure_checksum_(const palf::SCN scn) +void ObTxCallbackList::ensure_checksum_(const SCN scn) { - if (palf::SCN::min_scn() == scn) { + if (SCN::min_scn() == scn) { // Case1: no callback is invovled } else if (scn < checksum_scn_) { // Case2: no checksum is calculated - } else if (palf::SCN::max_scn() == scn) { - checksum_scn_ = palf::SCN::max_scn(); + } else if (SCN::max_scn() == scn) { + checksum_scn_ = SCN::max_scn(); ATOMIC_STORE(&checksum_, batch_checksum_.calc() ? : 1); } else { - checksum_scn_ = palf::SCN::plus(scn, 1); + checksum_scn_ = SCN::plus(scn, 1); ATOMIC_STORE(&tmp_checksum_, batch_checksum_.calc() ? : 1); } } diff --git a/src/storage/memtable/mvcc/ob_tx_callback_list.h b/src/storage/memtable/mvcc/ob_tx_callback_list.h index a4a411531..b7ecdb8a0 100644 --- a/src/storage/memtable/mvcc/ob_tx_callback_list.h +++ b/src/storage/memtable/mvcc/ob_tx_callback_list.h @@ -83,7 +83,7 @@ public: // tx_calc_checksum_before_scn will calculate checksum during execution. It will // remember the intermediate results for final result. - int tx_calc_checksum_before_scn(const palf::SCN scn); + int tx_calc_checksum_before_scn(const share::SCN scn); // tx_calc_checksum_all will calculate checksum when tx end. Finally it will set // checksum_scn to INT64_MAX and never allow more checksum calculation. @@ -108,7 +108,7 @@ public: // replay_fail will rollback all redo in a single log according to // scn - int replay_fail(const palf::SCN scn); + int replay_fail(const share::SCN scn); private: int callback_(ObITxCallbackFunctor &func); @@ -116,7 +116,7 @@ private: ObITransCallback *start, ObITransCallback *end); int64_t calc_need_remove_count_for_fast_commit_(); - void ensure_checksum_(const palf::SCN scn); + void ensure_checksum_(const share::SCN scn); public: ObITransCallback *get_guard() { return &head_; } ObITransCallback *get_tail() { return head_.get_prev(); } @@ -124,9 +124,9 @@ public: int64_t get_length() const { return length_; } int64_t get_checksum() const { return checksum_; } int64_t get_tmp_checksum() const { return tmp_checksum_; } - palf::SCN get_checksum_scn() const { return checksum_scn_; } - void get_checksum_and_scn(uint64_t &checksum, palf::SCN &checksum_scn); - void update_checksum(const uint64_t checksum, const palf::SCN checksum_scn); + share::SCN get_checksum_scn() const { return checksum_scn_; } + void get_checksum_and_scn(uint64_t &checksum, share::SCN &checksum_scn); + void update_checksum(const uint64_t checksum, const share::SCN checksum_scn); transaction::ObPartTransCtx *get_trans_ctx() const; DECLARE_TO_STRING; @@ -153,7 +153,7 @@ private: * checksum_scn_ by 1 to avoid duplicate calculation. */ common::ObBatchChecksum batch_checksum_; - palf::SCN checksum_scn_; + share::SCN checksum_scn_; uint64_t checksum_; uint64_t tmp_checksum_; diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index 51cdd0877..ff34823ca 100644 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -45,6 +45,7 @@ namespace oceanbase { using namespace common; +using namespace share; using namespace compaction; using namespace share::schema; @@ -107,9 +108,9 @@ ObMemtable::ObMemtable() logging_blocked_start_time(0), unset_active_memtable_logging_blocked_(false), resolve_active_memtable_left_boundary_(true), - freeze_scn_(palf::SCN::max_scn()), + freeze_scn_(SCN::max_scn()), max_end_scn_(ObScnRange::MIN_SCN), - rec_scn_(palf::SCN::max_scn()), + rec_scn_(SCN::max_scn()), state_(ObMemtableState::INVALID), freeze_state_(ObMemtableFreezeState::INVALID), timestamp_(0), @@ -250,7 +251,7 @@ void ObMemtable::destroy() unset_active_memtable_logging_blocked_ = false; resolve_active_memtable_left_boundary_ = true; max_end_scn_ = ObScnRange::MIN_SCN; - rec_scn_ = palf::SCN::max_scn(); + rec_scn_ = SCN::max_scn(); read_barrier_ = false; is_tablet_freeze_ = false; is_force_freeze_ = false; @@ -1070,7 +1071,7 @@ int ObMemtable::replay_row(ObStoreCtx &ctx, blocksstable::ObDmlFlag dml_flag = blocksstable::ObDmlFlag::DF_NOT_EXIST; ObMemtableCtx *mt_ctx = ctx.mvcc_acc_ctx_.mem_ctx_; ObPartTransCtx *part_ctx = static_cast(mt_ctx->get_trans_ctx()); - const palf::SCN scn = mt_ctx->get_redo_scn(); + const SCN scn = mt_ctx->get_redo_scn(); const int64_t log_id = mt_ctx->get_redo_log_id(); if (OB_FAIL(mmi->get_mutator_row().copy(table_id, rowkey, table_version, row, @@ -1134,7 +1135,7 @@ int ObMemtable::lock_row_on_frozen_stores_(ObStoreCtx &ctx, } else if (value->is_lower_lock_scaned()) { } else { bool row_locked = false; - palf::SCN max_trans_version = palf::SCN::min_scn(); + SCN max_trans_version = SCN::min_scn(); const ObIArray *stores = nullptr; common::ObSEArray iter_tables; ctx.table_iter_->resume(); @@ -1273,7 +1274,7 @@ ObDatumRange &ObMemtable::m_get_real_range(ObDatumRange &real_range, const ObDat int ObMemtable::row_compact(ObMvccRow *row, const bool for_replay, - const palf::SCN snapshot_version) + const SCN snapshot_version) { int ret = OB_SUCCESS; ObMemtableRowCompactor row_compactor; @@ -1439,7 +1440,7 @@ int ObMemtable::get_frozen_schema_version(int64_t &schema_version) const return OB_NOT_SUPPORTED; } -int ObMemtable::set_snapshot_version(const palf::SCN snapshot_version) +int ObMemtable::set_snapshot_version(const SCN snapshot_version) { int ret = OB_SUCCESS; @@ -1456,7 +1457,7 @@ int ObMemtable::set_snapshot_version(const palf::SCN snapshot_version) return ret; } -int ObMemtable::set_rec_scn(palf::SCN rec_scn) +int ObMemtable::set_rec_scn(SCN rec_scn) { int ret = OB_SUCCESS; share::ObLSID ls_id = freezer_->get_ls_id(); @@ -1471,8 +1472,8 @@ int ObMemtable::set_rec_scn(palf::SCN rec_scn) ret = OB_SCN_OUT_OF_BOUND; TRANS_LOG(ERROR, "cannot set freeze log ts smaller to start log ts", K(ret), K(rec_scn), K(ls_id), KPC(this)); } else { - palf::SCN old_rec_scn; - palf::SCN new_rec_scn = get_rec_scn(); + SCN old_rec_scn; + SCN new_rec_scn = get_rec_scn(); while ((old_rec_scn = new_rec_scn) > rec_scn) { if ((new_rec_scn = rec_scn_.atomic_vcas(old_rec_scn, rec_scn)) == old_rec_scn) { @@ -1484,7 +1485,7 @@ int ObMemtable::set_rec_scn(palf::SCN rec_scn) return ret; } -int ObMemtable::set_start_scn(const palf::SCN start_scn) +int ObMemtable::set_start_scn(const SCN start_scn) { int ret = OB_SUCCESS; share::ObLSID ls_id = freezer_->get_ls_id(); @@ -1496,7 +1497,7 @@ int ObMemtable::set_start_scn(const palf::SCN start_scn) ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid args", K(ret), K(start_scn)); } else if (start_scn >= get_end_scn() - || (max_end_scn_ != palf::SCN::min_scn() && start_scn >= max_end_scn_) + || (max_end_scn_ != SCN::min_scn() && start_scn >= max_end_scn_) || start_scn >= rec_scn_) { ret = OB_SCN_OUT_OF_BOUND; TRANS_LOG(ERROR, "cannot set start ts now", K(ret), K(start_scn), K(ls_id), KPC(this)); @@ -1507,7 +1508,7 @@ int ObMemtable::set_start_scn(const palf::SCN start_scn) return ret; } -int ObMemtable::set_end_scn(const palf::SCN freeze_scn) +int ObMemtable::set_end_scn(const SCN freeze_scn) { int ret = OB_SUCCESS; share::ObLSID ls_id = freezer_->get_ls_id(); @@ -1523,8 +1524,8 @@ int ObMemtable::set_end_scn(const palf::SCN freeze_scn) TRANS_LOG(ERROR, "cannot set freeze log ts smaller to start log ts", K(ret), K(freeze_scn), K(ls_id), KPC(this)); } else { - palf::SCN old_end_scn; - palf::SCN new_end_scn = get_end_scn(); + SCN old_end_scn; + SCN new_end_scn = get_end_scn(); while ((old_end_scn = new_end_scn) < freeze_scn || new_end_scn == ObScnRange::MAX_SCN) { if ((new_end_scn = @@ -1539,7 +1540,7 @@ int ObMemtable::set_end_scn(const palf::SCN freeze_scn) return ret; } -int ObMemtable::set_max_end_scn(const palf::SCN scn) +int ObMemtable::set_max_end_scn(const SCN scn) { int ret = OB_SUCCESS; share::ObLSID ls_id = freezer_->get_ls_id(); @@ -1555,8 +1556,8 @@ int ObMemtable::set_max_end_scn(const palf::SCN scn) TRANS_LOG(WARN, "cannot set max end log ts smaller to start log ts", K(ret), K(scn), K(ls_id), KPC(this)); } else { - palf::SCN old_max_end_scn; - palf::SCN new_max_end_scn = get_max_end_scn(); + SCN old_max_end_scn; + SCN new_max_end_scn = get_max_end_scn(); while ((old_max_end_scn = new_max_end_scn) < scn) { if ((new_max_end_scn = max_end_scn_.atomic_vcas(old_max_end_scn, scn)) @@ -1573,10 +1574,10 @@ bool ObMemtable::rec_scn_is_stable() { int ret = OB_SUCCESS; bool rec_scn_is_stable = false; - if (palf::SCN::max_scn() == rec_scn_) { + if (SCN::max_scn() == rec_scn_) { rec_scn_is_stable = (is_frozen_memtable() && write_ref_cnt_ == 0 && unsynced_cnt_ == 0); } else { - palf::SCN max_consequent_callbacked_scn; + SCN max_consequent_callbacked_scn; if (OB_FAIL(freezer_->get_max_consequent_callbacked_scn(max_consequent_callbacked_scn))) { STORAGE_LOG(WARN, "get_max_consequent_callbacked_scn failed", K(ret), K(freezer_->get_ls_id())); } else { @@ -1599,7 +1600,7 @@ bool ObMemtable::rec_scn_is_stable() return rec_scn_is_stable; } -int ObMemtable::get_current_right_boundary(palf::SCN ¤t_right_boundary) +int ObMemtable::get_current_right_boundary(SCN ¤t_right_boundary) { int ret = OB_SUCCESS; if (OB_ISNULL(freezer_)) { @@ -1629,7 +1630,7 @@ bool ObMemtable::ready_for_flush_() bool bool_ret = is_frozen_memtable() && 0 == get_write_ref() && 0 == get_unsynced_cnt(); int ret = OB_SUCCESS; - palf::SCN current_right_boundary = ObScnRange::MIN_SCN; + SCN current_right_boundary = ObScnRange::MIN_SCN; share::ObLSID ls_id = freezer_->get_ls_id(); if (bool_ret) { if (OB_FAIL(resolve_snapshot_version_())) { @@ -1693,8 +1694,8 @@ void ObMemtable::print_ready_for_flush() bool frozen_memtable_flag = is_frozen_memtable(); int64_t write_ref = get_write_ref(); int64_t unsynced_cnt = get_unsynced_cnt(); - palf::SCN end_scn = get_end_scn(); - palf::SCN current_right_boundary; + SCN end_scn = get_end_scn(); + SCN current_right_boundary; uint32_t logstream_freeze_clock = freezer_->get_freeze_clock(); uint32_t memtable_freeze_clock = freeze_clock_; if (OB_FAIL(get_current_right_boundary(current_right_boundary))) { @@ -1747,16 +1748,16 @@ void ObMemtable::print_ready_for_flush() int ObMemtable::resolve_snapshot_version_() { int ret = OB_SUCCESS; - palf::SCN freeze_snapshot_version; + SCN freeze_snapshot_version; - if (snapshot_version_ != palf::SCN::max_scn()) { + if (snapshot_version_ != SCN::max_scn()) { // Pass if snapshot is already set } else if (OB_ISNULL(freezer_)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "freezer should not be null", K(ret)); } else if (FALSE_IT(freeze_snapshot_version = freezer_->get_freeze_snapshot_version())) { TRANS_LOG(ERROR, "fail to get freeze_snapshot_version", K(ret)); - } else if (palf::SCN::invalid_scn() == freeze_snapshot_version) { + } else if (SCN::invalid_scn() == freeze_snapshot_version) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "fail to get freeze_snapshot_version", K(ret), KPC(this)); } else if (OB_FAIL(set_snapshot_version(freeze_snapshot_version))) { @@ -1780,14 +1781,14 @@ int ObMemtable::resolve_max_end_scn_() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - palf::SCN max_decided_scn; + SCN max_decided_scn; if (OB_ISNULL(freezer_)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "freezer should not be null", K(ret)); } else if (FALSE_IT(max_decided_scn = freezer_->get_max_decided_scn())) { TRANS_LOG(ERROR, "fail to get freeze_snapshot_version", K(ret)); - } else if (palf::SCN::invalid_scn() == max_decided_scn) { + } else if (SCN::invalid_scn() == max_decided_scn) { // Pass if not necessary } else if (OB_TMP_FAIL(set_max_end_scn(max_decided_scn))) { // ignore the error code @@ -1798,9 +1799,9 @@ int ObMemtable::resolve_max_end_scn_() int ObMemtable::resolve_right_boundary() { - palf::SCN max_end_scn = get_max_end_scn(); - palf::SCN end_scn = max_end_scn; - palf::SCN start_scn = get_start_scn(); + SCN max_end_scn = get_max_end_scn(); + SCN end_scn = max_end_scn; + SCN start_scn = get_start_scn(); int ret = OB_SUCCESS; if (ObScnRange::MIN_SCN == max_end_scn) { @@ -1814,7 +1815,7 @@ int ObMemtable::resolve_right_boundary() return ret; } -void ObMemtable::resolve_left_boundary(palf::SCN end_scn) +void ObMemtable::resolve_left_boundary(SCN end_scn) { set_start_scn(end_scn); } @@ -2380,7 +2381,7 @@ int ObMemtable::mvcc_write_(storage::ObStoreCtx &ctx, ObMvccRow *value = NULL; ObMvccWriteResult res; ObIMemtableCtx *mem_ctx = ctx.mvcc_acc_ctx_.get_mem_ctx(); - palf::SCN snapshot_version = ctx.mvcc_acc_ctx_.get_snapshot_version(); + SCN snapshot_version = ctx.mvcc_acc_ctx_.get_snapshot_version(); if (OB_FAIL(mvcc_engine_.create_kv(key, &stored_key, diff --git a/src/storage/memtable/ob_memtable.h b/src/storage/memtable/ob_memtable.h index a9bbb5d4d..fc0009f91 100644 --- a/src/storage/memtable/ob_memtable.h +++ b/src/storage/memtable/ob_memtable.h @@ -319,7 +319,7 @@ public: inline bool not_empty() const { return INT64_MAX != get_protection_clock(); }; void set_max_schema_version(const int64_t schema_version); virtual int64_t get_max_schema_version() const override; - int row_compact(ObMvccRow *value, const bool for_replay, const palf::SCN snapshot_version); + int row_compact(ObMvccRow *value, const bool for_replay, const share::SCN snapshot_version); int64_t get_hash_item_count() const; int64_t get_hash_alloc_memory() const; int64_t get_btree_item_count() const; @@ -330,12 +330,12 @@ public: virtual bool is_active_memtable() const override; virtual bool is_inner_tablet() const { return key_.tablet_id_.is_inner_tablet(); } ObTabletID get_tablet_id() const { return key_.tablet_id_; } - int set_snapshot_version(const palf::SCN snapshot_version); + int set_snapshot_version(const share::SCN snapshot_version); int64_t get_memtable_state() const { return state_; } int64_t get_freeze_state() const { return freeze_state_; } int64_t get_protection_clock() const { return local_allocator_.get_protection_clock(); } int64_t get_retire_clock() const { return local_allocator_.get_retire_clock(); } - int get_current_right_boundary(palf::SCN ¤t_right_boundary); + int get_current_right_boundary(share::SCN ¤t_right_boundary); inline bool& get_read_barrier() { return read_barrier_; } inline void set_write_barrier() { write_barrier_ = true; } @@ -374,15 +374,15 @@ public: virtual bool ready_for_flush() override; void print_ready_for_flush(); virtual int flush(share::ObLSID ls_id) override; - palf::SCN get_rec_scn() { return rec_scn_.atomic_get(); } + share::SCN get_rec_scn() { return rec_scn_.atomic_get(); } virtual bool is_frozen_checkpoint() const override { return is_frozen_memtable();} virtual bool is_active_checkpoint() const override { return is_active_memtable();} - virtual OB_INLINE palf::SCN get_end_scn() const + virtual OB_INLINE share::SCN get_end_scn() const { return key_.scn_range_.end_scn_; } - virtual OB_INLINE palf::SCN get_start_scn() const + virtual OB_INLINE share::SCN get_start_scn() const { return key_.scn_range_.start_scn_; } @@ -392,14 +392,14 @@ public: share::ObScnRange::MIN_SCN == get_max_end_scn(); } int resolve_right_boundary(); - void resolve_left_boundary(palf::SCN end_scn); + void resolve_left_boundary(share::SCN end_scn); int resolve_snapshot_version_(); int resolve_max_end_scn_(); - palf::SCN get_max_end_scn() const { return max_end_scn_.atomic_get(); } - int set_rec_scn(palf::SCN rec_scn); - int set_start_scn(const palf::SCN start_ts); - int set_end_scn(const palf::SCN freeze_ts); - int set_max_end_scn(const palf::SCN scn); + share::SCN get_max_end_scn() const { return max_end_scn_.atomic_get(); } + int set_rec_scn(share::SCN rec_scn); + int set_start_scn(const share::SCN start_ts); + int set_end_scn(const share::SCN freeze_ts); + int set_max_end_scn(const share::SCN scn); inline int set_logging_blocked() { logging_blocked_start_time = common::ObTimeUtility::current_time(); @@ -438,7 +438,7 @@ public: template int save_multi_source_data_unit(const T *const multi_source_data_unit, - const palf::SCN scn, + const share::SCN scn, const bool for_replay, const MemtableRefOp ref_op = MemtableRefOp::NONE, const bool is_callback = false); @@ -525,9 +525,9 @@ private: int64_t logging_blocked_start_time; // record the start time of logging blocked bool unset_active_memtable_logging_blocked_; bool resolve_active_memtable_left_boundary_; - palf::SCN freeze_scn_; - palf::SCN max_end_scn_; - palf::SCN rec_scn_; + share::SCN freeze_scn_; + share::SCN max_end_scn_; + share::SCN rec_scn_; int64_t state_; int64_t freeze_state_; int64_t timestamp_; @@ -546,7 +546,7 @@ private: template int ObMemtable::save_multi_source_data_unit(const T *const multi_source_data_unit, - const palf::SCN scn, + const share::SCN scn, const bool for_replay, const MemtableRefOp ref_op, const bool is_callback) @@ -590,7 +590,7 @@ int ObMemtable::save_multi_source_data_unit(const T *const multi_source_data_uni } else if (OB_FAIL(set_rec_scn(scn))) { TRANS_LOG(WARN, "failed to set rec_scn", K(ret), K(scn), KPC(this)); } - } else if (palf::SCN::invalid_scn() != scn && share::ObScnRange::MAX_SCN != scn) { + } else if (share::SCN::invalid_scn() != scn && share::ObScnRange::MAX_SCN != scn) { ret = common::OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "invalid scn", K(ret), K(scn), KPC(this)); } diff --git a/src/storage/memtable/ob_memtable_context.cpp b/src/storage/memtable/ob_memtable_context.cpp index 245ff992f..ab16fc35a 100644 --- a/src/storage/memtable/ob_memtable_context.cpp +++ b/src/storage/memtable/ob_memtable_context.cpp @@ -231,7 +231,7 @@ void ObMemtableCtx::set_replay() is_master_ = false; } -palf::SCN ObMemtableCtx::get_tx_end_scn() const +SCN ObMemtableCtx::get_tx_end_scn() const { return ctx_->get_tx_end_log_ts(); } @@ -314,8 +314,8 @@ void ObMemtableCtx::on_wlock_retry(const ObMemtableKey& key, const transaction:: } void ObMemtableCtx::on_tsc_retry(const ObMemtableKey& key, - const palf::SCN snapshot_version, - const palf::SCN max_trans_version, + const SCN snapshot_version, + const SCN max_trans_version, const transaction::ObTransID &conflict_tx_id) { mtstat_.on_tsc_retry(); @@ -417,7 +417,7 @@ int ObMemtableCtx::trans_begin() return ret; } -int ObMemtableCtx::replay_begin(const palf::SCN scn) +int ObMemtableCtx::replay_begin(const SCN scn) { ObByteLockGuard guard(lock_); @@ -427,7 +427,7 @@ int ObMemtableCtx::replay_begin(const palf::SCN scn) } int ObMemtableCtx::replay_end(const bool is_replay_succ, - const palf::SCN scn) + const SCN scn) { int ret = OB_SUCCESS; ObByteLockGuard guard(lock_); @@ -441,7 +441,7 @@ int ObMemtableCtx::replay_end(const bool is_replay_succ, return ret; } -int ObMemtableCtx::rollback_redo_callbacks(const palf::SCN scn) +int ObMemtableCtx::rollback_redo_callbacks(const SCN scn) { int ret = OB_SUCCESS; ObByteLockGuard guard(lock_); @@ -453,11 +453,16 @@ int ObMemtableCtx::rollback_redo_callbacks(const palf::SCN scn) int ObMemtableCtx::trans_end( const bool commit, - const palf::SCN trans_version, - const palf::SCN final_scn) + const SCN trans_version, + const SCN final_scn) { int ret = OB_SUCCESS; + if (commit && get_trans_version().is_max()) { + TRANS_LOG(ERROR, "unexpected prepare version", K(*this)); + // no retcode + } + ret = do_trans_end(commit, trans_version, final_scn, @@ -483,8 +488,8 @@ int ObMemtableCtx::elr_trans_preparing() int ObMemtableCtx::do_trans_end( const bool commit, - const palf::SCN trans_version, - const palf::SCN final_scn, + const SCN trans_version, + const SCN final_scn, const int end_code) { int ret = OB_SUCCESS; @@ -528,7 +533,7 @@ int ObMemtableCtx::trans_kill() { int ret = OB_SUCCESS; bool commit = false; - ret = do_trans_end(commit, palf::SCN::max_scn(), palf::SCN::max_scn(), OB_TRANS_KILLED); + ret = do_trans_end(commit, SCN::max_scn(), SCN::max_scn(), OB_TRANS_KILLED); return ret; } @@ -551,8 +556,8 @@ int ObMemtableCtx::trans_replay_begin() } int ObMemtableCtx::trans_replay_end(const bool commit, - const palf::SCN trans_version, - const palf::SCN final_scn, + const SCN trans_version, + const SCN final_scn, const uint64_t log_cluster_version, const uint64_t checksum) { @@ -662,7 +667,7 @@ int ObMemtableCtx::log_submitted(const ObRedoLogSubmitHelper &helper) return log_gen_.log_submitted(helper.callbacks_); } -int ObMemtableCtx::sync_log_succ(const palf::SCN scn, const ObCallbackScope &callbacks) +int ObMemtableCtx::sync_log_succ(const SCN scn, const ObCallbackScope &callbacks) { int ret = OB_SUCCESS; @@ -671,7 +676,10 @@ int ObMemtableCtx::sync_log_succ(const palf::SCN scn, const ObCallbackScope &cal TRANS_LOG(WARN, "sync log failed", K(ret)); } } else { - TRANS_LOG(INFO, "No memtable callbacks because of trans_end", K(end_code_), KPC(ctx_)); + if (!callbacks.is_empty()) { + TRANS_LOG(INFO, "No memtable callbacks because of trans_end", K(end_code_), K(scn), + KPC(ctx_)); + } } return ret; @@ -685,7 +693,9 @@ void ObMemtableCtx::sync_log_fail(const ObCallbackScope &callbacks) if (is_partial_rollbacked_() || OB_SUCCESS == ATOMIC_LOAD(&end_code_)) { log_gen_.sync_log_fail(callbacks); } else { - TRANS_LOG(INFO, "No memtable callbacks because of trans_end", K(end_code_), KPC(ctx_)); + if (!callbacks.is_empty()) { + TRANS_LOG(INFO, "No memtable callbacks because of trans_end", K(end_code_), KPC(ctx_)); + } } return; } @@ -997,9 +1007,9 @@ int ObMemtableCtx::reuse_log_generator_() return ret; } -int ObMemtableCtx::calc_checksum_before_scn(const palf::SCN scn, +int ObMemtableCtx::calc_checksum_before_scn(const SCN scn, uint64_t &checksum, - palf::SCN &checksum_scn) + SCN &checksum_scn) { int ret = OB_SUCCESS; ObByteLockGuard guard(lock_); @@ -1012,7 +1022,7 @@ int ObMemtableCtx::calc_checksum_before_scn(const palf::SCN scn, } void ObMemtableCtx::update_checksum(const uint64_t checksum, - const palf::SCN checksum_scn) + const SCN checksum_scn) { ObByteLockGuard guard(lock_); @@ -1081,7 +1091,7 @@ int ObMemtableCtx::recover_from_table_lock_durable_info(const ObTableLockInfo &t return ret; } -int ObMemtableCtx::check_lock_need_replay(const palf::SCN &scn, +int ObMemtableCtx::check_lock_need_replay(const SCN &scn, const tablelock::ObTableLockOp &lock_op, bool &need_replay) { @@ -1189,7 +1199,7 @@ int ObMemtableCtx::add_lock_record(const tablelock::ObTableLockOp &lock_op) int ObMemtableCtx::replay_add_lock_record( const tablelock::ObTableLockOp &lock_op, - const palf::SCN &scn) + const SCN &scn) { int ret = OB_SUCCESS; const bool is_replay = true; @@ -1228,14 +1238,14 @@ void ObMemtableCtx::remove_lock_record(ObMemCtxLockOpLinkNode *lock_op) lock_mem_ctx_.remove_lock_record(lock_op); } -void ObMemtableCtx::set_log_synced(ObMemCtxLockOpLinkNode *lock_op, const palf::SCN &scn) +void ObMemtableCtx::set_log_synced(ObMemCtxLockOpLinkNode *lock_op, const SCN &scn) { lock_mem_ctx_.set_log_synced(lock_op, scn); } int ObMemtableCtx::clear_table_lock_(const bool is_commit, - const palf::SCN &commit_version, - const palf::SCN &commit_scn) + const SCN &commit_version, + const SCN &commit_scn) { int ret = OB_SUCCESS; ObLockMemtable *memtable = nullptr; @@ -1300,7 +1310,7 @@ int ObMemtableCtx::register_multi_source_data_if_need_( } int ObMemtableCtx::replay_lock(const tablelock::ObTableLockOp &lock_op, - const palf::SCN &scn) + const SCN &scn) { int ret = OB_SUCCESS; ObLockMemtable *memtable = nullptr; diff --git a/src/storage/memtable/ob_memtable_context.h b/src/storage/memtable/ob_memtable_context.h index f1e9be3d6..ac74febc0 100644 --- a/src/storage/memtable/ob_memtable_context.h +++ b/src/storage/memtable/ob_memtable_context.h @@ -346,23 +346,23 @@ public: virtual int write_auth(const bool exclusive); virtual int write_done(); virtual int trans_begin(); - virtual int replay_begin(const palf::SCN scn); + virtual int replay_begin(const share::SCN scn); virtual int replay_end(const bool is_replay_succ, - const palf::SCN scn); - int rollback_redo_callbacks(const palf::SCN scn); + const share::SCN scn); + int rollback_redo_callbacks(const share::SCN scn); virtual uint64_t calc_checksum_all(); virtual void print_callbacks(); virtual int trans_end(const bool commit, - const palf::SCN trans_version, - const palf::SCN final_scn); + const share::SCN trans_version, + const share::SCN final_scn); virtual int trans_clear(); virtual int elr_trans_preparing(); virtual int trans_kill(); virtual int trans_publish(); virtual int trans_replay_begin(); virtual int trans_replay_end(const bool commit, - const palf::SCN trans_version, - const palf::SCN final_scn, + const share::SCN trans_version, + const share::SCN final_scn, const uint64_t log_cluster_version = 0, const uint64_t checksum = 0); //method called when leader takeover @@ -389,17 +389,17 @@ public: int64_t &buf_pos, ObRedoLogSubmitHelper &helper, const bool log_for_lock_node = true); - int calc_checksum_before_scn(const palf::SCN scn, + int calc_checksum_before_scn(const share::SCN scn, uint64_t &checksum, - palf::SCN &checksum_scn); + share::SCN &checksum_scn); void update_checksum(const uint64_t checksum, - const palf::SCN checksum_scn); + const share::SCN checksum_scn); int log_submitted(const ObRedoLogSubmitHelper &helper); // the function apply the side effect of dirty txn and return whether // remaining pending callbacks. // NB: the fact whether there remains pending callbacks currently is only used // for continuing logging when minor freeze - int sync_log_succ(const palf::SCN scn, const ObCallbackScope &callbacks); + int sync_log_succ(const share::SCN scn, const ObCallbackScope &callbacks); void sync_log_fail(const ObCallbackScope &callbacks); bool is_slow_query() const; virtual void set_trans_ctx(transaction::ObPartTransCtx *ctx); @@ -434,14 +434,14 @@ public: int check_tx_mem_size_overflow(bool &is_overflow); public: void on_tsc_retry(const ObMemtableKey& key, - const palf::SCN snapshot_version, - const palf::SCN max_trans_version, + const share::SCN snapshot_version, + const share::SCN max_trans_version, const transaction::ObTransID &conflict_tx_id); void on_wlock_retry(const ObMemtableKey& key, const transaction::ObTransID &conflict_tx_id); virtual int64_t to_string(char *buf, const int64_t buf_len) const; virtual storage::ObTxTableGuard *get_tx_table_guard() override { return &tx_table_guard_; } virtual transaction::ObTransID get_tx_id() const override; - virtual palf::SCN get_tx_end_scn() const override; + virtual share::SCN get_tx_end_scn() const override; // mainly used by revert ref void reset_trans_table_guard(); @@ -454,7 +454,7 @@ public: void replay_done(); int64_t get_checksum() const { return trans_mgr_.get_checksum(); } int64_t get_tmp_checksum() const { return trans_mgr_.get_tmp_checksum(); } - palf::SCN get_checksum_scn() const { return trans_mgr_.get_checksum_scn(); } + share::SCN get_checksum_scn() const { return trans_mgr_.get_checksum_scn(); } public: // table lock. int enable_lock_table(storage::ObTableHandleV2 &handle); @@ -468,18 +468,18 @@ public: int check_modify_time_elapsed(const common::ObTabletID &tablet_id, const int64_t timestamp); int iterate_tx_obj_lock_op(ObLockOpIterator &iter) const; - int check_lock_need_replay(const palf::SCN &scn, + int check_lock_need_replay(const share::SCN &scn, const transaction::tablelock::ObTableLockOp &lock_op, bool &need_replay); int add_lock_record(const transaction::tablelock::ObTableLockOp &lock_op); int replay_add_lock_record(const transaction::tablelock::ObTableLockOp &lock_op, - const palf::SCN &scn); + const share::SCN &scn); void remove_lock_record(ObMemCtxLockOpLinkNode *lock_op); - void set_log_synced(ObMemCtxLockOpLinkNode *lock_op, const palf::SCN &scn); + void set_log_synced(ObMemCtxLockOpLinkNode *lock_op, const share::SCN &scn); // replay lock to lock map and trans part ctx. // used by the replay process of multi data source. int replay_lock(const transaction::tablelock::ObTableLockOp &lock_op, - const palf::SCN &scn); + const share::SCN &scn); int recover_from_table_lock_durable_info(const ObTableLockInfo &table_lock_info); int get_table_lock_store_info(ObTableLockInfo &table_lock_info); // for deadlock detect. @@ -488,12 +488,12 @@ public: private: int do_trans_end( const bool commit, - const palf::SCN trans_version, - const palf::SCN final_scn, + const share::SCN trans_version, + const share::SCN final_scn, const int end_code); int clear_table_lock_(const bool is_commit, - const palf::SCN &commit_version, - const palf::SCN &commit_scn); + const share::SCN &commit_version, + const share::SCN &commit_scn); int rollback_table_lock_(int64_t seq_no); int register_multi_source_data_if_need_( const transaction::tablelock::ObTableLockOp &lock_op, diff --git a/src/storage/memtable/ob_memtable_interface.h b/src/storage/memtable/ob_memtable_interface.h index 9313924a8..63e9e6de0 100644 --- a/src/storage/memtable/ob_memtable_interface.h +++ b/src/storage/memtable/ob_memtable_interface.h @@ -59,15 +59,15 @@ public: virtual void inc_ref() = 0; virtual void dec_ref() = 0; virtual int trans_begin() = 0; - virtual int trans_end(const bool commit, const palf::SCN trans_version, const palf::SCN final_scn) = 0; + virtual int trans_end(const bool commit, const share::SCN trans_version, const share::SCN final_scn) = 0; virtual int trans_clear() = 0; virtual int elr_trans_preparing() = 0; virtual int trans_kill() = 0; virtual int trans_publish() = 0; virtual int trans_replay_begin() = 0; virtual int trans_replay_end(const bool commit, - const palf::SCN trans_version, - const palf::SCN final_scn, + const share::SCN trans_version, + const share::SCN final_scn, const uint64_t log_cluster_version = 0, const uint64_t checksum = 0) = 0; virtual void print_callbacks() = 0; @@ -148,7 +148,7 @@ struct ObMergePriorityInfo class ObIMemtable: public storage::ObITable { public: - ObIMemtable() : snapshot_version_(palf::SCN::max_scn()) + ObIMemtable() : snapshot_version_(share::SCN::max_scn()) {} virtual ~ObIMemtable() {} @@ -209,9 +209,9 @@ public: virtual void dec_pending_lob_count() {} virtual int on_memtable_flushed() { return common::OB_SUCCESS; } virtual bool can_be_minor_merged() { return false; } - void set_snapshot_version(const palf::SCN snapshot_version) { snapshot_version_ = snapshot_version; } + void set_snapshot_version(const share::SCN snapshot_version) { snapshot_version_ = snapshot_version; } virtual int64_t get_snapshot_version() const override { return snapshot_version_.get_val_for_tx(); } - virtual palf::SCN get_snapshot_version_scn() const { return snapshot_version_; } + virtual share::SCN get_snapshot_version_scn() const { return snapshot_version_; } virtual int64_t get_upper_trans_version() const override { return OB_NOT_SUPPORTED; } virtual int64_t get_max_merged_trans_version() const override @@ -272,7 +272,7 @@ public: return false; } protected: - palf::SCN snapshot_version_; + share::SCN snapshot_version_; }; //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/storage/memtable/ob_memtable_iterator.cpp b/src/storage/memtable/ob_memtable_iterator.cpp index 30ffef879..3f01d0c5c 100644 --- a/src/storage/memtable/ob_memtable_iterator.cpp +++ b/src/storage/memtable/ob_memtable_iterator.cpp @@ -29,6 +29,7 @@ namespace oceanbase { using namespace common; +using namespace share; using namespace storage; using namespace blocksstable; using namespace transaction; @@ -1147,7 +1148,7 @@ int ObMemtableMultiVersionScanIterator::iterate_uncommitted_row_value_(ObDatumRo int64_t sql_seq = -1; int64_t first_sql_sequence = -1; int64_t trans_version = INT64_MAX; - palf::SCN trans_scn; + SCN trans_scn; bool same_sql_sequence_flag = true; if (OB_ISNULL(value_iter_)) { ret = OB_INVALID_ARGUMENT; diff --git a/src/storage/memtable/ob_memtable_iterator.h b/src/storage/memtable/ob_memtable_iterator.h index d998aa0c2..d26869f17 100644 --- a/src/storage/memtable/ob_memtable_iterator.h +++ b/src/storage/memtable/ob_memtable_iterator.h @@ -132,10 +132,10 @@ public: virtual void reuse() override { reset(); } ObIMemtable* get_memtable() { return memtable_; } int get_key_val(const ObMemtableKey*& key, ObMvccRow*& row) { return row_iter_.get_key_val(key, row); } - palf::SCN get_read_snapshot() const + share::SCN get_read_snapshot() const { return (NULL == context_ || NULL == context_->store_ctx_ || !context_->store_ctx_->is_valid()) - ? palf::SCN::min_scn() + ? share::SCN::min_scn() : context_->store_ctx_->mvcc_acc_ctx_.get_snapshot_version(); } uint8_t get_iter_flag() { return iter_flag_; } diff --git a/src/storage/memtable/ob_multi_source_data.h b/src/storage/memtable/ob_multi_source_data.h index 1fc90d56a..c3700b3f6 100644 --- a/src/storage/memtable/ob_multi_source_data.h +++ b/src/storage/memtable/ob_multi_source_data.h @@ -78,7 +78,7 @@ public: virtual bool is_valid() const = 0; virtual int64_t get_data_size() const = 0; virtual MultiSourceDataUnitType type() const = 0; - virtual int set_scn(const palf::SCN &scn) + virtual int set_scn(const share::SCN &scn) { UNUSED(scn); return common::OB_SUCCESS; diff --git a/src/storage/memtable/ob_redo_log_generator.cpp b/src/storage/memtable/ob_redo_log_generator.cpp index 08baee135..8e3c5dca4 100644 --- a/src/storage/memtable/ob_redo_log_generator.cpp +++ b/src/storage/memtable/ob_redo_log_generator.cpp @@ -22,6 +22,7 @@ namespace oceanbase { using namespace common; +using namespace share; namespace memtable { @@ -201,7 +202,7 @@ int ObRedoLogGenerator::log_submitted(const ObCallbackScope &callbacks) return ret; } -int ObRedoLogGenerator::sync_log_succ(const palf::SCN scn, const ObCallbackScope &callbacks) +int ObRedoLogGenerator::sync_log_succ(const SCN scn, const ObCallbackScope &callbacks) { // no need to submit log // since the number of log callback is enough now diff --git a/src/storage/memtable/ob_redo_log_generator.h b/src/storage/memtable/ob_redo_log_generator.h index ad96855a6..77e59db95 100644 --- a/src/storage/memtable/ob_redo_log_generator.h +++ b/src/storage/memtable/ob_redo_log_generator.h @@ -74,7 +74,7 @@ public: const bool log_for_lock_node); int search_unsubmitted_dup_tablet_redo(); int log_submitted(const ObCallbackScope &callbacks); - int sync_log_succ(const palf::SCN scn, const ObCallbackScope &callbacks); + int sync_log_succ(const share::SCN scn, const ObCallbackScope &callbacks); void sync_log_fail(const ObCallbackScope &callbacks); ObITransCallback *get_generate_cursor() { return (ObITransCallback *)*generate_cursor_; } diff --git a/src/storage/memtable/ob_row_compactor.cpp b/src/storage/memtable/ob_row_compactor.cpp index a65d8f406..452644eb3 100644 --- a/src/storage/memtable/ob_row_compactor.cpp +++ b/src/storage/memtable/ob_row_compactor.cpp @@ -30,6 +30,7 @@ class ObTxTable; }; using namespace common; +using namespace share; using namespace storage; using namespace blocksstable; namespace memtable @@ -272,13 +273,13 @@ int ObMemtableRowCompactor::init(ObMvccRow *row, // So modification is guaranteed to be safety with another modification, // while we need pay attention to the concurrency between lock_for_read // and modification(such as compact) -int ObMemtableRowCompactor::compact(const palf::SCN snapshot_version) +int ObMemtableRowCompactor::compact(const SCN snapshot_version) { int ret = OB_SUCCESS; if (!is_inited_) { ret = OB_NOT_INIT; - } else if (!snapshot_version.is_valid() || palf::SCN::max_scn() == snapshot_version) { + } else if (!snapshot_version.is_valid() || SCN::max_scn() == snapshot_version) { STORAGE_LOG(ERROR, "unexpected snapshot version", K(ret), K(snapshot_version)); ret = OB_ERR_UNEXPECTED; } else if (NULL != row_->latest_compact_node_ && @@ -306,7 +307,7 @@ int ObMemtableRowCompactor::compact(const palf::SCN snapshot_version) // Find position from where compaction started forward or backward until reached // oldest node or latest compaction node -void ObMemtableRowCompactor::find_start_pos_(const palf::SCN snapshot_version, +void ObMemtableRowCompactor::find_start_pos_(const SCN snapshot_version, ObMvccTransNode *&start) { int64_t search_cnt = 0; @@ -317,7 +318,7 @@ void ObMemtableRowCompactor::find_start_pos_(const palf::SCN snapshot_version, if (NULL == row_->latest_compact_node_) { // Traverse forward from list_head // We go from head to find the suitable node for compact node start - if (palf::SCN::max_scn() == start->trans_version_ // skip uncommited + if (SCN::max_scn() == start->trans_version_ // skip uncommited || snapshot_version < start->trans_version_ // skip bigger txn || !start->is_committed()) { // skip uncommited start = start->prev_; @@ -332,7 +333,7 @@ void ObMemtableRowCompactor::find_start_pos_(const palf::SCN snapshot_version, if (NULL != start->next_ // stop at null && snapshot_version >= start->next_->trans_version_ // stop at bigger txn && start->next_->is_committed() // stop at uncommitted - && palf::SCN::max_scn() != start->next_->trans_version_) { // stop at uncommitted + && SCN::max_scn() != start->next_->trans_version_) { // stop at uncommitted start = start->next_; search_cnt++; } else { @@ -397,7 +398,7 @@ int ObMemtableRowCompactor::try_cleanout_tx_node_during_compact_(ObTxTableGuard return ret; } -ObMvccTransNode *ObMemtableRowCompactor::construct_compact_node_(const palf::SCN snapshot_version, +ObMvccTransNode *ObMemtableRowCompactor::construct_compact_node_(const SCN snapshot_version, ObMvccTransNode *save) { int ret = OB_SUCCESS; @@ -537,7 +538,7 @@ ObMvccTransNode *ObMemtableRowCompactor::construct_compact_node_(const palf::SCN } else if (is_lock_node) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "unexpected lock node", K(ret), "node", *save); - } else if (palf::SCN::max_scn() == save->trans_version_) { + } else if (SCN::max_scn() == save->trans_version_) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "unexpected trans version", K(ret), "node", *save); } else { diff --git a/src/storage/memtable/ob_row_compactor.h b/src/storage/memtable/ob_row_compactor.h index b18a4f521..db5e1f494 100644 --- a/src/storage/memtable/ob_row_compactor.h +++ b/src/storage/memtable/ob_row_compactor.h @@ -19,7 +19,7 @@ #include "lib/allocator/ob_small_allocator.h" #include "lib/lock/ob_spin_lock.h" #include "common/object/ob_object.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -118,11 +118,11 @@ public: common::ObIAllocator *node_alloc, const bool for_replay); // compact and refresh the update counter by snapshot version - int compact(const palf::SCN snapshot_version); + int compact(const share::SCN snapshot_version); private: - void find_start_pos_(const palf::SCN snapshot_version, + void find_start_pos_(const share::SCN snapshot_version, ObMvccTransNode *&save); - ObMvccTransNode *construct_compact_node_(const palf::SCN snapshot_version, + ObMvccTransNode *construct_compact_node_(const share::SCN snapshot_version, ObMvccTransNode *save); int try_cleanout_tx_node_during_compact_(storage::ObTxTableGuard &tx_table_guard, ObMvccTransNode *tnode); diff --git a/src/storage/meta_mem/ob_meta_pointer_map.h b/src/storage/meta_mem/ob_meta_pointer_map.h index e083dc64a..5fc8d0437 100644 --- a/src/storage/meta_mem/ob_meta_pointer_map.h +++ b/src/storage/meta_mem/ob_meta_pointer_map.h @@ -239,18 +239,18 @@ int ObMetaPointerMap::erase(const Key &key) ret = common::OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "invalid argument", K(ret), K(key)); } else { - common::ObBucketHashWLockGuard lock_guard(ResourceMap::bucket_lock_, ResourceMap::hash_func_(key)); - if (OB_FAIL(ResourceMap::map_.get_refactored(key, ptr))) { - STORAGE_LOG(WARN, "fail to get from map", K(ret)); - } else if (OB_FAIL(ResourceMap::map_.erase_refactored(key))) { - STORAGE_LOG(WARN, "fail to erase from map", K(ret)); - } else { - ObMetaPointer *value = ptr->get_value_ptr(); - value->reset_obj(); - if (OB_FAIL(ResourceMap::dec_handle_ref(ptr))) { - STORAGE_LOG(WARN, "fail to dec handle ref", K(ret)); - } - } + common::ObBucketHashWLockGuard lock_guard(ResourceMap::bucket_lock_, ResourceMap::hash_func_(key)); + if (OB_FAIL(ResourceMap::map_.get_refactored(key, ptr))) { + STORAGE_LOG(WARN, "fail to get from map", K(ret)); + } else if (OB_FAIL(ResourceMap::map_.erase_refactored(key))) { + STORAGE_LOG(WARN, "fail to erase from map", K(ret)); + } else { + ObMetaPointer *value = ptr->get_value_ptr(); + value->reset_obj(); + if (OB_FAIL(ResourceMap::dec_handle_ref(ptr))) { + STORAGE_LOG(WARN, "fail to dec handle ref", K(ret)); + } + } } return ret; } diff --git a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp index 9fde70b3d..81b3364dc 100644 --- a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp +++ b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp @@ -365,7 +365,7 @@ void ObTenantMetaMemMgr::gc_sstable(ObSSTable *sstable) } } -int ObTenantMetaMemMgr::get_min_end_scn_for_ls(const share::ObLSID &ls_id, palf::SCN &end_scn) +int ObTenantMetaMemMgr::get_min_end_scn_for_ls(const share::ObLSID &ls_id, SCN &end_scn) { int ret = OB_SUCCESS; end_scn = ObScnRange::MAX_SCN; diff --git a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h index b09df99e9..a62f46ec6 100644 --- a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h +++ b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h @@ -123,7 +123,7 @@ public: // TIPS: // - only for tx data table to find min log ts. - int get_min_end_scn_for_ls(const share::ObLSID &ls_id, palf::SCN &end_scn); + int get_min_end_scn_for_ls(const share::ObLSID &ls_id, share::SCN &end_scn); // garbage collector for sstable and memtable. int push_table_into_gc_queue(ObITable *table, const ObITable::TableType table_type); diff --git a/src/storage/ob_dml_running_ctx.cpp b/src/storage/ob_dml_running_ctx.cpp index 4208bfea9..a416eaf01 100644 --- a/src/storage/ob_dml_running_ctx.cpp +++ b/src/storage/ob_dml_running_ctx.cpp @@ -26,6 +26,7 @@ namespace oceanbase { using namespace common; +using namespace share; using namespace blocksstable; namespace storage { @@ -128,7 +129,7 @@ int ObDMLRunningCtx::prepare_column_desc( int ObDMLRunningCtx::prepare_relative_table( const share::schema::ObTableSchemaParam &schema, ObTabletHandle &tablet_handle, - const palf::SCN &read_snapshot) + const SCN &read_snapshot) { int ret = OB_SUCCESS; if (OB_FAIL(relative_table_.init(&schema, tablet_handle.get_obj()->get_tablet_meta().tablet_id_, diff --git a/src/storage/ob_dml_running_ctx.h b/src/storage/ob_dml_running_ctx.h index e995db820..2f2d846c2 100644 --- a/src/storage/ob_dml_running_ctx.h +++ b/src/storage/ob_dml_running_ctx.h @@ -16,7 +16,7 @@ #include "lib/container/ob_iarray.h" #include "storage/ob_i_store.h" #include "storage/ob_relative_table.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -66,7 +66,7 @@ private: int prepare_relative_table( const share::schema::ObTableSchemaParam &schema, ObTabletHandle &tablet_handle, - const palf::SCN &read_snapshot); + const share::SCN &read_snapshot); int check_schema_version(share::schema::ObMultiVersionSchemaService &schema_service, const uint64_t tenant_id, const uint64_t table_id, diff --git a/src/storage/ob_i_memtable_mgr.cpp b/src/storage/ob_i_memtable_mgr.cpp index c3d278cd8..e7bd557f5 100644 --- a/src/storage/ob_i_memtable_mgr.cpp +++ b/src/storage/ob_i_memtable_mgr.cpp @@ -74,7 +74,7 @@ int ObIMemtableMgr::get_all_memtables(ObTableHdlArray &handles) return ret; } -int ObIMemtableMgr::get_newest_clog_checkpoint_scn(palf::SCN &clog_checkpoint_scn) +int ObIMemtableMgr::get_newest_clog_checkpoint_scn(SCN &clog_checkpoint_scn) { int ret = OB_SUCCESS; @@ -92,7 +92,7 @@ int ObIMemtableMgr::get_newest_clog_checkpoint_scn(palf::SCN &clog_checkpoint_sc return ret; } -int ObIMemtableMgr::get_newest_snapshot_version(palf::SCN &snapshot_version) +int ObIMemtableMgr::get_newest_snapshot_version(SCN &snapshot_version) { int ret = OB_SUCCESS; @@ -109,7 +109,7 @@ int ObIMemtableMgr::get_newest_snapshot_version(palf::SCN &snapshot_version) return ret; } -int ObIMemtableMgr::release_memtables(const palf::SCN &scn) +int ObIMemtableMgr::release_memtables(const SCN &scn) { SpinWLockGuard lock_guard(lock_); int ret = OB_SUCCESS; diff --git a/src/storage/ob_i_memtable_mgr.h b/src/storage/ob_i_memtable_mgr.h index 620c83b94..7d41d9b5e 100644 --- a/src/storage/ob_i_memtable_mgr.h +++ b/src/storage/ob_i_memtable_mgr.h @@ -61,7 +61,7 @@ public: logservice::ObLogHandler *log_handler, ObFreezer *freezer, ObTenantMetaMemMgr *t3m); - virtual int create_memtable(const palf::SCN clog_checkpoint_scn, + virtual int create_memtable(const share::SCN clog_checkpoint_scn, const int64_t schema_version, const bool for_replay = false) { @@ -81,7 +81,7 @@ public: virtual int get_boundary_memtable(ObTableHandleV2 &handle) { return OB_SUCCESS; } - virtual int get_memtable_for_replay(palf::SCN replay_scn, ObTableHandleV2 &handle) + virtual int get_memtable_for_replay(share::SCN replay_scn, ObTableHandleV2 &handle) { return OB_SUCCESS; } @@ -94,7 +94,7 @@ public: memtable::ObMemtable *&memtable, const memtable::MultiSourceDataUnitType type) const; - int release_memtables(const palf::SCN &scn); + int release_memtables(const share::SCN &scn); // force release all memtables // WARNING: this will release all the ref of memtable, make sure you will not use it again. int release_memtables(); @@ -105,9 +105,9 @@ public: return has_memtable_(); } - int get_newest_clog_checkpoint_scn(palf::SCN &clog_checkpoint_scn); + int get_newest_clog_checkpoint_scn(share::SCN &clog_checkpoint_scn); - int get_newest_snapshot_version(palf::SCN &snapshot_version); + int get_newest_snapshot_version(share::SCN &snapshot_version); OB_INLINE int64_t dec_ref() { return ATOMIC_SAF(&ref_cnt_, 1 /* just sub 1 */); } OB_INLINE int64_t get_ref() const { return ATOMIC_LOAD(&ref_cnt_); } diff --git a/src/storage/ob_i_store.cpp b/src/storage/ob_i_store.cpp index 1a7b79fa7..0213864af 100644 --- a/src/storage/ob_i_store.cpp +++ b/src/storage/ob_i_store.cpp @@ -80,7 +80,7 @@ void ObStoreCtx::reset() int ObStoreCtx::init_for_read(const ObLSID &ls_id, const int64_t timeout, const int64_t tx_lock_timeout, - const palf::SCN &snapshot_version) + const SCN &snapshot_version) { int ret = OB_SUCCESS; ObLSService *ls_svr = MTL(ObLSService*); @@ -96,7 +96,7 @@ int ObStoreCtx::init_for_read(const ObLSID &ls_id, int ObStoreCtx::init_for_read(const ObLSHandle &ls_handle, const int64_t timeout, const int64_t tx_lock_timeout, - const palf::SCN &snapshot_version) + const SCN &snapshot_version) { int ret = OB_SUCCESS; ObLS *ls = nullptr; @@ -130,7 +130,7 @@ void ObStoreCtx::force_print_trace_log() void ObStoreRowLockState::reset() { is_locked_ = false; - trans_version_ = palf::SCN::min_scn(); + trans_version_ = SCN::min_scn(); lock_trans_id_.reset(); lock_data_sequence_ = 0; is_delayed_cleanout_ = false; diff --git a/src/storage/ob_i_store.h b/src/storage/ob_i_store.h index 2b2f1c1c4..49f28d14e 100644 --- a/src/storage/ob_i_store.h +++ b/src/storage/ob_i_store.h @@ -239,7 +239,7 @@ struct ObStoreRowLockState { public: ObStoreRowLockState() - : is_locked_(false), trans_version_(palf::SCN::min_scn()), lock_trans_id_(), + : is_locked_(false), trans_version_(share::SCN::min_scn()), lock_trans_id_(), lock_data_sequence_(0), is_delayed_cleanout_(false), mvcc_row_(NULL) {} @@ -249,7 +249,7 @@ public: KP_(mvcc_row)); bool is_locked_; - palf::SCN trans_version_; + share::SCN trans_version_; transaction::ObTransID lock_trans_id_; int64_t lock_data_sequence_; bool is_delayed_cleanout_; @@ -420,11 +420,11 @@ struct ObStoreCtx int init_for_read(const share::ObLSID &ls_id, const int64_t timeout, const int64_t lock_timeout_us, - const palf::SCN &snapshot_version); + const share::SCN &snapshot_version); int init_for_read(const storage::ObLSHandle &ls_handle, const int64_t timeout, const int64_t lock_timeout_us, - const palf::SCN &snapshot_version); + const share::SCN &snapshot_version); void force_print_trace_log(); TO_STRING_KV(KP(this), K_(ls_id), @@ -442,7 +442,7 @@ struct ObStoreCtx int64_t table_version_; // used to update memtable's max_schema_version int64_t timeout_; memtable::ObMvccAccessCtx mvcc_acc_ctx_; // all txn relative context - palf::SCN replay_log_scn_; // used in replay pass log_ts + share::SCN replay_log_scn_; // used in replay pass log_ts }; diff --git a/src/storage/ob_i_table.cpp b/src/storage/ob_i_table.cpp index 3a74d0bb5..76cda204d 100644 --- a/src/storage/ob_i_table.cpp +++ b/src/storage/ob_i_table.cpp @@ -24,6 +24,7 @@ using namespace oceanbase; using namespace oceanbase::blocksstable; using namespace oceanbase::storage; using namespace oceanbase::common; +using namespace oceanbase::share; using namespace oceanbase::memtable; using namespace oceanbase::transaction::tablelock; @@ -677,13 +678,13 @@ int ObTablesHandleArray::check_continues(const share::ObScnRange *scn_range) con // there can only be one major or buf minor const ObITable *last_table = nullptr; const ObITable *table = nullptr; - palf::SCN base_end_scn = palf::SCN::min_scn(); + SCN base_end_scn = SCN::min_scn(); int64_t i = 0; if (OB_ISNULL(table = tables_.at(i))) { ret = OB_ERR_SYS; LOG_WARN("table is NULL", KPC(table)); } else if (table->is_major_sstable() || table->is_buf_minor_sstable()) { - base_end_scn = table->is_buf_minor_sstable() ? table->get_end_scn() : palf::SCN::min_scn(); + base_end_scn = table->is_buf_minor_sstable() ? table->get_end_scn() : SCN::min_scn(); i++; } // 2:check minor sstable diff --git a/src/storage/ob_i_table.h b/src/storage/ob_i_table.h index 410fbada0..414b316c2 100644 --- a/src/storage/ob_i_table.h +++ b/src/storage/ob_i_table.h @@ -26,7 +26,7 @@ #include "storage/meta_mem/ob_tenant_meta_obj_pool.h" #include "share/leak_checker/obj_leak_checker.h" #include "share/ob_table_range.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -57,11 +57,6 @@ namespace share struct ObScnRange; } -namespace palf -{ -class SCN; -} - namespace storage { class ObTxDataMemtable; @@ -129,8 +124,8 @@ public: OB_INLINE bool is_remote_logical_minor_sstable() const { return ObITable::is_remote_logical_minor_sstable(table_type_); } OB_INLINE const common::ObTabletID &get_tablet_id() const { return tablet_id_; } - OB_INLINE palf::SCN get_start_scn() const { return scn_range_.start_scn_; } - OB_INLINE palf::SCN get_end_scn() const { return scn_range_.end_scn_; } + OB_INLINE share::SCN get_start_scn() const { return scn_range_.start_scn_; } + OB_INLINE share::SCN get_end_scn() const { return scn_range_.end_scn_; } OB_INLINE int64_t get_snapshot_version() const { OB_ASSERT(is_major_sstable()); @@ -203,8 +198,8 @@ public: const common::ObIArray &ranges, ObStoreRowIterator *&row_iter) = 0; - virtual OB_INLINE palf::SCN get_start_scn() const; - virtual OB_INLINE palf::SCN get_end_scn() const; + virtual OB_INLINE share::SCN get_start_scn() const; + virtual OB_INLINE share::SCN get_end_scn() const; virtual OB_INLINE share::ObScnRange &get_scn_range() { return key_.scn_range_; } virtual OB_INLINE bool is_trans_state_deterministic() { return get_upper_trans_version() < INT64_MAX; } virtual int64_t get_snapshot_version() const { return key_.get_snapshot_version(); } @@ -456,12 +451,12 @@ OB_INLINE bool ObITable::TableKey::is_valid() const } -OB_INLINE palf::SCN ObITable::get_start_scn() const +OB_INLINE share::SCN ObITable::get_start_scn() const { return key_.get_start_scn(); } -OB_INLINE palf::SCN ObITable::get_end_scn() const +OB_INLINE share::SCN ObITable::get_end_scn() const { return key_.get_end_scn(); } diff --git a/src/storage/ob_partition_range_spliter.cpp b/src/storage/ob_partition_range_spliter.cpp index 95c27b011..73b02d613 100644 --- a/src/storage/ob_partition_range_spliter.cpp +++ b/src/storage/ob_partition_range_spliter.cpp @@ -1603,7 +1603,7 @@ int ObPartitionIncrementalRangeSpliter::ObIncrementalIterator::prepare_store_ctx int ret = OB_SUCCESS; auto ls_id = merge_ctx_.param_.ls_id_; auto &snapshot = merge_ctx_.sstable_version_range_.snapshot_version_; - palf::SCN scn; + SCN scn; if (OB_FAIL(scn.convert_for_tx(snapshot))) { STORAGE_LOG(WARN, "convert for tx fail", K(ret), K(ls_id), K(snapshot)); } else if (OB_FAIL(store_ctx_.init_for_read(ls_id, diff --git a/src/storage/ob_storage_rpc.cpp b/src/storage/ob_storage_rpc.cpp index 6609809c6..39ceb117b 100644 --- a/src/storage/ob_storage_rpc.cpp +++ b/src/storage/ob_storage_rpc.cpp @@ -115,7 +115,7 @@ ObCopyMacroBlockRangeArg::ObCopyMacroBlockRangeArg() ls_id_(), table_key_(), data_version_(0), - backfill_tx_scn_(palf::SCN::min_scn()), + backfill_tx_scn_(SCN::min_scn()), copy_macro_range_info_() { } @@ -136,7 +136,7 @@ bool ObCopyMacroBlockRangeArg::is_valid() const && ls_id_.is_valid() && table_key_.is_valid() && data_version_ >= 0 - && backfill_tx_scn_ >= palf::SCN::min_scn() + && backfill_tx_scn_ >= SCN::min_scn() && copy_macro_range_info_.is_valid(); } diff --git a/src/storage/ob_storage_rpc.h b/src/storage/ob_storage_rpc.h index fe4456d47..27e6efbda 100644 --- a/src/storage/ob_storage_rpc.h +++ b/src/storage/ob_storage_rpc.h @@ -93,7 +93,7 @@ public: share::ObLSID ls_id_; storage::ObITable::TableKey table_key_; int64_t data_version_; - palf::SCN backfill_tx_scn_; + share::SCN backfill_tx_scn_; storage::ObCopyMacroRangeInfo copy_macro_range_info_; DISALLOW_COPY_AND_ASSIGN(ObCopyMacroBlockRangeArg); }; diff --git a/src/storage/ob_storage_schema_recorder.cpp b/src/storage/ob_storage_schema_recorder.cpp index 6acc9fd23..b10e58248 100644 --- a/src/storage/ob_storage_schema_recorder.cpp +++ b/src/storage/ob_storage_schema_recorder.cpp @@ -25,28 +25,22 @@ #include "storage/meta_mem/ob_tablet_handle.h" #include "storage/tx_storage/ob_ls_handle.h"//ObLSHandle #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { using namespace common; +using namespace share; using namespace clog; using namespace share::schema; namespace storage { -int ObStorageSchemaRecorder::ObStorageSchemaLogCb::set_table_version(const int64_t table_version) +void ObStorageSchemaRecorder::ObStorageSchemaLogCb::set_table_version(const int64_t table_version) { - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!ATOMIC_BCAS(&table_version_, OB_INVALID_VERSION, table_version))) { - ret = OB_ERR_UNEXPECTED; - LOG_ERROR("double set table_version", K(ret), K(table_version_), K(table_version)); - } - - return ret; + ATOMIC_SET(&table_version_, table_version); } int ObStorageSchemaRecorder::ObStorageSchemaLogCb::on_success() @@ -151,7 +145,7 @@ int ObStorageSchemaRecorder::init( // schema log is barrier, there is no concurrency problem, no need to lock int ObStorageSchemaRecorder::replay_schema_log( - const palf::SCN &scn, + const SCN &scn, const char *buf, const int64_t size, int64_t &pos) @@ -311,7 +305,7 @@ int ObStorageSchemaRecorder::get_tablet_handle(ObTabletHandle &tablet_handle) return ret; } -int ObStorageSchemaRecorder::replay_get_tablet_handle(const palf::SCN &scn, ObTabletHandle &tablet_handle) +int ObStorageSchemaRecorder::replay_get_tablet_handle(const SCN &scn, ObTabletHandle &tablet_handle) { int ret = OB_SUCCESS; ObLSHandle ls_handle; @@ -492,24 +486,25 @@ int ObStorageSchemaRecorder::submit_schema_log(const int64_t table_id) logcb_ptr_ = new(buf) ObStorageSchemaLogCb(*this); } } - if (FAILEDx(logcb_ptr_->set_table_version(storage_schema_->get_schema_version()))) { - LOG_ERROR("fail to set table version", K(ret), K_(tablet_id)); - } else if (FALSE_IT(ATOMIC_STORE(&logcb_finish_flag_, false))) { - } else if (FALSE_IT(storage_schema_->set_sync_finish(false))) { - } else if (OB_FAIL(tablet_handle_.get_obj()->save_multi_source_data_unit(storage_schema_, - palf::SCN::max_scn(), false/*for_replay*/, memtable::MemtableRefOp::INC_REF))) { - if (OB_BLOCK_FROZEN != ret) { - LOG_WARN("failed to inc ref for storage schema", K(ret), K_(tablet_id), K(storage_schema_)); + if (OB_SUCC(ret)) { + logcb_ptr_->set_table_version(storage_schema_->get_schema_version()); + ATOMIC_STORE(&logcb_finish_flag_, false); + storage_schema_->set_sync_finish(false); + if (OB_FAIL(tablet_handle_.get_obj()->save_multi_source_data_unit(storage_schema_, + SCN::max_scn(), false/*for_replay*/, memtable::MemtableRefOp::INC_REF))) { + if (OB_BLOCK_FROZEN != ret) { + LOG_WARN("failed to inc ref for storage schema", K(ret), K_(tablet_id), K(storage_schema_)); + } + } else if (OB_FAIL(log_handler_->append(clog_buf_, clog_len_, SCN::min_scn(), need_nonblock, logcb_ptr_, lsn, clog_scn_))) { + LOG_WARN("fail to submit log", K(ret), K_(tablet_id)); + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(dec_ref_on_memtable(false))) { + LOG_ERROR("failed to dec ref on memtable", K(tmp_ret), K_(ls_id), K_(tablet_id)); + } + } else { + LOG_INFO("submit schema log succeed", K(ret), K_(ls_id), K_(tablet_id), K_(clog_scn), K_(clog_len), + "schema_version", storage_schema_->get_schema_version()); } - } else if (OB_FAIL(log_handler_->append(clog_buf_, clog_len_, palf::SCN::min_scn(), need_nonblock, logcb_ptr_, lsn, clog_scn_))) { - LOG_WARN("fail to submit log", K(ret), K_(tablet_id)); - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(dec_ref_on_memtable(false))) { - LOG_ERROR("failed to dec ref on memtable", K(tmp_ret), K_(ls_id), K_(tablet_id)); - } - } else { - LOG_INFO("submit schema log succeed", K(ret), K_(ls_id), K_(tablet_id), K_(clog_scn), K_(clog_len), - "schema_version", storage_schema_->get_schema_version()); } return ret; diff --git a/src/storage/ob_storage_schema_recorder.h b/src/storage/ob_storage_schema_recorder.h index c49986e3a..302674bb2 100644 --- a/src/storage/ob_storage_schema_recorder.h +++ b/src/storage/ob_storage_schema_recorder.h @@ -21,7 +21,7 @@ #include "storage/ob_storage_schema.h" #include "storage/meta_mem/ob_tablet_handle.h" #include "share/schema/ob_multi_version_schema_service.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -60,9 +60,17 @@ public: logservice::ObLogHandler *log_handler); void reset(); bool is_inited() const { return is_inited_; } + bool is_valid() const + { + return is_inited_ + && ls_id_.is_valid() + && tablet_id_.is_valid() + && nullptr != log_handler_ + && max_saved_table_version_ >= 0; + } // follower - int replay_schema_log(const palf::SCN &scn, const char *buf, const int64_t size, int64_t &pos); + int replay_schema_log(const share::SCN &scn, const char *buf, const int64_t size, int64_t &pos); // leader int try_update_storage_schema( const int64_t table_id, @@ -73,6 +81,7 @@ public: ObStorageSchemaRecorder(const ObStorageSchemaRecorder&) = delete; ObStorageSchemaRecorder& operator=(const ObStorageSchemaRecorder&) = delete; int64_t get_max_sync_version() const { return ATOMIC_LOAD(&max_saved_table_version_); } + TO_STRING_KV(K_(is_inited), K_(ls_id), K_(tablet_id)); private: class ObStorageSchemaLogCb : public logservice::AppendCb @@ -81,7 +90,7 @@ private: virtual int on_success() override; virtual int on_failure() override; - int set_table_version(const int64_t table_version); + void set_table_version(const int64_t table_version); ObStorageSchemaLogCb(ObStorageSchemaRecorder &recorder) : recorder_(recorder), @@ -110,7 +119,7 @@ private: void free_allocated_info(); int try_update_with_lock(const int64_t table_id, const int64_t table_version, const int64_t expire_ts); int get_tablet_handle(ObTabletHandle &tablet_handle); - int replay_get_tablet_handle(const palf::SCN &scn, ObTabletHandle &tablet_handle); + int replay_get_tablet_handle(const share::SCN &scn, ObTabletHandle &tablet_handle); // clog callback void update_table_schema_fail(); void update_table_schema_succ(const int64_t table_version, bool &finish_flag); @@ -129,7 +138,7 @@ private: int64_t max_saved_table_version_; char *clog_buf_; int64_t clog_len_; - palf::SCN clog_scn_; + share::SCN clog_scn_; share::schema::ObSchemaGetterGuard *schema_guard_; ObStorageSchema *storage_schema_; diff --git a/src/storage/ob_storage_struct.cpp b/src/storage/ob_storage_struct.cpp index 7b509ca87..1513f0097 100644 --- a/src/storage/ob_storage_struct.cpp +++ b/src/storage/ob_storage_struct.cpp @@ -69,7 +69,7 @@ ObPartitionBarrierLogStateEnum ObPartitionBarrierLogState::to_persistent_state() return persistent_state; } -void ObPartitionBarrierLogState::set_log_info(const ObPartitionBarrierLogStateEnum state, const int64_t log_id, const palf::SCN &scn, const int64_t schema_version) +void ObPartitionBarrierLogState::set_log_info(const ObPartitionBarrierLogStateEnum state, const int64_t log_id, const SCN &scn, const int64_t schema_version) { state_ = state; log_id_ = log_id; @@ -221,7 +221,10 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam( need_check_sstable_(false), ddl_checkpoint_scn_(), ddl_start_scn_(), - ddl_snapshot_version_(0) + ddl_snapshot_version_(0), + tx_data_(), + binding_info_(), + auto_inc_seq_() { clog_checkpoint_scn_.set_min(); } @@ -233,7 +236,7 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam( const ObStorageSchema *storage_schema, const int64_t rebuild_seq, const bool need_report, - const palf::SCN clog_checkpoint_scn, + const SCN clog_checkpoint_scn, const bool need_check_sstable) : table_handle_(table_handle), snapshot_version_(snapshot_version), @@ -247,7 +250,10 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam( need_check_sstable_(need_check_sstable), ddl_checkpoint_scn_(), ddl_start_scn_(), - ddl_snapshot_version_(0) + ddl_snapshot_version_(0), + tx_data_(), + binding_info_(), + auto_inc_seq_() { clog_checkpoint_scn_ = clog_checkpoint_scn; } @@ -272,7 +278,10 @@ ObUpdateTableStoreParam::ObUpdateTableStoreParam( need_check_sstable_(false), ddl_checkpoint_scn_(), ddl_start_scn_(), - ddl_snapshot_version_(0) + ddl_snapshot_version_(0), + tx_data_(), + binding_info_(), + auto_inc_seq_() { clog_checkpoint_scn_.set_min(); } @@ -295,7 +304,7 @@ ObBatchUpdateTableStoreParam::ObBatchUpdateTableStoreParam() need_report_(false), rebuild_seq_(OB_INVALID_VERSION), update_logical_minor_sstable_(false), - start_scn_(palf::SCN::min_scn()), + start_scn_(SCN::min_scn()), tablet_meta_(nullptr) { } @@ -317,7 +326,7 @@ bool ObBatchUpdateTableStoreParam::is_valid() const && multi_version_start_ >= 0 && rebuild_seq_ > OB_INVALID_VERSION && ((!update_logical_minor_sstable_ && OB_NOT_NULL(tablet_meta_)) - || (update_logical_minor_sstable_ && start_scn_ > palf::SCN::min_scn())); + || (update_logical_minor_sstable_ && start_scn_ > SCN::min_scn())); } int ObBatchUpdateTableStoreParam::assign( @@ -340,7 +349,7 @@ int ObBatchUpdateTableStoreParam::assign( return ret; } -int ObBatchUpdateTableStoreParam::get_max_clog_checkpoint_scn(palf::SCN &clog_checkpoint_scn) const +int ObBatchUpdateTableStoreParam::get_max_clog_checkpoint_scn(SCN &clog_checkpoint_scn) const { int ret = OB_SUCCESS; clog_checkpoint_scn.set_min(); diff --git a/src/storage/ob_storage_struct.h b/src/storage/ob_storage_struct.h index a313b4875..b6c71a7b0 100644 --- a/src/storage/ob_storage_struct.h +++ b/src/storage/ob_storage_struct.h @@ -17,12 +17,15 @@ #include "lib/ob_replica_define.h" #include "common/ob_store_range.h" #include "common/ob_member_list.h" +#include "share/ob_tablet_autoincrement_param.h" #include "share/schema/ob_schema_struct.h" #include "share/schema/ob_table_schema.h" #include "storage/ob_i_table.h" #include "storage/ob_storage_schema.h" #include "storage/tablet/ob_tablet_table_store_flag.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" +#include "storage/tablet/ob_tablet_multi_source_data.h" +#include "storage/tablet/ob_tablet_binding_helper.h" namespace oceanbase { @@ -39,14 +42,14 @@ class ObMigrationTabletParam; typedef common::ObSEArray GetRowkeyArray; typedef common::ObSEArray ScanRangeArray; -static const int64_t EXIST_READ_SNAPSHOT_VERSION = palf::OB_MAX_SCN_TS_NS - 1; -static const int64_t MERGE_READ_SNAPSHOT_VERSION = palf::OB_MAX_SCN_TS_NS - 2; +static const int64_t EXIST_READ_SNAPSHOT_VERSION = share::OB_MAX_SCN_TS_NS - 1; +static const int64_t MERGE_READ_SNAPSHOT_VERSION = share::OB_MAX_SCN_TS_NS - 2; // static const int64_t MV_LEFT_MERGE_READ_SNAPSHOT_VERSION = INT64_MAX - 3; // static const int64_t MV_RIGHT_MERGE_READ_SNAPSHOT_VERSION = INT64_MAX - 4; // static const int64_t MV_MERGE_READ_SNAPSHOT_VERSION = INT64_MAX - 5; // static const int64_t BUILD_INDEX_READ_SNAPSHOT_VERSION = INT64_MAX - 6; // static const int64_t WARM_UP_READ_SNAPSHOT_VERSION = INT64_MAX - 7; -static const int64_t GET_BATCH_ROWS_READ_SNAPSHOT_VERSION = palf::OB_MAX_SCN_TS_NS - 8; +static const int64_t GET_BATCH_ROWS_READ_SNAPSHOT_VERSION = share::OB_MAX_SCN_TS_NS - 8; // static const int64_t GET_SCAN_COST_READ_SNAPSHOT_VERSION = INT64_MAX - 9; @@ -225,9 +228,9 @@ public: ~ObPartitionBarrierLogState() = default; ObPartitionBarrierLogStateEnum &get_state() { return state_; } int64_t get_log_id() { return log_id_; } - palf::SCN get_scn() { return scn_; } + share::SCN get_scn() { return scn_; } int64_t get_schema_version() { return schema_version_; } - void set_log_info(const ObPartitionBarrierLogStateEnum state, const int64_t log_id, const palf::SCN &scn, const int64_t schema_version); + void set_log_info(const ObPartitionBarrierLogStateEnum state, const int64_t log_id, const share::SCN &scn, const int64_t schema_version); NEED_SERIALIZE_AND_DESERIALIZE; TO_STRING_KV(K_(state)); private: @@ -235,7 +238,7 @@ private: private: ObPartitionBarrierLogStateEnum state_; int64_t log_id_; - palf::SCN scn_; + share::SCN scn_; int64_t schema_version_; }; @@ -301,7 +304,7 @@ struct ObUpdateTableStoreParam const ObStorageSchema *storage_schema, const int64_t rebuild_seq, const bool need_report = false, - const palf::SCN clog_checkpoint_scn = palf::SCN::min_scn(), + const share::SCN clog_checkpoint_scn = share::SCN::min_scn(), const bool need_check_sstable = false); ObUpdateTableStoreParam( // for ddl merge task only @@ -316,11 +319,12 @@ struct ObUpdateTableStoreParam bool is_valid() const; TO_STRING_KV(K_(table_handle), K_(snapshot_version), K_(clog_checkpoint_scn), K_(multi_version_start), K_(keep_old_ddl_sstable), K_(need_report), KPC_(storage_schema), K_(rebuild_seq), K_(update_with_major_flag), - K_(need_check_sstable), K_(ddl_checkpoint_scn), K_(ddl_start_scn), K_(ddl_snapshot_version)); + K_(need_check_sstable), K_(ddl_checkpoint_scn), K_(ddl_start_scn), K_(ddl_snapshot_version), + K_(tx_data), K_(binding_info), K_(auto_inc_seq)); ObTableHandleV2 table_handle_; int64_t snapshot_version_; - palf::SCN clog_checkpoint_scn_; + share::SCN clog_checkpoint_scn_; int64_t multi_version_start_; bool keep_old_ddl_sstable_; bool need_report_; @@ -328,9 +332,14 @@ struct ObUpdateTableStoreParam int64_t rebuild_seq_; bool update_with_major_flag_; bool need_check_sstable_; - palf::SCN ddl_checkpoint_scn_; - palf::SCN ddl_start_scn_; + share::SCN ddl_checkpoint_scn_; + share::SCN ddl_start_scn_; int64_t ddl_snapshot_version_; + + // msd + ObTabletTxMultiSourceDataUnit tx_data_; + ObTabletBindingInfo binding_info_; + share::ObTabletAutoincSeq auto_inc_seq_; }; struct ObBatchUpdateTableStoreParam final @@ -340,7 +349,7 @@ struct ObBatchUpdateTableStoreParam final bool is_valid() const; void reset(); int assign(const ObBatchUpdateTableStoreParam ¶m); - int get_max_clog_checkpoint_scn(palf::SCN &clog_checkpoint_scn) const; + int get_max_clog_checkpoint_scn(share::SCN &clog_checkpoint_scn) const; TO_STRING_KV(K_(tables_handle), K_(snapshot_version), K_(multi_version_start), K_(need_report), K_(rebuild_seq), K_(update_logical_minor_sstable), K_(start_scn), KP_(tablet_meta)); @@ -351,7 +360,7 @@ struct ObBatchUpdateTableStoreParam final bool need_report_; int64_t rebuild_seq_; bool update_logical_minor_sstable_; - palf::SCN start_scn_; + share::SCN start_scn_; const ObMigrationTabletParam *tablet_meta_; DISALLOW_COPY_AND_ASSIGN(ObBatchUpdateTableStoreParam); diff --git a/src/storage/ob_storage_table_guard.cpp b/src/storage/ob_storage_table_guard.cpp index f737ee712..c279c99b2 100644 --- a/src/storage/ob_storage_table_guard.cpp +++ b/src/storage/ob_storage_table_guard.cpp @@ -27,12 +27,13 @@ namespace oceanbase { namespace storage { +using namespace share; ObStorageTableGuard::ObStorageTableGuard( ObTablet *tablet, ObStoreCtx &store_ctx, const bool need_control_mem, const bool for_replay, - const palf::SCN replay_scn, + const SCN replay_scn, const bool for_multi_source_data) : tablet_(tablet), store_ctx_(store_ctx), @@ -139,7 +140,7 @@ int ObStorageTableGuard::refresh_and_protect_memtable() ObTableHandleV2 handle; const share::ObLSID &ls_id = tablet_->get_tablet_meta().ls_id_; const common::ObTabletID &tablet_id = tablet_->get_tablet_meta().tablet_id_; - palf::SCN clog_checkpoint_scn; + SCN clog_checkpoint_scn; bool bool_ret = true; const int64_t start = ObTimeUtility::current_time(); diff --git a/src/storage/ob_storage_table_guard.h b/src/storage/ob_storage_table_guard.h index fded5abcf..7a80d395a 100644 --- a/src/storage/ob_storage_table_guard.h +++ b/src/storage/ob_storage_table_guard.h @@ -14,7 +14,7 @@ #define OCEANBASE_STORAGE_OB_STORAGE_TABLE_GUARD #include -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -40,7 +40,7 @@ public: ObStoreCtx &store_ctx, const bool need_control_mem, const bool for_replay = false, - const palf::SCN replay_scn = palf::SCN(), + const share::SCN replay_scn = share::SCN(), const bool for_multi_source_data = false); ~ObStorageTableGuard(); @@ -71,7 +71,7 @@ private: int64_t retry_count_; int64_t last_ts_; bool for_replay_; - palf::SCN replay_scn_; + share::SCN replay_scn_; bool for_multi_source_data_; }; } // namespace storage diff --git a/src/storage/ob_sync_tablet_seq_clog.cpp b/src/storage/ob_sync_tablet_seq_clog.cpp index 4b43e1083..970f9c13e 100644 --- a/src/storage/ob_sync_tablet_seq_clog.cpp +++ b/src/storage/ob_sync_tablet_seq_clog.cpp @@ -173,7 +173,7 @@ int ObSyncTabletSeqLogCb::on_failure() } else if (OB_FAIL(tablet_handle.get_obj()->get_latest_autoinc_seq(autoinc_seq))) { LOG_WARN("fail to get latest autoinc seq", K(ret)); } else if (OB_FAIL(tablet_handle.get_obj()->save_multi_source_data_unit(&autoinc_seq, - palf::SCN::invalid_scn()/*scn*/, + SCN::invalid_scn()/*scn*/, false/*for_replay*/, memtable::MemtableRefOp::DEC_REF, true/*is_callback*/))) { diff --git a/src/storage/restore/ob_ls_restore_args.h b/src/storage/restore/ob_ls_restore_args.h index 13d4537ec..77615559d 100644 --- a/src/storage/restore/ob_ls_restore_args.h +++ b/src/storage/restore/ob_ls_restore_args.h @@ -36,7 +36,7 @@ struct ObTenantRestoreCtx int assign(const ObTenantRestoreCtx &args); int64_t get_job_id() const { return job_id_; } const share::ObRestoreType &get_restore_type() const { return restore_type_; } - const palf::SCN &get_restore_scn() const { return restore_scn_; } + const share::SCN &get_restore_scn() const { return restore_scn_; } uint64_t get_tenant_id() const { return tenant_id_; } int64_t get_backup_cluster_version() const { return backup_cluster_version_; } const common::ObArray &get_backup_set_list() const { return backup_set_list_; } @@ -52,7 +52,7 @@ struct ObTenantRestoreCtx int64_t job_id_; share::ObRestoreType restore_type_; // quick restore or normal restore - palf::SCN restore_scn_; // restore end scn + share::SCN restore_scn_; // restore end scn uint64_t tenant_id_; int64_t backup_cluster_version_; // every set path is integral. diff --git a/src/storage/tablelock/ob_lock_memtable.cpp b/src/storage/tablelock/ob_lock_memtable.cpp index e79ea0dd6..014980d36 100644 --- a/src/storage/tablelock/ob_lock_memtable.cpp +++ b/src/storage/tablelock/ob_lock_memtable.cpp @@ -244,7 +244,7 @@ int ObLockMemtable::unlock_( int ObLockMemtable::check_lock_need_replay_( ObMemtableCtx *mem_ctx, const ObTableLockOp &lock_op, - const palf::SCN &scn, + const SCN &scn, bool &need_replay) { // 1. filter the lock/unlock op that has been dumped at lock memtable. @@ -268,7 +268,7 @@ int ObLockMemtable::check_lock_need_replay_( int ObLockMemtable::replay_lock_( ObMemtableCtx *mem_ctx, const ObTableLockOp &lock_op, - const palf::SCN &scn) + const SCN &scn) { int ret = OB_SUCCESS; bool need_replay = true; @@ -498,8 +498,8 @@ void ObLockMemtable::remove_lock_record(const ObTableLockOp &lock_op) int ObLockMemtable::update_lock_status( const ObTableLockOp &op_info, - const palf::SCN &commit_version, - const palf::SCN &commit_scn, + const SCN &commit_version, + const SCN &commit_scn, const ObTableLockOpStatus status) { int ret = OB_SUCCESS; @@ -735,7 +735,7 @@ int ObLockMemtable::get_frozen_schema_version(int64_t &schema_version) const return OB_NOT_SUPPORTED; } -palf::SCN ObLockMemtable::get_rec_scn() +SCN ObLockMemtable::get_rec_scn() { // no need lock because rec_scn_ aesc except INT64_MAX LOG_INFO("rec_scn of ObLockMemtable is ", K(rec_scn_), K(flushed_scn_), @@ -794,14 +794,14 @@ bool ObLockMemtable::is_active_memtable() const return !ATOMIC_LOAD(&is_frozen_); } -int ObLockMemtable::flush(palf::SCN recycle_scn, +int ObLockMemtable::flush(SCN recycle_scn, bool need_freeze) { int ret = OB_SUCCESS; UNUSED(need_freeze); { WLockGuard guard(flush_lock_); - palf::SCN rec_scn = get_rec_scn(); + SCN rec_scn = get_rec_scn(); if (rec_scn >= recycle_scn) { LOG_INFO("lock memtable no need to flush", K(rec_scn), K(recycle_scn), K(is_frozen_), K(ls_id_)); @@ -895,7 +895,7 @@ int ObLockMemtable::replay_row( int ObLockMemtable::replay_lock( ObMemtableCtx *mem_ctx, const ObTableLockOp &lock_op, - const palf::SCN &scn) + const SCN &scn) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { diff --git a/src/storage/tablelock/ob_lock_memtable.h b/src/storage/tablelock/ob_lock_memtable.h index c40c2795e..fd5e9e8d0 100644 --- a/src/storage/tablelock/ob_lock_memtable.h +++ b/src/storage/tablelock/ob_lock_memtable.h @@ -17,7 +17,7 @@ #include "storage/memtable/ob_memtable_interface.h" #include "storage/tablelock/ob_obj_lock.h" #include "lib/lock/ob_spin_rwlock.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -86,8 +86,8 @@ public: // @param[in] commit_scn, if it is called by commit, set the commit logts too. // @param[in] status, the lock op status will be set. int update_lock_status(const ObTableLockOp &op_info, - const palf::SCN &commit_version, - const palf::SCN &commit_scn, + const share::SCN &commit_version, + const share::SCN &commit_scn, const ObTableLockOpStatus status); int get_table_lock_store_info(ObIArray &store_arr); @@ -118,8 +118,8 @@ public: bool is_active_memtable() const override; // =========== INHERITED FROM ObCommonCheckPoint ========== - virtual palf::SCN get_rec_scn(); - virtual int flush(palf::SCN recycle_scn, bool need_freeze = true); + virtual share::SCN get_rec_scn(); + virtual int flush(share::SCN recycle_scn, bool need_freeze = true); virtual ObTabletID get_tablet_id() const; @@ -132,7 +132,7 @@ public: // used by the replay process of multi data source. int replay_lock(memtable::ObMemtableCtx *mem_ctx, const ObTableLockOp &lock_op, - const palf::SCN &scn); + const share::SCN &scn); // ================ NOT SUPPORTED INTERFACE =============== @@ -179,7 +179,7 @@ public: virtual int get_frozen_schema_version(int64_t &schema_version) const override; - void set_flushed_scn(const palf::SCN &flushed_scn) { flushed_scn_ = flushed_scn; } + void set_flushed_scn(const share::SCN &flushed_scn) { flushed_scn_ = flushed_scn; } INHERIT_TO_STRING_KV("ObITable", ObITable, KP(this), K_(snapshot_version), K_(ls_id)); private: @@ -199,11 +199,11 @@ private: const int64_t expired_time = 0); int check_lock_need_replay_(memtable::ObMemtableCtx *mem_ctx, const ObTableLockOp &lock_op, - const palf::SCN &scn, + const share::SCN &scn, bool &need_replay); int replay_lock_(memtable::ObMemtableCtx *mem_ctx, const ObTableLockOp &lock_op, - const palf::SCN &scn); + const share::SCN &scn); int post_obj_lock_conflict_(memtable::ObMvccAccessCtx &acc_ctx, const ObLockID &lock_id, const ObTransID &conflict_tx_id, @@ -219,11 +219,11 @@ private: share::ObLSID ls_id_; // the lock map store lock data ObOBJLockMap obj_lock_map_; - palf::SCN freeze_scn_; + share::SCN freeze_scn_; // data before the flushed_scn_ have been flushed - palf::SCN flushed_scn_; - palf::SCN rec_scn_; - palf::SCN max_committed_scn_; + share::SCN flushed_scn_; + share::SCN rec_scn_; + share::SCN max_committed_scn_; bool is_frozen_; storage::ObFreezer *freezer_; diff --git a/src/storage/tablelock/ob_lock_memtable_mgr.cpp b/src/storage/tablelock/ob_lock_memtable_mgr.cpp index 2052d9562..02beaadea 100644 --- a/src/storage/tablelock/ob_lock_memtable_mgr.cpp +++ b/src/storage/tablelock/ob_lock_memtable_mgr.cpp @@ -71,7 +71,7 @@ void ObLockMemtableMgr::reset() is_inited_ = false; } -int ObLockMemtableMgr::create_memtable(const palf::SCN clog_checkpoint_scn, +int ObLockMemtableMgr::create_memtable(const SCN clog_checkpoint_scn, const int64_t schema_version, const bool for_replay) { @@ -91,8 +91,8 @@ int ObLockMemtableMgr::create_memtable(const palf::SCN clog_checkpoint_scn, table_key.table_type_ = ObITable::LOCK_MEMTABLE; table_key.tablet_id_ = LS_LOCK_TABLET; - table_key.scn_range_.start_scn_ = palf::SCN::base_scn();//fake - table_key.scn_range_.end_scn_ = palf::SCN::plus(table_key.scn_range_.start_scn_, 1);//fake + table_key.scn_range_.start_scn_ = SCN::base_scn();//fake + table_key.scn_range_.end_scn_ = SCN::plus(table_key.scn_range_.start_scn_, 1);//fake if (get_memtable_count_() > 0) { ret = OB_ERR_UNEXPECTED; diff --git a/src/storage/tablelock/ob_lock_memtable_mgr.h b/src/storage/tablelock/ob_lock_memtable_mgr.h index 27fa8dd48..c9f7e4aef 100644 --- a/src/storage/tablelock/ob_lock_memtable_mgr.h +++ b/src/storage/tablelock/ob_lock_memtable_mgr.h @@ -61,7 +61,7 @@ public: storage::ObTenantMetaMemMgr *t3m) override; virtual void destroy() override; - virtual int create_memtable(const palf::SCN clog_checkpoint_scn, + virtual int create_memtable(const share::SCN clog_checkpoint_scn, const int64_t schema_version, const bool for_replay = false) override; diff --git a/src/storage/tablelock/ob_lock_table.cpp b/src/storage/tablelock/ob_lock_table.cpp index fc9b8cc2d..9c2ca2b9a 100644 --- a/src/storage/tablelock/ob_lock_table.cpp +++ b/src/storage/tablelock/ob_lock_table.cpp @@ -239,7 +239,7 @@ int ObLockTable::gen_create_tablet_arg_( false/*is_create_bind_hidden_tablets*/))) { LOG_WARN("create tablet info init failed", K(ret), K(tablet_ids), K(tablet_id)); // create ObBatchCreateTabletArg - } else if (OB_FAIL(arg.init_create_tablet(ls_id, palf::SCN::base_scn()))) { + } else if (OB_FAIL(arg.init_create_tablet(ls_id, SCN::base_scn()))) { LOG_WARN("ObBatchCreateTabletArg init create tablet failed", K(ret), K(tenant_id), K(ls_id)); } else if (OB_FAIL(arg.table_schemas_.push_back(table_schema))) { LOG_WARN("add table schema failed", K(ret), K(table_schema)); @@ -351,7 +351,7 @@ int ObLockTable::online() return ret; } -int ObLockTable::create_tablet(const lib::Worker::CompatMode compat_mode, const palf::SCN &create_scn) +int ObLockTable::create_tablet(const lib::Worker::CompatMode compat_mode, const SCN &create_scn) { int ret = OB_SUCCESS; uint64_t tenant_id = parent_->get_tenant_id(); diff --git a/src/storage/tablelock/ob_lock_table.h b/src/storage/tablelock/ob_lock_table.h index 0c2991086..f9fae0d41 100644 --- a/src/storage/tablelock/ob_lock_table.h +++ b/src/storage/tablelock/ob_lock_table.h @@ -76,7 +76,7 @@ public: int offline(); int online(); // create lock table tablet. - int create_tablet(const lib::Worker::CompatMode compat_mode, const palf::SCN &create_scn); + int create_tablet(const lib::Worker::CompatMode compat_mode, const share::SCN &create_scn); // remove lock table tablet. int remove_tablet(); // load lock for tablet. diff --git a/src/storage/tablelock/ob_mem_ctx_table_lock.cpp b/src/storage/tablelock/ob_mem_ctx_table_lock.cpp index 6ecdb4f95..98e937c0c 100644 --- a/src/storage/tablelock/ob_mem_ctx_table_lock.cpp +++ b/src/storage/tablelock/ob_mem_ctx_table_lock.cpp @@ -20,6 +20,7 @@ namespace oceanbase { using namespace common; +using namespace share; using namespace memtable; using namespace storage; namespace transaction @@ -113,7 +114,7 @@ void ObLockMemCtx::abort_table_lock_() } } -int ObLockMemCtx::commit_table_lock_(const palf::SCN &commit_version, const palf::SCN &commit_scn) +int ObLockMemCtx::commit_table_lock_(const SCN &commit_version, const SCN &commit_scn) { int ret = OB_SUCCESS; ObLockMemtable *memtable = nullptr; @@ -188,8 +189,8 @@ int ObLockMemCtx::get_table_lock_store_info(ObTableLockInfo &table_lock_info) int ObLockMemCtx::clear_table_lock( const bool is_committed, - const palf::SCN &commit_version, - const palf::SCN &commit_scn) + const SCN &commit_version, + const SCN &commit_scn) { int ret = OB_SUCCESS; if (lock_list_.is_empty()) { @@ -284,7 +285,7 @@ void ObLockMemCtx::remove_lock_record( void ObLockMemCtx::set_log_synced( ObMemCtxLockOpLinkNode *lock_op, - const palf::SCN &scn) + const SCN &scn) { if (OB_ISNULL(lock_op)) { LOG_WARN("invalid argument.", K(lock_op)); @@ -394,7 +395,7 @@ int ObLockMemCtx::iterate_tx_obj_lock_op(ObLockOpIterator &iter) const } int ObLockMemCtx::check_lock_need_replay( - const palf::SCN &scn, + const SCN &scn, const ObTableLockOp &lock_op, bool &need_replay) { diff --git a/src/storage/tablelock/ob_mem_ctx_table_lock.h b/src/storage/tablelock/ob_mem_ctx_table_lock.h index 7d7d18620..b08bd19b7 100644 --- a/src/storage/tablelock/ob_mem_ctx_table_lock.h +++ b/src/storage/tablelock/ob_mem_ctx_table_lock.h @@ -13,7 +13,7 @@ #ifndef OCEANBASE_STORAGE_TABLELOCK_OB_MEM_CTX_TABLE_LOCK_ #define OCEANBASE_STORAGE_TABLELOCK_OB_MEM_CTX_TABLE_LOCK_ -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/memtable/mvcc/ob_mvcc_ctx.h" #include "storage/tablelock/ob_table_lock_common.h" #include "storage/tablelock/ob_table_lock_callback.h" @@ -68,7 +68,7 @@ public: int init(storage::ObTableHandleV2 &handle); int get_lock_memtable(ObLockMemtable *&memtable); void reset(); - void set_log_synced(ObMemCtxLockOpLinkNode *lock_op, const palf::SCN &scn); + void set_log_synced(ObMemCtxLockOpLinkNode *lock_op, const share::SCN &scn); int add_lock_record( const ObTableLockOp &lock_op, @@ -79,7 +79,7 @@ public: void remove_lock_record( ObMemCtxLockOpLinkNode *lock_op); int check_lock_need_replay( - const palf::SCN &scn, + const share::SCN &scn, const ObTableLockOp &lock_op, bool &need_replay); int check_lock_exist( @@ -97,8 +97,8 @@ public: int iterate_tx_obj_lock_op(ObLockOpIterator &iter) const; int clear_table_lock( const bool is_committed, - const palf::SCN &commit_version, - const palf::SCN &commit_scn); + const share::SCN &commit_version, + const share::SCN &commit_scn); int rollback_table_lock(const int64_t seq_no); void *alloc_lock_op_callback(); void free_lock_op_callback(void *cb); @@ -115,7 +115,7 @@ private: void free_lock_op_(void *op); void print() const; void rollback_table_lock_(const int64_t seq_no); - int commit_table_lock_(const palf::SCN &commit_version, const palf::SCN &commit_scn); + int commit_table_lock_(const share::SCN &commit_version, const share::SCN &commit_scn); void abort_table_lock_(); private: // for performance. @@ -128,7 +128,7 @@ private: // the flag of whether the tx is killed. // used by deadlock detector. bool is_killed_; - palf::SCN max_durable_scn_; + share::SCN max_durable_scn_; // the lock memtable pointer point to LS lock table's memtable. storage::ObTableHandleV2 memtable_handle_; }; diff --git a/src/storage/tablelock/ob_obj_lock.cpp b/src/storage/tablelock/ob_obj_lock.cpp index 0e57bd7c4..5476c8dbe 100644 --- a/src/storage/tablelock/ob_obj_lock.cpp +++ b/src/storage/tablelock/ob_obj_lock.cpp @@ -320,8 +320,8 @@ int ObOBJLock::recover_lock( int ObOBJLock::update_lock_status_( const ObTableLockOp &lock_op, - const palf::SCN &commit_version, - const palf::SCN &commit_scn, + const SCN &commit_version, + const SCN &commit_scn, const ObTableLockOpStatus status, ObTableLockOpList *op_list) { @@ -346,8 +346,8 @@ int ObOBJLock::update_lock_status_( } int ObOBJLock::update_lock_status(const ObTableLockOp &lock_op, - const palf::SCN commit_version, - const palf::SCN commit_scn, + const SCN commit_version, + const SCN commit_scn, const ObTableLockOpStatus status, ObMalloc &allocator) { @@ -594,10 +594,10 @@ void ObOBJLock::remove_lock_op( LOG_DEBUG("ObOBJLock::remove_lock_op finish."); } -palf::SCN ObOBJLock::get_min_ddl_lock_committed_scn(const palf::SCN &flushed_scn) const +SCN ObOBJLock::get_min_ddl_lock_committed_scn(const SCN &flushed_scn) const { int ret = OB_SUCCESS; - palf::SCN min_rec_scn = SCN::max_scn(); + SCN min_rec_scn = SCN::max_scn(); RDLockGuard guard(rwlock_); for (int i = 0; i < TABLE_LOCK_MODE_COUNT; i++) { ObTableLockOpList *op_list = map_[i]; @@ -617,7 +617,7 @@ palf::SCN ObOBJLock::get_min_ddl_lock_committed_scn(const palf::SCN &flushed_scn int ObOBJLock::get_table_lock_store_info( ObIArray &store_arr, - const palf::SCN &freeze_scn) + const SCN &freeze_scn) { int ret = OB_SUCCESS; RDLockGuard guard(rwlock_); @@ -1679,7 +1679,7 @@ void ObOBJLockMap::print() lock_map_.for_each(fn); } -palf::SCN ObOBJLockMap::get_min_ddl_committed_scn(palf::SCN &flushed_scn) +SCN ObOBJLockMap::get_min_ddl_committed_scn(SCN &flushed_scn) { int ret = OB_SUCCESS; SCN min_ddl_committed_scn = SCN::max_scn(); @@ -1958,8 +1958,8 @@ int ObOBJLockMap::recover_obj_lock(const ObTableLockOp &lock_op) } int ObOBJLockMap::update_lock_status(const ObTableLockOp &lock_op, - const palf::SCN commit_version, - const palf::SCN commit_scn, + const SCN commit_version, + const SCN commit_scn, const ObTableLockOpStatus status) { int ret = OB_SUCCESS; diff --git a/src/storage/tablelock/ob_obj_lock.h b/src/storage/tablelock/ob_obj_lock.h index 1a843c421..b2a709ce3 100644 --- a/src/storage/tablelock/ob_obj_lock.h +++ b/src/storage/tablelock/ob_obj_lock.h @@ -19,7 +19,7 @@ #include "lib/hash/ob_iteratable_hashset.h" #include "lib/hash/ob_link_hashmap.h" #include "lib/hash/ob_hashmap.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/tablelock/ob_table_lock_common.h" #include "storage/ob_i_store.h" @@ -107,8 +107,8 @@ public: // only update the status of the exact one with the same lock_op int update_lock_status( const ObTableLockOp &lock_op, - const palf::SCN commit_version, - const palf::SCN commit_scn, + const share::SCN commit_version, + const share::SCN commit_scn, const ObTableLockOpStatus status, ObMalloc &allocator); int check_allow_lock( @@ -118,10 +118,10 @@ public: bool &conflict_with_dml_lock, const bool include_finish_tx = true, const bool only_check_dml_lock = false); - palf::SCN get_min_ddl_lock_committed_scn(const palf::SCN &flushed_scn) const; + share::SCN get_min_ddl_lock_committed_scn(const share::SCN &flushed_scn) const; int get_table_lock_store_info( ObIArray &store_arr, - const palf::SCN &freeze_scn); + const share::SCN &freeze_scn); void reset(ObMalloc &allocator); void reset_without_lock(ObMalloc &allocator); @@ -151,8 +151,8 @@ private: const bool only_check_dml_lock = false); int update_lock_status_( const ObTableLockOp &lock_op, - const palf::SCN &commit_version, - const palf::SCN &commit_scn, + const share::SCN &commit_version, + const share::SCN &commit_scn, const ObTableLockOpStatus status, ObTableLockOpList *op_list); int recover_( @@ -330,8 +330,8 @@ public: const ObTableLockOp &lock_op); int update_lock_status( const ObTableLockOp &lock_op, - const palf::SCN commit_version, - const palf::SCN commit_scn, + const share::SCN commit_version, + const share::SCN commit_scn, const ObTableLockOpStatus status); bool is_inited() const { return is_inited_; } int check_allow_lock( @@ -341,8 +341,8 @@ public: const bool include_finish_tx = true, const bool only_check_dml_lock = false); void print(); - palf::SCN get_min_ddl_committed_scn(palf::SCN &flushed_scn); - int get_table_lock_store_info(ObIArray &store_arr, palf::SCN freeze_scn); + share::SCN get_min_ddl_committed_scn(share::SCN &flushed_scn); + int get_table_lock_store_info(ObIArray &store_arr, share::SCN freeze_scn); // get all the lock id in the lock map // @param[out] iter, the iterator returned. // int get_lock_id_iter(ObLockIDIterator &iter); @@ -407,28 +407,28 @@ private: class GetMinCommittedDDLLogtsFunctor { public: - explicit GetMinCommittedDDLLogtsFunctor(palf::SCN &flushed_scn) - : min_committed_scn_(palf::SCN::max_scn()), + explicit GetMinCommittedDDLLogtsFunctor(share::SCN &flushed_scn) + : min_committed_scn_(share::SCN::max_scn()), flushed_scn_(flushed_scn) {} bool operator()(const ObLockID &lock_id, ObOBJLock *obj_lock); - palf::SCN get_min_committed_scn() { return min_committed_scn_; } + share::SCN get_min_committed_scn() { return min_committed_scn_; } private: - palf::SCN min_committed_scn_; - palf::SCN flushed_scn_; + share::SCN min_committed_scn_; + share::SCN flushed_scn_; }; class GetTableLockStoreInfoFunctor { public: explicit GetTableLockStoreInfoFunctor(ObIArray &store_arr, - palf::SCN freeze_scn) + share::SCN freeze_scn) : store_arr_(store_arr), freeze_scn_(freeze_scn) {} bool operator()(const ObLockID &lock_id, ObOBJLock *obj_lock); private: ObIArray &store_arr_; - palf::SCN freeze_scn_; + share::SCN freeze_scn_; }; private: diff --git a/src/storage/tablelock/ob_table_lock_callback.cpp b/src/storage/tablelock/ob_table_lock_callback.cpp index 28bc97da1..b2a16c385 100644 --- a/src/storage/tablelock/ob_table_lock_callback.cpp +++ b/src/storage/tablelock/ob_table_lock_callback.cpp @@ -22,6 +22,7 @@ namespace oceanbase { using namespace memtable; +using namespace share; using namespace storage; namespace transaction { @@ -39,11 +40,11 @@ memtable::ObIMemtable* ObOBJLockCallback::get_memtable() const return memtable_; } -int ObOBJLockCallback::log_sync(const palf::SCN scn) +int ObOBJLockCallback::log_sync(const SCN scn) { int ret = OB_SUCCESS; ObMemtableCtx *mem_ctx = static_cast(ctx_); - if (OB_UNLIKELY(palf::SCN::max_scn() == scn)) { + if (OB_UNLIKELY(SCN::max_scn() == scn)) { ret = OB_INVALID_ARGUMENT; LOG_ERROR("log ts should not be invalid", K(ret), K(scn), K(*this)); } else if (OB_ISNULL(mem_ctx) || OB_ISNULL(lock_op_)) { diff --git a/src/storage/tablelock/ob_table_lock_callback.h b/src/storage/tablelock/ob_table_lock_callback.h index 7df2b674f..426e6b682 100644 --- a/src/storage/tablelock/ob_table_lock_callback.h +++ b/src/storage/tablelock/ob_table_lock_callback.h @@ -13,7 +13,7 @@ #ifndef OCEANBASE_STORAGE_TABLELOCK_OB_TABLE_LOCK_CALLBACK_ #define OCEANBASE_STORAGE_TABLELOCK_OB_TABLE_LOCK_CALLBACK_ -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/tx/ob_trans_define.h" #include "storage/memtable/mvcc/ob_mvcc_trans_ctx.h" #include "storage/memtable/ob_memtable_mutator.h" @@ -63,9 +63,9 @@ public: virtual int del() override; int64_t get_seq_no() const override; bool is_table_lock_callback() const override { return true; } - bool log_synced() const override { return palf::SCN::max_scn() != scn_; } + bool log_synced() const override { return share::SCN::max_scn() != scn_; } bool must_log() const; - int log_sync(const palf::SCN scn) override; + int log_sync(const share::SCN scn) override; int64_t get_data_size() override { return 0; } // size of trans node. memtable::MutatorType get_mutator_type() const override { return memtable::MutatorType::MUTATOR_TABLE_LOCK; } diff --git a/src/storage/tablelock/ob_table_lock_common.h b/src/storage/tablelock/ob_table_lock_common.h index f96e78432..ebf9183f2 100644 --- a/src/storage/tablelock/ob_table_lock_common.h +++ b/src/storage/tablelock/ob_table_lock_common.h @@ -16,7 +16,7 @@ #include "lib/list/ob_dlist.h" #include "lib/utility/ob_print_utils.h" #include "lib/allocator/ob_mod_define.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/tx/ob_trans_define.h" namespace oceanbase @@ -391,8 +391,8 @@ public: ObTableLockOpType op_type_; ObTableLockOpStatus lock_op_status_; int64_t lock_seq_no_; - palf::SCN commit_version_; - palf::SCN commit_scn_; + share::SCN commit_version_; + share::SCN commit_scn_; // used to check whether a trans modify before a schema_version or timestamp. int64_t create_timestamp_; int64_t create_schema_version_; @@ -407,7 +407,7 @@ public: void reset(); TO_STRING_KV(K_(table_lock_ops), K_(max_durable_scn)); ObTableLockOpArray table_lock_ops_; - palf::SCN max_durable_scn_; + share::SCN max_durable_scn_; }; static inline diff --git a/src/storage/tablet/ob_table_store_util.cpp b/src/storage/tablet/ob_table_store_util.cpp index 0f064c9f7..5a30fff0f 100644 --- a/src/storage/tablet/ob_table_store_util.cpp +++ b/src/storage/tablet/ob_table_store_util.cpp @@ -23,10 +23,11 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tablet/ob_tablet_table_store.h" #include "observer/ob_server_struct.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" using namespace oceanbase; +using namespace share; using namespace common; using namespace storage; using namespace blocksstable; @@ -587,7 +588,7 @@ int ObMemtableArray::rebuild(common::ObIArray &handle_array) LOG_ERROR("ObMemtableArray not inited", K(ret), KPC(this), K(handle_array)); } else { ObITable *last_memtable = get_table(count() - 1); - palf::SCN end_scn = (NULL == last_memtable) ? palf::SCN::min_scn() : last_memtable->get_end_scn(); + SCN end_scn = (NULL == last_memtable) ? SCN::min_scn() : last_memtable->get_end_scn(); for (int64_t i = 0; OB_SUCC(ret) && i < handle_array.count(); ++i) { memtable::ObMemtable *memtable = nullptr; @@ -655,7 +656,7 @@ int ObMemtableArray::find( } int ObMemtableArray::find( - const palf::SCN &start_scn, + const SCN &start_scn, const int64_t base_version, ObITable *&table, int64_t &mem_pos) const @@ -670,7 +671,7 @@ int ObMemtableArray::find( } else if (OB_UNLIKELY(!start_scn.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid arguments", K(ret), K(start_scn), K(base_version)); - } else if (palf::SCN::min_scn() == start_scn) { + } else if (SCN::min_scn() == start_scn) { mem_pos = 0; table = get_table(0); } else { diff --git a/src/storage/tablet/ob_table_store_util.h b/src/storage/tablet/ob_table_store_util.h index 36ca1adad..4b5776cff 100644 --- a/src/storage/tablet/ob_table_store_util.h +++ b/src/storage/tablet/ob_table_store_util.h @@ -18,7 +18,7 @@ namespace oceanbase { -namespace palf +namespace share { class SCN; } @@ -114,7 +114,7 @@ public: int rebuild(common::ObIArray &handle_array); int prepare_allocate(); int find(const ObITable::TableKey &table_key, ObTableHandleV2 &handle) const; - int find(const palf::SCN &start_scn, const int64_t base_version, ObITable *&table, int64_t &mem_pos) const; + int find(const share::SCN &start_scn, const int64_t base_version, ObITable *&table, int64_t &mem_pos) const; TO_STRING_KV(K_(is_inited), KPC_(array)); private: int add_table(ObITable * const table); diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index ce242464a..624851a1e 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -240,7 +240,7 @@ int ObTablet::init( set_next_tablet_guard(old_tablet.next_tablet_guard_); } is_inited_ = true; - LOG_INFO("succeeded to init tablet", K(ret), K(param), K(old_tablet), K(tx_data), K(autoinc_seq)); + LOG_INFO("succeeded to init tablet", K(ret), K(param), K(old_tablet), K(tx_data), K(ddl_data), K(autoinc_seq), KPC(this)); } if (OB_SUCC(ret) && param.need_report_ && param.table_handle_.get_table()->is_major_sstable()) { @@ -258,7 +258,6 @@ int ObTablet::init( reset(); } - LOG_INFO("Update tablet info", K(ret), K(param), K(old_tablet), KPC(this)); return ret; } @@ -1500,7 +1499,7 @@ int ObTablet::assign_pointer_handle(const ObTabletPointerHandle &ptr_hdl) } int ObTablet::replay_update_storage_schema( - const palf::SCN &scn, + const SCN &scn, const char *buf, const int64_t buf_size, int64_t &pos) @@ -1622,7 +1621,7 @@ int ObTablet::inner_create_memtable( return ret; } -int ObTablet::release_memtables(const palf::SCN scn) +int ObTablet::release_memtables(const SCN scn) { int ret = OB_SUCCESS; ObIMemtableMgr *memtable_mgr = nullptr; @@ -1802,7 +1801,7 @@ int ObTablet::try_update_start_scn() { int ret = OB_SUCCESS; ObSSTable *first_minor = static_cast(table_store_.get_minor_sstables().get_boundary_table(false /*first*/)); - const palf::SCN &start_scn = OB_NOT_NULL(first_minor) ? first_minor->get_start_scn() : tablet_meta_.clog_checkpoint_scn_; + const SCN &start_scn = OB_NOT_NULL(first_minor) ? first_minor->get_start_scn() : tablet_meta_.clog_checkpoint_scn_; if (OB_UNLIKELY(start_scn < tablet_meta_.start_scn_)) { // ignore ret on purpose LOG_WARN("tablet start scn can not fallback", K(start_scn), K(tablet_meta_)); @@ -1817,7 +1816,7 @@ int ObTablet::try_update_ddl_checkpoint_scn() int ret = OB_SUCCESS; ObSSTable *last_ddl_sstable = static_cast(table_store_.get_ddl_sstables().get_boundary_table(true/*last*/)); if (OB_NOT_NULL(last_ddl_sstable)) { - const palf::SCN &ddl_checkpoint_scn = last_ddl_sstable->get_end_scn(); + const SCN &ddl_checkpoint_scn = last_ddl_sstable->get_end_scn(); if (OB_UNLIKELY(ddl_checkpoint_scn < tablet_meta_.ddl_checkpoint_scn_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected clog checkpoint ts", K(ret), K(ddl_checkpoint_scn), K(tablet_meta_)); @@ -2162,64 +2161,6 @@ int ObTablet::update_tablet_autoinc_seq( return ret; } -int ObTablet::get_active_ddl_kv(ObDDLKVHandle &ddl_kvs_handle) -{ - int ret = OB_NOT_SUPPORTED; - UNUSED(ddl_kvs_handle); - return ret; -} - -int ObTablet::get_or_create_active_ddl_kv(ObDDLKVHandle &ddl_kvs_handle) -{ - int ret = OB_NOT_SUPPORTED; - UNUSED(ddl_kvs_handle); - return ret; -} - -int ObTablet::check_has_effective_ddl_kv(bool &has_ddl_kv) -{ - int ret = OB_SUCCESS; - ObDDLKvMgrHandle ddl_kv_mgr_handle; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else if (OB_FAIL(get_ddl_kv_mgr(ddl_kv_mgr_handle))) { - if (OB_ENTRY_NOT_EXIST != ret) { - LOG_WARN("failed to get ddl kv mgr", K(ret)); - } else { - has_ddl_kv = false; - ret = OB_SUCCESS; - } - } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->check_has_effective_ddl_kv(has_ddl_kv))) { - LOG_WARN("fail to check has effective ddl kv", K(ret)); - } - - return ret; -} - -int ObTablet::get_ddl_kv_min_scn(palf::SCN &min_scn) -{ - int ret = OB_SUCCESS; - ObDDLKvMgrHandle ddl_kv_mgr_handle; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else if (OB_FAIL(get_ddl_kv_mgr(ddl_kv_mgr_handle))) { - if (OB_ENTRY_NOT_EXIST != ret) { - LOG_WARN("failed to get ddl kv mgr", K(ret)); - } else { - min_scn = palf::SCN::max_scn(); - ret = OB_SUCCESS; - } - } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_ddl_kv_min_scn(min_scn))) { - LOG_WARN("fail to get ddl kv min log ts", K(ret)); - } - - return ret; -} - int ObTablet::start_ddl_if_need() { int ret = OB_SUCCESS; @@ -2531,7 +2472,7 @@ int ObTablet::get_ddl_info(int64_t &schema_version, int64_t &schema_refreshed_ts // only for redo int ObTablet::set_tx_scn( const transaction::ObTransID &tx_id, - const palf::SCN &scn, + const SCN &scn, const bool for_replay) { int ret = OB_SUCCESS; @@ -2560,7 +2501,7 @@ int ObTablet::set_tx_scn( // only for commit, abort int ObTablet::set_tablet_final_status( ObTabletTxMultiSourceDataUnit &tx_data, - const palf::SCN &memtable_scn, + const SCN &memtable_scn, const bool for_replay, const memtable::MemtableRefOp ref_op) { @@ -2900,6 +2841,9 @@ int ObTablet::check_max_sync_schema_version() const if (OB_FAIL(get_memtable_mgr(memtable_mgr))) { LOG_WARN("failed to get memtable mgr", K(ret)); } else if (FALSE_IT(data_memtable_mgr = static_cast(memtable_mgr))) { + } else if (OB_UNLIKELY(!data_memtable_mgr->get_storage_schema_recorder().is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("schema recorder is invalid", K(ret), K_(tablet_meta), KPC(data_memtable_mgr)); } else if (OB_FAIL(data_memtable_mgr->get_multi_source_data_unit(&storage_schema, &tmp_allocator))) { LOG_ERROR("failed to storage schema from memtable, max_sync_schema_version is invalid", K(ret), K(max_sync_schema_version), KPC(data_memtable_mgr)); diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index 728060e5c..ecdde89e9 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -28,7 +28,7 @@ #include "storage/tablet/ob_tablet_table_store.h" #include "storage/tablet/ob_tablet_table_store_flag.h" #include "storage/tx/ob_trans_define.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/meta_mem/ob_tablet_pointer.h" namespace oceanbase @@ -64,11 +64,6 @@ namespace blocksstable { class ObSSTable; } -namespace palf -{ -class SCN; -} - namespace transaction { class ObTransID; @@ -116,7 +111,7 @@ public: int64_t dec_ref(); int64_t get_ref() const { return ATOMIC_LOAD(&ref_cnt_); } int64_t get_wash_score() const { return ATOMIC_LOAD(&wash_score_); } - int get_rec_log_scn(palf::SCN &rec_scn); + int get_rec_log_scn(share::SCN &rec_scn); public: // first time create tablet int init( @@ -125,7 +120,7 @@ public: const common::ObTabletID &data_tablet_id, const common::ObTabletID &lob_meta_tablet_id, const common::ObTabletID &lob_piece_tablet_id, - const palf::SCN &create_scn, + const share::SCN &create_scn, const int64_t snapshot_version, const share::schema::ObTableSchema &table_schema, const lib::Worker::CompatMode compat_mode, @@ -219,7 +214,7 @@ public: ObIMemtableMgr *get_memtable_mgr() const { return memtable_mgr_; } // TODO(bowen.gbw): get memtable mgr from tablet pointer handle // get the active memtable for write or replay. int get_active_memtable(ObTableHandleV2 &handle) const; - int release_memtables(const palf::SCN scn); + int release_memtables(const share::SCN scn); // force release all memtables // just for rebuild or migrate retry. int release_memtables(); @@ -236,12 +231,12 @@ public: template int back_fill_scn_for_commit(T &multi_source_data_unit); template - int set_multi_data_for_commit(T &multi_source_data_unit, const palf::SCN &log_scn, const bool for_replay, const memtable::MemtableRefOp ref_op); + int set_multi_data_for_commit(T &multi_source_data_unit, const share::SCN &log_scn, const bool for_replay, const memtable::MemtableRefOp ref_op); template int save_multi_source_data_unit( const T *const msd, - const palf::SCN &memtable_scn, + const share::SCN &memtable_scn, const bool for_replay, const memtable::MemtableRefOp ref_op = memtable::MemtableRefOp::NONE, const bool is_callback = false); @@ -285,11 +280,6 @@ public: // check whether we have dumped a sstable or not. int check_has_sstable(bool &has_sstable) const; - // ddl kv - int get_active_ddl_kv(ObDDLKVHandle &ddl_kvs_handle); - int get_or_create_active_ddl_kv(ObDDLKVHandle &ddl_kvs_handle); - int check_has_effective_ddl_kv(bool &has_ddl_kv); - int get_ddl_kv_min_scn(palf::SCN &min_scn); int get_ddl_kv_mgr(ObDDLKvMgrHandle &ddl_kv_mgr_handle, bool try_create = false); void remove_ddl_kv_mgr(); int start_ddl_if_need(); @@ -299,11 +289,11 @@ public: // other const ObTabletMeta &get_tablet_meta() const { return tablet_meta_; } const ObTabletTableStore &get_table_store() const { return table_store_; } - palf::SCN get_clog_checkpoint_scn() const { return tablet_meta_.clog_checkpoint_scn_; } + share::SCN get_clog_checkpoint_scn() const { return tablet_meta_.clog_checkpoint_scn_; } int64_t get_snapshot_version() const { return tablet_meta_.snapshot_version_; } int64_t get_multi_version_start() const { return tablet_meta_.multi_version_start_; } - int get_multi_version_start(palf::SCN &scn) const; - int get_snapshot_version(palf::SCN &scn) const; + int get_multi_version_start(share::SCN &scn) const; + int get_snapshot_version(share::SCN &scn) const; // deprecated later, DO NOT use it! ObTabletTableStore &get_table_store() { return table_store_; } @@ -319,7 +309,7 @@ public: int assign_pointer_handle(const ObTabletPointerHandle &ptr_hdl); int replay_update_storage_schema( - const palf::SCN &scn, + const share::SCN &scn, const char *buf, const int64_t buf_size, int64_t &pos); @@ -331,7 +321,7 @@ public: int get_latest_autoinc_seq(share::ObTabletAutoincSeq &autoinc_seq) const; int update_tablet_autoinc_seq( const uint64_t autoinc_seq, - const palf::SCN &replay_scn); + const share::SCN &replay_scn); int get_kept_multi_version_start( int64_t &multi_version_start, @@ -350,7 +340,7 @@ public: const bool need_checksums = true); int set_tx_data( const ObTabletTxMultiSourceDataUnit &tx_data, - const palf::SCN &memtable_log_scn, + const share::SCN &memtable_log_scn, const bool for_replay, const bool update_cache, const memtable::MemtableRefOp ref_op = memtable::MemtableRefOp::NONE, @@ -410,20 +400,20 @@ private: // used for freeze_tablet int inner_create_memtable( - const palf::SCN clog_checkpoint_scn = palf::SCN::base_scn(),/*1 for first memtable, filled later*/ + const share::SCN clog_checkpoint_scn = share::SCN::base_scn(),/*1 for first memtable, filled later*/ const int64_t schema_version = 0/*0 for first memtable*/, const bool for_replay=false); int write_sync_tablet_seq_log(share::ObTabletAutoincSeq &autoinc_seq, const uint64_t new_autoinc_seq, - palf::SCN &scn); + share::SCN &scn); int update_ddl_info( const int64_t schema_version, - const palf::SCN &scn, + const share::SCN &scn, int64_t &schema_refreshed_ts); int write_tablet_schema_version_change_clog( const int64_t schema_version, - palf::SCN &scn); + share::SCN &scn); int get_ddl_info( int64_t &refreshed_schema_version, int64_t &refreshed_schema_ts) const; @@ -443,11 +433,11 @@ private: const bool for_replay); int set_tx_scn( const transaction::ObTransID &tx_id, - const palf::SCN &scn, + const share::SCN &scn, const bool for_replay); int set_tablet_final_status( ObTabletTxMultiSourceDataUnit &tx_data, - const palf::SCN &memtable_scn, + const share::SCN &memtable_scn, const bool for_replay, const memtable::MemtableRefOp ref_op); int set_tx_data( @@ -594,7 +584,7 @@ int ObTablet::prepare_data(T &multi_source_data_unit, const transaction::ObMulSo { int ret = OB_SUCCESS; - const palf::SCN scn = trans_flags.for_replay_ ? trans_flags.scn_ : palf::SCN::max_scn(); + const share::SCN scn = trans_flags.for_replay_ ? trans_flags.scn_ : share::SCN::max_scn(); TRANS_LOG(INFO, "prepare data when tx_end", K(multi_source_data_unit), K(tablet_meta_.tablet_id_)); @@ -617,14 +607,14 @@ int ObTablet::prepare_data(T &multi_source_data_unit, const transaction::ObMulSo template int ObTablet::set_multi_data_for_commit( T &multi_source_data_unit, - const palf::SCN &log_scn, + const share::SCN &log_scn, const bool for_replay, const memtable::MemtableRefOp ref_op) { int ret = OB_SUCCESS; bool is_callback = true; - TRANS_LOG(INFO, "set_multi_data_for_commit", K(multi_source_data_unit)); + TRANS_LOG(INFO, "set_multi_data_for_commit", K(multi_source_data_unit), K(ref_op)); if (IS_NOT_INIT) { ret = OB_NOT_INIT; TRANS_LOG(WARN, "not inited", K(ret), K_(is_inited)); @@ -647,7 +637,7 @@ int ObTablet::back_fill_scn_for_commit(T &multi_source_data_unit) { int ret = OB_SUCCESS; - const palf::SCN scn = palf::SCN::max_scn(); + const share::SCN scn = share::SCN::max_scn(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; TRANS_LOG(WARN, "not inited", K(ret), K_(is_inited)); @@ -668,7 +658,7 @@ int ObTablet::back_fill_scn_for_commit(T &multi_source_data_unit) template int ObTablet::save_multi_source_data_unit( const T *const msd, - const palf::SCN &memtable_scn, + const share::SCN &memtable_scn, const bool for_replay, const memtable::MemtableRefOp ref_op, const bool is_callback) @@ -723,7 +713,7 @@ int ObTablet::save_multi_source_data_unit( const int64_t start = ObTimeUtility::current_time(); while (OB_SUCC(ret) && memtable->get_logging_blocked() && - palf::SCN::max_scn() == memtable_scn) { + share::SCN::max_scn() == memtable_scn) { if (ObTimeUtility::current_time() - start > 100 * 1000) { ret = OB_BLOCK_FROZEN; TRANS_LOG(WARN, "logging_block costs too much time", K(ret), K(ls_id), K(tablet_id), K(memtable_scn), K(ref_op), K(for_replay)); diff --git a/src/storage/tablet/ob_tablet_binding_helper.cpp b/src/storage/tablet/ob_tablet_binding_helper.cpp index 88578b485..187700767 100644 --- a/src/storage/tablet/ob_tablet_binding_helper.cpp +++ b/src/storage/tablet/ob_tablet_binding_helper.cpp @@ -40,21 +40,40 @@ namespace storage { ObTabletBindingInfo::ObTabletBindingInfo() + : redefined_(false), + snapshot_version_(INT64_MAX), + schema_version_(INT64_MAX), + data_tablet_id_(), + hidden_tablet_ids_(), + lob_meta_tablet_id_(), + lob_piece_tablet_id_() { - reset(); } void ObTabletBindingInfo::reset() { redefined_ = false; - snapshot_version_ = OB_INVALID_VERSION; - schema_version_ = OB_INVALID_VERSION; + snapshot_version_ = INT64_MAX; + schema_version_ = INT64_MAX; data_tablet_id_.reset(); hidden_tablet_ids_.reset(); lob_meta_tablet_id_.reset(); lob_piece_tablet_id_.reset(); } +bool ObTabletBindingInfo::is_valid() const +{ + bool valid = true; + + if (INT64_MAX == snapshot_version_) { + valid = false; + } else if (INT64_MAX == schema_version_) { + valid = false; + } + + return valid; +} + int ObTabletBindingInfo::assign(const ObTabletBindingInfo &other) { int ret = OB_SUCCESS; @@ -449,7 +468,7 @@ int ObTabletBindingHelper::check_skip_tx_end(const ObTabletID &tablet_id, const ObTabletTxMultiSourceDataUnit tx_data; ObTabletBindingHelper helper(ls, trans_flags); - if (palf::SCN::invalid_scn() == trans_flags.scn_) { + if (SCN::invalid_scn() == trans_flags.scn_) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(tablet_id), K(ls)); } else if (OB_FAIL(helper.get_tablet(tablet_id, tablet_handle))) { @@ -609,9 +628,9 @@ int ObTabletBindingHelper::modify_tablet_binding_for_unbind( int ret = OB_SUCCESS; ObLSHandle ls_handle; const ObTransID &tx_id = trans_flags.tx_id_; - const palf::SCN scn = trans_flags.scn_; + const SCN scn = trans_flags.scn_; const bool for_replay = trans_flags.for_replay_; - const palf::SCN commit_version = trans_flags.trans_version_; + const SCN commit_version = trans_flags.trans_version_; if (OB_FAIL(get_ls(arg.ls_id_, ls_handle))) { LOG_WARN("failed to get ls", K(ret)); } else { @@ -758,7 +777,7 @@ int ObTabletBindingHelper::get_tablet(const ObTabletID &tablet_id, ObTabletHandl ObTabletTxMultiSourceDataUnit tx_data; if (OB_FAIL(handle.get_obj()->get_tx_data(tx_data))) { LOG_WARN("failed to get tx data", K(ret), K(key)); - } else if (palf::SCN::invalid_scn() != trans_flags_.scn_ && trans_flags_.scn_ <= tx_data.tx_scn_) { + } else if (SCN::invalid_scn() != trans_flags_.scn_ && trans_flags_.scn_ <= tx_data.tx_scn_) { ret = OB_NO_NEED_UPDATE; LOG_INFO("tablet frozen", K(ret), K(key), K(trans_flags_), K(tx_data)); } @@ -772,7 +791,7 @@ int ObTabletBindingHelper::replay_get_tablet(const ObTabletMapKey &key, ObTablet { // NOTICE: temporarily used, will be removed later! int ret = OB_SUCCESS; - const palf::SCN tablet_change_checkpoint_scn = ls_.get_tablet_change_checkpoint_scn(); + const SCN tablet_change_checkpoint_scn = ls_.get_tablet_change_checkpoint_scn(); ObTabletHandle tablet_handle; if (OB_FAIL(ObTabletCreateDeleteHelper::get_tablet(key, tablet_handle))) { @@ -870,7 +889,7 @@ int ObTabletBindingHelper::lock_tablet_binding(ObTabletHandle &handle, const ObM { int ret = OB_SUCCESS; const ObTransID &tx_id = trans_flags.tx_id_; - const palf::SCN scn = trans_flags.scn_; + const SCN scn = trans_flags.scn_; const bool for_replay = trans_flags.for_replay_; ObTablet *tablet = handle.get_obj(); ObTabletTxMultiSourceDataUnit tx_data; @@ -935,7 +954,7 @@ int ObTabletBindingHelper::set_scn(ObTabletHandle &handle, const ObMulSourceData { int ret = OB_SUCCESS; const ObTransID &tx_id = trans_flags.tx_id_; - const palf::SCN scn = trans_flags.scn_; + const SCN scn = trans_flags.scn_; const bool for_replay = trans_flags.for_replay_; ObTablet *tablet = handle.get_obj(); ObTabletTxMultiSourceDataUnit data; @@ -987,12 +1006,27 @@ int ObTabletBindingHelper::set_scn(const ObIArray &tablet_ids) const return ret; } +int ObTabletBindingHelper::check_need_dec_cnt_for_abort(const ObTabletTxMultiSourceDataUnit &tx_data, bool &need_dec) +{ + int ret = OB_SUCCESS; + const int cnt = tx_data.get_unsync_cnt_for_multi_data(); + need_dec = false; + if ((tx_data.is_tx_end() && cnt == 2) || (!tx_data.is_tx_end() && cnt == 1)) { + need_dec = true; + } else if ((tx_data.is_tx_end() && cnt == 1) || (!tx_data.is_tx_end() && cnt == 0)) { + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid cnt", K(ret), K(tx_data)); + } + return ret; +} + /// for commit or abort, reentrant for replay int ObTabletBindingHelper::unlock_tablet_binding(ObTabletHandle &handle, const ObMulSourceDataNotifyArg &trans_flags) { int ret = OB_SUCCESS; const ObTransID &tx_id = trans_flags.tx_id_; - const palf::SCN scn = trans_flags.scn_; + const SCN scn = trans_flags.scn_; const bool for_replay = trans_flags.for_replay_; const bool for_commit = trans_flags.notify_type_ == NotifyType::ON_COMMIT; ObTablet *tablet = handle.get_obj(); @@ -1014,9 +1048,13 @@ int ObTabletBindingHelper::unlock_tablet_binding(ObTabletHandle &handle, const O const bool abort_without_redo = !for_commit && !for_replay && !trans_flags.is_redo_synced(); tx_data.tx_id_ = ObTabletCommon::FINAL_TX_ID; tx_data.tx_scn_ = abort_without_redo ? old_scn : scn; - const SCN memtable_log_scn = (!scn.is_valid()) ? SCN::max_scn(): scn; - MemtableRefOp ref_op = (abort_without_redo ? MemtableRefOp::DEC_REF : MemtableRefOp::NONE); - if (OB_FAIL(tablet->set_tablet_final_status(tx_data, memtable_log_scn, for_replay, ref_op))) { + const SCN memtable_scn = (!scn.is_valid()) ? SCN::max_scn(): scn; + bool need_dec = false; + MemtableRefOp ref_op = MemtableRefOp::NONE; + if (OB_FAIL(check_need_dec_cnt_for_abort(tx_data, need_dec))) { + LOG_WARN("failed to save tx data", K(ret), K(tx_data), K(scn), K(for_replay)); + } else if (FALSE_IT(ref_op = (need_dec ? MemtableRefOp::DEC_REF : MemtableRefOp::NONE))) { + } else if (OB_FAIL(tablet->set_tablet_final_status(tx_data, memtable_scn, for_replay, ref_op))) { LOG_WARN("failed to save tx data", K(ret), K(tx_data), K(scn), K(for_replay), K(ref_op)); } } diff --git a/src/storage/tablet/ob_tablet_binding_helper.h b/src/storage/tablet/ob_tablet_binding_helper.h index 2fb8de65d..e8f008164 100644 --- a/src/storage/tablet/ob_tablet_binding_helper.h +++ b/src/storage/tablet/ob_tablet_binding_helper.h @@ -18,7 +18,7 @@ #include "lib/ob_define.h" #include "storage/memtable/ob_multi_source_data.h" #include "storage/tx/ob_trans_define.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -62,7 +62,7 @@ public: virtual int deep_copy(const memtable::ObIMultiSourceDataUnit *src, ObIAllocator *allocator = nullptr) override; virtual void reset() override; - virtual bool is_valid() const override { return true; } + virtual bool is_valid() const override; virtual int64_t get_data_size() const override { return sizeof(ObTabletBindingInfo); } virtual memtable::MultiSourceDataUnitType type() const override { return memtable::MultiSourceDataUnitType::TABLET_BINDING_INFO; } @@ -156,6 +156,7 @@ public: // common static bool has_lob_tablets(const obrpc::ObBatchCreateTabletArg &arg, const obrpc::ObCreateTabletInfo &info); + static int check_need_dec_cnt_for_abort(const ObTabletTxMultiSourceDataUnit &tx_data, bool &need_dec); static int lock_and_set_tx_data(ObTabletHandle &handle, ObTabletTxMultiSourceDataUnit &tx_data, const bool for_replay); static int lock_tablet_binding(ObTabletHandle &handle, const transaction::ObMulSourceDataNotifyArg &trans_flags); static int set_scn(ObTabletHandle &handle, const transaction::ObMulSourceDataNotifyArg &trans_flags); diff --git a/src/storage/tablet/ob_tablet_create_delete_helper.cpp b/src/storage/tablet/ob_tablet_create_delete_helper.cpp index 4f80b03fc..28eb2f549 100644 --- a/src/storage/tablet/ob_tablet_create_delete_helper.cpp +++ b/src/storage/tablet/ob_tablet_create_delete_helper.cpp @@ -31,7 +31,7 @@ #include "storage/tablet/ob_tablet_status.h" #include "storage/tx/ob_trans_define.h" #include "storage/tx_storage/ob_ls_service.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #define USING_LOG_PREFIX STORAGE @@ -96,7 +96,7 @@ int ObTabletCreateDeleteHelper::prepare_create_tablets( int ret = OB_SUCCESS; const ObLSID &ls_id = ls_.get_ls_id(); const ObTransID &tx_id = trans_flags.tx_id_; - const palf::SCN scn = trans_flags.scn_; + const SCN scn = trans_flags.scn_; const bool is_clog_replaying = trans_flags.for_replay_; const int64_t create_begin_ts = ObTimeUtility::current_monotonic_time(); ObTabletBindingPrepareCtx binding_ctx; @@ -845,12 +845,11 @@ int ObTabletCreateDeleteHelper::roll_back_remove_tablet( } else { LOG_WARN("failed to get tablet", K(ret), K(ls_id), K(tablet_id)); } - } else if (trans_flags.for_replay_) { - // for replay, no need to dec ref for memtable } else { ObTablet *tablet = tablet_handle.get_obj(); ObTabletMemtableMgr *memtable_mgr = static_cast(tablet->get_memtable_mgr()); ObSEArray memtable_handle_array; + bool need_dec = false; if (OB_FAIL(memtable_mgr->get_all_memtables(memtable_handle_array))) { LOG_WARN("failed to get all memtables", K(ret), K(ls_id), K(tablet_id)); } else if (memtable_handle_array.empty()) { @@ -869,8 +868,10 @@ int ObTabletCreateDeleteHelper::roll_back_remove_tablet( } else if (!last_memtable->has_multi_source_data_unit(MultiSourceDataUnitType::TABLET_TX_DATA)) { LOG_INFO("last memtable does not have msd, do nothing", K(ret), K(ls_id), K(tablet_id), K(cnt)); } else if (OB_FAIL(tablet->get_tx_data(tx_data))) { - LOG_WARN("failed to get tx data", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(tablet->save_multi_source_data_unit(&tx_data, ObScnRange::MAX_SCN, + LOG_WARN("failed to get tx data", K(ret), K(ls_id), K(tablet_id), K(cnt)); + } else if (OB_FAIL(ObTabletBindingHelper::check_need_dec_cnt_for_abort(tx_data, need_dec))) { + LOG_WARN("failed to save tx data", K(ret), K(tx_data), K(trans_flags)); + } else if (need_dec && OB_FAIL(tablet->save_multi_source_data_unit(&tx_data, ObScnRange::MAX_SCN, trans_flags.for_replay_, MemtableRefOp::DEC_REF, true/*is_callback*/))) { LOG_WARN("failed to save msd", K(ret), K(ls_id), K(tablet_id), K(cnt)); } else { @@ -912,7 +913,7 @@ int ObTabletCreateDeleteHelper::do_abort_create_tablet( // replaying procedure, clog ts is smaller than tx log ts, just skip LOG_INFO("skip abort create tablet", K(ret), K(tablet_id), K(trans_flags), K(tx_data)); } else if (OB_UNLIKELY(!trans_flags.for_replay_ - && tx_data.tx_scn_ != palf::SCN::max_scn() + && tx_data.tx_scn_ != SCN::max_scn() && trans_flags.scn_ <= tx_data.tx_scn_)) { // If tx log ts equals SCN::max_scn(), it means redo callback has not been called. // Thus, we should handle this situation @@ -991,11 +992,6 @@ int ObTabletCreateDeleteHelper::prepare_remove_tablets( if (OB_FAIL(ObTabletBindingHelper::lock_and_set_tx_data(tablet_handle, tx_data, trans_flags.for_replay_))) { LOG_WARN("failed to lock tablet binding", K(ret), K(key), K(tx_data)); - // TODO(bowen.gbw): temproarily swallow 4200 error while prepare remove tablets - if (trans_flags.for_replay_ && OB_HASH_EXIST == ret) { - ret = OB_SUCCESS; - LOG_INFO("swallow 4200 error", K(ret), K(key), K(tx_data), K(trans_flags)); - } } } } @@ -1060,7 +1056,7 @@ int ObTabletCreateDeleteHelper::redo_remove_tablets( int ret = OB_SUCCESS; const ObLSID &ls_id = ls_.get_ls_id(); const bool is_clog_replaying = trans_flags.for_replay_; - const palf::SCN scn = trans_flags.scn_; + const SCN scn = trans_flags.scn_; if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_INVALID_ARGUMENT; @@ -1241,7 +1237,7 @@ int ObTabletCreateDeleteHelper::do_abort_remove_tablet( } else if (OB_FAIL(tablet_handle.get_obj()->get_tx_data(tx_data))) { LOG_WARN("failed to get tx data", K(ret), K(key)); } else if (OB_UNLIKELY(!trans_flags.for_replay_ - && trans_flags.scn_ != palf::SCN::invalid_scn() + && trans_flags.scn_ != SCN::invalid_scn() && trans_flags.scn_ <= tx_data.tx_scn_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("log ts is smaller than tx log ts", K(ret), K(key), K(trans_flags), K(tx_data)); @@ -1252,11 +1248,10 @@ int ObTabletCreateDeleteHelper::do_abort_remove_tablet( is_valid = false; LOG_INFO("tablet status is not DELETING", K(ret), K(key), K(trans_flags), K(tx_data)); } else { - if (trans_flags.for_replay_) { - } else if (trans_flags.is_redo_synced()) { - // on redo cb has been called - // do nothing(on redo cb has already done DEC_REF) - } else { + bool need_dec = false; + if (OB_FAIL(ObTabletBindingHelper::check_need_dec_cnt_for_abort(tx_data, need_dec))) { + LOG_WARN("failed to save tx data", K(ret), K(tx_data), K(trans_flags)); + } else if (need_dec) { ref_op = MemtableRefOp::DEC_REF; } } @@ -2247,7 +2242,7 @@ int ObTabletCreateDeleteHelper::do_create_tablet( const ObTransID &tx_id = trans_flags.tx_id_; const bool for_replay = trans_flags.for_replay_; MemtableRefOp ref_op = for_replay ? MemtableRefOp::NONE : MemtableRefOp::INC_REF; - SCN scn = trans_flags.for_replay_ ? trans_flags.scn_ : palf::SCN::max_scn(); + SCN scn = trans_flags.for_replay_ ? trans_flags.scn_ : SCN::max_scn(); SCN create_scn = trans_flags.scn_; ObMetaDiskAddr mem_addr; ObTabletTableStoreFlag table_store_flag; diff --git a/src/storage/tablet/ob_tablet_create_delete_helper.h b/src/storage/tablet/ob_tablet_create_delete_helper.h index 57c8bcb09..53855b414 100644 --- a/src/storage/tablet/ob_tablet_create_delete_helper.h +++ b/src/storage/tablet/ob_tablet_create_delete_helper.h @@ -20,7 +20,7 @@ #include "common/ob_tablet_id.h" #include "storage/memtable/ob_memtable.h" #include "storage/meta_mem/ob_tablet_handle.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/tablet/ob_tablet_status.h" namespace oceanbase @@ -175,7 +175,7 @@ private: common::ObIArray &tablet_create_info_array); static int get_all_existed_tablets( const obrpc::ObBatchCreateTabletArg &arg, - const palf::SCN &scn, + const share::SCN &scn, common::ObIArray &existed_tablet_id_array, NonLockedHashSet &existed_tablet_id_set); static int build_batch_create_tablet_arg( @@ -189,8 +189,8 @@ private: static int set_tablet_final_status( ObTabletHandle &tablet_handle, const ObTabletStatus::Status status, - const palf::SCN &tx_scn, - const palf::SCN &memtable_scn, + const share::SCN &tx_scn, + const share::SCN &memtable_scn, const bool for_replay, const memtable::MemtableRefOp ref_op = memtable::MemtableRefOp::NONE); static bool check_tablet_status( @@ -231,7 +231,7 @@ private: const transaction::ObMulSourceDataNotifyArg &trans_flags); int ensure_skip_create_all_tablets_safe( const obrpc::ObBatchCreateTabletArg &arg, - const palf::SCN &scn); + const share::SCN &scn); int build_pure_data_tablet( const obrpc::ObBatchCreateTabletArg &arg, const obrpc::ObCreateTabletInfo &info, @@ -297,7 +297,7 @@ private: const transaction::ObMulSourceDataNotifyArg &trans_flags); int replay_verify_tablets( const obrpc::ObBatchRemoveTabletArg &arg, - const palf::SCN &scn, + const share::SCN &scn, common::ObIArray &tablet_id_array); private: ObLS &ls_; diff --git a/src/storage/tablet/ob_tablet_create_sstable_param.cpp b/src/storage/tablet/ob_tablet_create_sstable_param.cpp index 54cf2e4b5..273705b10 100644 --- a/src/storage/tablet/ob_tablet_create_sstable_param.cpp +++ b/src/storage/tablet/ob_tablet_create_sstable_param.cpp @@ -22,6 +22,7 @@ namespace oceanbase { +using namespace share; namespace storage { ObTabletCreateSSTableParam::ObTabletCreateSSTableParam() @@ -51,7 +52,7 @@ ObTabletCreateSSTableParam::ObTabletCreateSSTableParam() original_size_(0), max_merged_trans_version_(0), ddl_scn_(), - filled_tx_scn_(palf::SCN::min_scn()), + filled_tx_scn_(SCN::min_scn()), contain_uncommitted_row_(false), compressor_type_(ObCompressorType::INVALID_COMPRESSOR), encrypt_id_(0), diff --git a/src/storage/tablet/ob_tablet_create_sstable_param.h b/src/storage/tablet/ob_tablet_create_sstable_param.h index bf5bc3e24..9005db07d 100644 --- a/src/storage/tablet/ob_tablet_create_sstable_param.h +++ b/src/storage/tablet/ob_tablet_create_sstable_param.h @@ -18,7 +18,7 @@ #include "storage/blocksstable/ob_macro_block_id.h" #include "storage/blocksstable/ob_imicro_block_reader.h" #include "storage/meta_mem/ob_meta_obj_struct.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -97,8 +97,8 @@ public: int64_t occupy_size_; int64_t original_size_; int64_t max_merged_trans_version_; - palf::SCN ddl_scn_; - palf::SCN filled_tx_scn_; + share::SCN ddl_scn_; + share::SCN filled_tx_scn_; bool contain_uncommitted_row_; common::ObCompressorType compressor_type_; int64_t encrypt_id_; diff --git a/src/storage/tablet/ob_tablet_ddl_info.cpp b/src/storage/tablet/ob_tablet_ddl_info.cpp index 26e9dc30c..40aa5566e 100644 --- a/src/storage/tablet/ob_tablet_ddl_info.cpp +++ b/src/storage/tablet/ob_tablet_ddl_info.cpp @@ -16,10 +16,11 @@ #include "lib/ob_define.h" #include "lib/utility/ob_print_utils.h" #include "lib/utility/utility.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { +using namespace share; namespace storage { ObTabletDDLInfo::ObTabletDDLInfo() @@ -54,7 +55,7 @@ int ObTabletDDLInfo::get(int64_t &schema_version, int64_t &schema_refreshed_ts) return ret; } int ObTabletDDLInfo::update(const int64_t schema_version, - const palf::SCN &scn, + const SCN &scn, int64_t &schema_refreshed_ts) { int ret = OB_SUCCESS; @@ -76,4 +77,4 @@ int ObTabletDDLInfo::update(const int64_t schema_version, } } // namespace storage -} // namespace oceanbase \ No newline at end of file +} // namespace oceanbase diff --git a/src/storage/tablet/ob_tablet_ddl_info.h b/src/storage/tablet/ob_tablet_ddl_info.h index 8c17f2e76..31cd51c4f 100644 --- a/src/storage/tablet/ob_tablet_ddl_info.h +++ b/src/storage/tablet/ob_tablet_ddl_info.h @@ -16,7 +16,7 @@ #include #include "lib/lock/ob_tc_rwlock.h" #include "lib/utility/ob_print_utils.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -32,13 +32,13 @@ public: void reset(); int get(int64_t &schema_version, int64_t &schema_refreshed_ts); int update(const int64_t schema_version, - const palf::SCN &scn, + const share::SCN &scn, int64_t &schema_refreshed_ts); TO_STRING_KV(K_(ddl_schema_version), K_(ddl_schema_refreshed_ts), K_(schema_version_change_scn)); private: int64_t ddl_schema_version_; int64_t ddl_schema_refreshed_ts_; - palf::SCN schema_version_change_scn_; + share::SCN schema_version_change_scn_; common::TCRWLock rwlock_; }; } // namespace storage diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.cpp b/src/storage/tablet/ob_tablet_memtable_mgr.cpp index e4bef0a41..f912d5f4d 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.cpp +++ b/src/storage/tablet/ob_tablet_memtable_mgr.cpp @@ -137,7 +137,7 @@ int ObTabletMemtableMgr::destroy_storage_schema_recorder() // There are two cases: // 1. create the first memtable for tablet // 2. create the new memtable after freezing the old memtable -int ObTabletMemtableMgr::create_memtable(const palf::SCN clog_checkpoint_scn, +int ObTabletMemtableMgr::create_memtable(const SCN clog_checkpoint_scn, const int64_t schema_version, const bool for_replay) { @@ -195,8 +195,8 @@ int ObTabletMemtableMgr::create_memtable(const palf::SCN clog_checkpoint_scn, LOG_WARN("failed to init memtable", K(ret), K(ls_id), K(table_key), KP(freezer_), KP(this), K(schema_version), K(logstream_freeze_clock)); } else { - palf::SCN new_clog_checkpoint_scn; - palf::SCN new_snapshot_version; + SCN new_clog_checkpoint_scn; + SCN new_snapshot_version; memtable::ObMemtable *last_frozen_memtable = get_last_frozen_memtable_(); if (OB_NOT_NULL(last_frozen_memtable)) { // keep the check order: is_frozen, write_ref_cnt, then unsubmitted_cnt and unsynced_cnt @@ -378,8 +378,8 @@ ObMemtable *ObTabletMemtableMgr::get_last_frozen_memtable_() const } int ObTabletMemtableMgr::resolve_left_boundary_for_active_memtable(ObIMemtable *memtable, - palf::SCN start_scn, - palf::SCN snapshot_scn) + SCN start_scn, + SCN snapshot_scn) { ObTableHandleV2 handle; ObIMemtable *active_memtable = nullptr; @@ -460,7 +460,7 @@ int ObTabletMemtableMgr::set_is_tablet_freeze_for_active_memtable(ObIMemtable *& return ret; } -int ObTabletMemtableMgr::get_memtable_for_replay(palf::SCN replay_scn, +int ObTabletMemtableMgr::get_memtable_for_replay(SCN replay_scn, ObTableHandleV2 &handle) { int ret = OB_SUCCESS; @@ -489,7 +489,7 @@ int ObTabletMemtableMgr::get_memtable_for_replay(palf::SCN replay_scn, } } if (OB_SUCC(ret) && !handle.is_valid() && i < memtable_head_) { - palf::SCN clog_checkpoint_scn; + SCN clog_checkpoint_scn; if (OB_FAIL(get_newest_clog_checkpoint_scn(clog_checkpoint_scn))) { } else if (replay_scn <= clog_checkpoint_scn) { // no need to replay the log @@ -796,6 +796,7 @@ int64_t ObTabletMemtableMgr::to_string(char *buf, const int64_t buf_len) const J_COMMA(); } } + J_KV("schema_recorder", schema_recorder_); J_ARRAY_END(); J_OBJ_END(); J_OBJ_END(); diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.h b/src/storage/tablet/ob_tablet_memtable_mgr.h index 45dd9b3af..12bd5a429 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.h +++ b/src/storage/tablet/ob_tablet_memtable_mgr.h @@ -54,7 +54,7 @@ public: bool has_active_memtable(); int64_t get_memtable_count() const; - virtual int get_memtable_for_replay(palf::SCN replay_scn, + virtual int get_memtable_for_replay(share::SCN replay_scn, ObTableHandleV2 &handle) override; memtable::ObMemtable *get_last_frozen_memtable() const; memtable::ObMemtable *get_last_frozen_memtable_() const; @@ -66,7 +66,7 @@ public: memtable::ObMemtable *&memtable, const memtable::MultiSourceDataUnitType type) const override; int release_tail_memtable(memtable::ObIMemtable *memtable); - int create_memtable(const palf::SCN clog_checkpoint_scn, + int create_memtable(const share::SCN clog_checkpoint_scn, const int64_t schema_version, const bool for_replay); int get_memtables( @@ -83,8 +83,8 @@ public: int get_memtables_nolock(ObTableHdlArray &handle); int get_first_frozen_memtable(ObTableHandleV2 &handle) const; int resolve_left_boundary_for_active_memtable(memtable::ObIMemtable *memtable, - palf::SCN start_scn, - palf::SCN snapshot_version); + share::SCN start_scn, + share::SCN snapshot_version); int unset_logging_blocked_for_active_memtable(memtable::ObIMemtable *memtable); int set_is_tablet_freeze_for_active_memtable(memtable::ObIMemtable *&memtable, bool is_force_freeze = false); diff --git a/src/storage/tablet/ob_tablet_meta.cpp b/src/storage/tablet/ob_tablet_meta.cpp index f60917808..b8a92e82b 100644 --- a/src/storage/tablet/ob_tablet_meta.cpp +++ b/src/storage/tablet/ob_tablet_meta.cpp @@ -17,7 +17,7 @@ #include "lib/ob_define.h" #include "lib/utility/serialization.h" #include "share/schema/ob_table_schema.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -26,7 +26,7 @@ using namespace blocksstable; using namespace palf; namespace storage { -const palf::SCN ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN = palf::SCN::base_scn(); +const SCN ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN = SCN::base_scn(); ObTabletMeta::ObTabletMeta() : version_(TABLET_META_VERSION), @@ -68,7 +68,7 @@ int ObTabletMeta::init( const common::ObTabletID &data_tablet_id, const common::ObTabletID &lob_meta_tablet_id, const common::ObTabletID &lob_piece_tablet_id, - const palf::SCN create_scn, + const SCN create_scn, const int64_t snapshot_version, const lib::Worker::CompatMode compat_mode, const ObTabletTableStoreFlag &table_store_flag, @@ -113,6 +113,8 @@ int ObTabletMeta::init( report_status_.data_checksum_ = 0; report_status_.row_count_ = 0; + ddl_data_.snapshot_version_ = OB_INVALID_VERSION; + ddl_data_.schema_version_ = OB_INVALID_VERSION; ddl_data_.lob_meta_tablet_id_ = lob_meta_tablet_id; ddl_data_.lob_piece_tablet_id_ = lob_piece_tablet_id; @@ -138,9 +140,9 @@ int ObTabletMeta::init( const ObTabletBindingInfo &ddl_data, const ObTabletAutoincSeq &autoinc_seq, const int64_t max_sync_storage_schema_version, - const palf::SCN clog_checkpoint_scn, - const palf::SCN ddl_checkpoint_scn, - const palf::SCN ddl_start_scn, + const SCN clog_checkpoint_scn, + const SCN ddl_checkpoint_scn, + const SCN ddl_start_scn, const int64_t ddl_snapshot_version) { int ret = OB_SUCCESS; @@ -164,8 +166,8 @@ int ObTabletMeta::init( ref_tablet_id_ = old_tablet_meta.ref_tablet_id_; create_scn_ = old_tablet_meta.create_scn_; start_scn_ = old_tablet_meta.start_scn_; - ddl_start_scn_ = palf::SCN::max(ddl_start_scn, old_tablet_meta.ddl_start_scn_); - clog_checkpoint_scn_ = palf::SCN::max(clog_checkpoint_scn, old_tablet_meta.clog_checkpoint_scn_); + ddl_start_scn_ = SCN::max(ddl_start_scn, old_tablet_meta.ddl_start_scn_); + clog_checkpoint_scn_ = SCN::max(clog_checkpoint_scn, old_tablet_meta.clog_checkpoint_scn_); compat_mode_ = old_tablet_meta.compat_mode_; ha_status_ = old_tablet_meta.ha_status_; report_status_ = old_tablet_meta.report_status_; @@ -175,7 +177,7 @@ int ObTabletMeta::init( tx_data_ = tx_data; table_store_flag_ = old_tablet_meta.table_store_flag_; max_sync_storage_schema_version_ = max_sync_storage_schema_version; - ddl_checkpoint_scn_ = palf::SCN::max(old_tablet_meta.ddl_checkpoint_scn_, ddl_checkpoint_scn); + ddl_checkpoint_scn_ = SCN::max(old_tablet_meta.ddl_checkpoint_scn_, ddl_checkpoint_scn); ddl_snapshot_version_ = MAX(old_tablet_meta.ddl_snapshot_version_, ddl_snapshot_version); is_inited_ = true; } @@ -635,7 +637,7 @@ int ObTabletMeta::update(const ObMigrationTabletParam ¶m) return ret; } -int ObTabletMeta::update_create_scn(const palf::SCN create_scn) +int ObTabletMeta::update_create_scn(const SCN create_scn) { int ret = OB_SUCCESS; diff --git a/src/storage/tablet/ob_tablet_meta.h b/src/storage/tablet/ob_tablet_meta.h index 565d353eb..6bc431384 100644 --- a/src/storage/tablet/ob_tablet_meta.h +++ b/src/storage/tablet/ob_tablet_meta.h @@ -32,7 +32,7 @@ #include "storage/tx/ob_trans_define.h" #include "storage/high_availability/ob_tablet_ha_status.h" #include "storage/tablet/ob_tablet_table_store_flag.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -43,7 +43,7 @@ struct ObMigrationTabletParam; class ObTabletMeta final { public: - static const palf::SCN INIT_CLOG_CHECKPOINT_SCN; + static const share::SCN INIT_CLOG_CHECKPOINT_SCN; public: ObTabletMeta(); @@ -58,7 +58,7 @@ public: const common::ObTabletID &data_tablet_id, const common::ObTabletID &lob_meta_tablet_id, const common::ObTabletID &lob_piece_tablet_id, - const palf::SCN create_scn, + const share::SCN create_scn, const int64_t snapshot_version, const lib::Worker::CompatMode compat_mode, const ObTabletTableStoreFlag &table_store_flag, @@ -72,9 +72,9 @@ public: const ObTabletBindingInfo &ddl_data, const share::ObTabletAutoincSeq &autoinc_seq, const int64_t max_sync_storage_schema_version, - const palf::SCN clog_checkpoint_scn = palf::SCN::min_scn(), - const palf::SCN ddl_checkpoint_scn = palf::SCN::min_scn(), - const palf::SCN ddl_start_scn = palf::SCN::min_scn(), + const share::SCN clog_checkpoint_scn = share::SCN::min_scn(), + const share::SCN ddl_checkpoint_scn = share::SCN::min_scn(), + const share::SCN ddl_start_scn = share::SCN::min_scn(), const int64_t ddl_snapshot_version = 0); int init( common::ObIAllocator &allocator, @@ -100,7 +100,7 @@ public: int64_t get_serialize_size() const; int update(const ObMigrationTabletParam ¶m); - int update_create_scn(const palf::SCN create_scn); + int update_create_scn(const share::SCN create_scn); public: static int deserialize_id( const char *buf, @@ -143,10 +143,10 @@ public: common::ObTabletID data_tablet_id_; common::ObTabletID ref_tablet_id_; bool has_next_tablet_; - palf::SCN create_scn_; - palf::SCN start_scn_; - palf::SCN clog_checkpoint_scn_; // may less than last_minor->end_log_ts - palf::SCN ddl_checkpoint_scn_; + share::SCN create_scn_; + share::SCN start_scn_; + share::SCN clog_checkpoint_scn_; // may less than last_minor->end_log_ts + share::SCN ddl_checkpoint_scn_; // snapshot_version of last minor int64_t snapshot_version_; int64_t multi_version_start_; @@ -157,7 +157,7 @@ public: ObTabletTxMultiSourceDataUnit tx_data_; ObTabletBindingInfo ddl_data_; ObTabletTableStoreFlag table_store_flag_; - palf::SCN ddl_start_scn_; + share::SCN ddl_start_scn_; int64_t ddl_snapshot_version_; int64_t max_sync_storage_schema_version_; //ATTENTION : Add a new variable need consider ObMigrationTabletParam @@ -224,10 +224,10 @@ public: common::ObTabletID tablet_id_; common::ObTabletID data_tablet_id_; common::ObTabletID ref_tablet_id_; - palf::SCN create_scn_; - palf::SCN start_scn_; // for migration - palf::SCN clog_checkpoint_scn_; - palf::SCN ddl_checkpoint_scn_; + share::SCN create_scn_; + share::SCN start_scn_; // for migration + share::SCN clog_checkpoint_scn_; + share::SCN ddl_checkpoint_scn_; int64_t snapshot_version_; int64_t multi_version_start_; lib::Worker::CompatMode compat_mode_; @@ -239,7 +239,7 @@ public: ObStorageSchema storage_schema_; compaction::ObMediumCompactionInfoList medium_info_list_; ObTabletTableStoreFlag table_store_flag_; - palf::SCN ddl_start_scn_; + share::SCN ddl_start_scn_; int64_t ddl_snapshot_version_; // max_sync_version may less than storage_schema.schema_version_ when major update schema int64_t max_sync_storage_schema_version_; diff --git a/src/storage/tablet/ob_tablet_multi_source_data.cpp b/src/storage/tablet/ob_tablet_multi_source_data.cpp index b5ef78d39..a2ead296b 100644 --- a/src/storage/tablet/ob_tablet_multi_source_data.cpp +++ b/src/storage/tablet/ob_tablet_multi_source_data.cpp @@ -20,7 +20,7 @@ using namespace oceanbase::common; using namespace oceanbase::memtable; -using namespace oceanbase::palf; +using namespace oceanbase::share; namespace oceanbase { @@ -205,7 +205,7 @@ int64_t ObTabletTxMultiSourceDataUnit::get_serialize_size() const return size; } -int ObTabletTxMultiSourceDataUnit::set_scn(const palf::SCN &scn) +int ObTabletTxMultiSourceDataUnit::set_scn(const SCN &scn) { tx_scn_ = scn; diff --git a/src/storage/tablet/ob_tablet_multi_source_data.h b/src/storage/tablet/ob_tablet_multi_source_data.h index 79555a447..d1fc9a212 100644 --- a/src/storage/tablet/ob_tablet_multi_source_data.h +++ b/src/storage/tablet/ob_tablet_multi_source_data.h @@ -19,7 +19,7 @@ #include "storage/tablet/ob_tablet_common.h" #include "storage/tablet/ob_tablet_status.h" #include "storage/tx/ob_trans_define.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -39,7 +39,7 @@ public: virtual bool is_valid() const override; virtual int64_t get_data_size() const override; virtual memtable::MultiSourceDataUnitType type() const override; - virtual int set_scn(const palf::SCN &scn) override; + virtual int set_scn(const share::SCN &scn) override; public: int serialize(char *buf, const int64_t len, int64_t &pos) const; int deserialize(const char *buf, const int64_t len, int64_t &pos); @@ -50,7 +50,7 @@ public: int32_t version_; mutable int32_t length_; // length is assigned when serializing transaction::ObTransID tx_id_; - palf::SCN tx_scn_; + share::SCN tx_scn_; ObTabletStatus tablet_status_; private: static const int32_t TX_DATA_VERSION = 1; diff --git a/src/storage/tablet/ob_tablet_service_clog_replay_executor.cpp b/src/storage/tablet/ob_tablet_service_clog_replay_executor.cpp index e9ed6ec11..37d4954fd 100644 --- a/src/storage/tablet/ob_tablet_service_clog_replay_executor.cpp +++ b/src/storage/tablet/ob_tablet_service_clog_replay_executor.cpp @@ -18,6 +18,7 @@ #include "storage/meta_mem/ob_tablet_handle.h" using namespace oceanbase::logservice; +using namespace oceanbase::share; namespace oceanbase { @@ -26,7 +27,7 @@ namespace storage ObTabletServiceClogReplayExecutor::ObTabletServiceClogReplayExecutor( ObLS *ls, const palf::LSN &lsn, - const palf::SCN &scn) + const SCN &scn) : ls_(ls), lsn_(lsn), scn_(scn) @@ -39,7 +40,7 @@ int ObTabletServiceClogReplayExecutor::execute( const int64_t buf_size, const int64_t pos, const palf::LSN &lsn, - const palf::SCN &scn) + const SCN &scn) { int ret = OB_SUCCESS; int64_t tmp_pos = pos; @@ -73,7 +74,7 @@ int ObTabletServiceClogReplayExecutor::execute( } int ObTabletServiceClogReplayExecutor::replay_update_storage_schema( - const palf::SCN &scn, + const SCN &scn, const char *buf, const int64_t buf_size, const int64_t pos) diff --git a/src/storage/tablet/ob_tablet_service_clog_replay_executor.h b/src/storage/tablet/ob_tablet_service_clog_replay_executor.h index d9d5d26f6..d6981d60a 100644 --- a/src/storage/tablet/ob_tablet_service_clog_replay_executor.h +++ b/src/storage/tablet/ob_tablet_service_clog_replay_executor.h @@ -15,7 +15,7 @@ #include #include "logservice/palf/lsn.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -32,7 +32,7 @@ public: const int64_t log_size, const int64_t pos, const palf::LSN &lsn, - const palf::SCN &scn); + const share::SCN &scn); public: TO_STRING_KV(KP(ls_), K(lsn_), K(scn_)); @@ -40,20 +40,20 @@ private: ObTabletServiceClogReplayExecutor( ObLS *ls, const palf::LSN &lsn, - const palf::SCN &scn); + const share::SCN &scn); ~ObTabletServiceClogReplayExecutor() = default; ObTabletServiceClogReplayExecutor(const ObTabletServiceClogReplayExecutor&) = delete; ObTabletServiceClogReplayExecutor &operator=(const ObTabletServiceClogReplayExecutor&) = delete; private: int replay_update_storage_schema( - const palf::SCN &scn, + const share::SCN &scn, const char *buf, const int64_t buf_size, const int64_t pos); private: ObLS *ls_; palf::LSN lsn_; - palf::SCN scn_; + share::SCN scn_; }; } diff --git a/src/storage/tablet/ob_tablet_status.cpp b/src/storage/tablet/ob_tablet_status.cpp index 1c639c781..a942d32ee 100644 --- a/src/storage/tablet/ob_tablet_status.cpp +++ b/src/storage/tablet/ob_tablet_status.cpp @@ -21,11 +21,11 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tablet/ob_tablet_multi_source_data.h" #include "storage/tx/ob_trans_define.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { -using namespace palf; +using namespace share; namespace storage { ObTabletStatus::ObTabletStatus() diff --git a/src/storage/tablet/ob_tablet_status.h b/src/storage/tablet/ob_tablet_status.h index 0f94ea410..ebd2ae594 100644 --- a/src/storage/tablet/ob_tablet_status.h +++ b/src/storage/tablet/ob_tablet_status.h @@ -28,7 +28,7 @@ namespace transaction { class ObTransID; } -namespace palf +namespace share { class SCN; } @@ -111,7 +111,7 @@ public: int check(const uint64_t time_us); int wake_up( ObTabletTxMultiSourceDataUnit &tx_data, - const palf::SCN &memtable_scn, + const share::SCN &memtable_scn, const bool for_replay, const memtable::MemtableRefOp ref_op); private: diff --git a/src/storage/tablet/ob_tablet_table_store.cpp b/src/storage/tablet/ob_tablet_table_store.cpp index 9c15054b7..df7819b1e 100644 --- a/src/storage/tablet/ob_tablet_table_store.cpp +++ b/src/storage/tablet/ob_tablet_table_store.cpp @@ -19,7 +19,7 @@ #include "storage/ob_i_memtable_mgr.h" #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" #include "storage/tablet/ob_tablet_table_store.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" using namespace oceanbase; using namespace oceanbase::blocksstable; @@ -503,7 +503,7 @@ int ObTabletTableStore::calculate_read_tables( } else if (OB_NOT_NULL(base_table)) { // find read minor tables ObITable *table = nullptr; int64_t inc_pos = -1; - palf::SCN last_scn = base_table->is_major_sstable() ? palf::SCN::max_scn() : base_table->get_end_scn(); + SCN last_scn = base_table->is_major_sstable() ? SCN::max_scn() : base_table->get_end_scn(); for (int64_t i = 0; OB_SUCC(ret) && i < minor_tables_.count_; ++i) { table = minor_tables_[i]; if ((base_table->is_major_sstable() && table->get_upper_trans_version() >= base_table->get_snapshot_version()) @@ -544,7 +544,7 @@ int ObTabletTableStore::calculate_read_memtables(const ObTablet &tablet, ObTable { int ret = OB_SUCCESS; int64_t start_snapshot_version = tablet.get_snapshot_version(); - palf::SCN start_scn = tablet.get_clog_checkpoint_scn(); + SCN start_scn = tablet.get_clog_checkpoint_scn(); int64_t mem_pos = -1; ObITable *memtable = nullptr; @@ -572,7 +572,7 @@ int ObTabletTableStore::pull_memtables() LOG_WARN("failed to get all memtables from memtable_mgr", K(ret)); } else { int64_t start_snapshot_version = tablet_ptr_->get_snapshot_version(); - palf::SCN clog_checkpoint_scn = tablet_ptr_->get_clog_checkpoint_scn(); + SCN clog_checkpoint_scn = tablet_ptr_->get_clog_checkpoint_scn(); int64_t start_pos = -1; for (int64_t i = 0; OB_SUCC(ret) && i < memtable_handles.count(); ++i) { @@ -800,11 +800,11 @@ int ObTabletTableStore::build_minor_tables( ObSSTable *new_sstable = static_cast(new_table); if (sstable->get_meta().get_basic_meta().max_merged_trans_version_ < new_sstable->get_meta().get_basic_meta().max_merged_trans_version_) { - need_add = true; + need_add = false; LOG_INFO("new sstable max merge trans version not equal to old sstable, " "need add new sstable when table key is same", KPC(sstable), KPC(new_sstable)); } else { - need_add = false; + need_add = true; // just keep old sstable } } else if (ObTableStoreUtil::check_include_by_scn_range(*new_table, *table)) { LOG_DEBUG("table purged", K(*new_table), K(*table)); @@ -931,8 +931,8 @@ int ObTabletTableStore::build_ddl_sstables( // already pushed, do nothing } else { ObITable *last_ddl_sstable = ddl_sstables.at(ddl_sstables.count() - 1); - const palf::SCN old_ddl_start_scn = static_cast(last_ddl_sstable)->get_meta().get_basic_meta().get_ddl_scn(); - const palf::SCN new_ddl_start_scn = static_cast(new_table)->get_meta().get_basic_meta().get_ddl_scn(); + const SCN old_ddl_start_scn = static_cast(last_ddl_sstable)->get_meta().get_basic_meta().get_ddl_scn(); + const SCN new_ddl_start_scn = static_cast(new_table)->get_meta().get_basic_meta().get_ddl_scn(); if (new_ddl_start_scn > old_ddl_start_scn) { // ddl start log ts changed means task retry, clean up old ddl sstable ddl_sstables.reset(); @@ -1333,7 +1333,7 @@ int ObTabletTableStore::cut_ha_sstable_scn_range_( common::ObIArray &minor_sstables) { int ret = OB_SUCCESS; - palf::SCN last_end_scn = palf::SCN::min_scn(); + SCN last_end_scn = SCN::min_scn(); for (int64_t i = 0; OB_SUCC(ret) && i < minor_sstables.count(); ++i) { ObITable *table = minor_sstables.at(i); @@ -1397,9 +1397,9 @@ int ObTabletTableStore::combin_ha_minor_sstables_( common::ObIArray &new_minor_sstables) { int ret = OB_SUCCESS; - palf::SCN logical_start_scn = ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN; - palf::SCN logical_end_scn = ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN; - palf::SCN max_copy_end_scn = ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN; + SCN logical_start_scn = ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN; + SCN logical_end_scn = ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN; + SCN max_copy_end_scn = ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN; int64_t old_store_minor_tables_index = 0; //get remote logical minor sstable log ts @@ -1416,7 +1416,7 @@ int ObTabletTableStore::combin_ha_minor_sstables_( } //push new sstable into array - const palf::SCN checkpoint_scn = tablet_ptr_->get_clog_checkpoint_scn(); + const SCN checkpoint_scn = tablet_ptr_->get_clog_checkpoint_scn(); if (OB_SUCC(ret)) { if (need_add_minor_sstables.empty()) { //do nothing diff --git a/src/storage/tx/ob_ctx_tx_data.cpp b/src/storage/tx/ob_ctx_tx_data.cpp index 57e475c06..f93345829 100644 --- a/src/storage/tx/ob_ctx_tx_data.cpp +++ b/src/storage/tx/ob_ctx_tx_data.cpp @@ -19,7 +19,7 @@ namespace oceanbase { using namespace storage; -using namespace palf; +using namespace share; namespace transaction { @@ -272,7 +272,7 @@ int ObCtxTxData::set_state(int32_t state) return ret; } -int ObCtxTxData::set_commit_version(const palf::SCN &commit_version) +int ObCtxTxData::set_commit_version(const SCN &commit_version) { int ret = OB_SUCCESS; RLockGuard guard(lock_); @@ -286,10 +286,10 @@ int ObCtxTxData::set_commit_version(const palf::SCN &commit_version) return ret; } -int ObCtxTxData::set_start_log_ts(const palf::SCN &start_ts) +int ObCtxTxData::set_start_log_ts(const SCN &start_ts) { int ret = OB_SUCCESS; - const palf::SCN tmp_start_ts = (start_ts.is_valid() ? start_ts : palf::SCN::max_scn()); + const SCN tmp_start_ts = (start_ts.is_valid() ? start_ts : SCN::max_scn()); RLockGuard guard(lock_); if (OB_FAIL(check_tx_data_writable_())) { @@ -301,7 +301,7 @@ int ObCtxTxData::set_start_log_ts(const palf::SCN &start_ts) return ret; } -int ObCtxTxData::set_end_log_ts(const palf::SCN &end_scn) +int ObCtxTxData::set_end_log_ts(const SCN &end_scn) { int ret = OB_SUCCESS; RLockGuard guard(lock_); @@ -321,24 +321,24 @@ int32_t ObCtxTxData::get_state() const return (NULL != tx_data_ ? tx_data_->state_: tx_commit_data_.state_); } -const palf::SCN ObCtxTxData::get_commit_version() const +const SCN ObCtxTxData::get_commit_version() const { RLockGuard guard(lock_); SCN commit_version = (NULL != tx_data_ ? tx_data_->commit_version_ : tx_commit_data_.commit_version_); return commit_version; } -const palf::SCN ObCtxTxData::get_start_log_ts() const +const SCN ObCtxTxData::get_start_log_ts() const { RLockGuard guard(lock_); - palf::SCN start_log_scn = (NULL != tx_data_ ? tx_data_->start_scn_ : tx_commit_data_.start_scn_); + SCN start_log_scn = (NULL != tx_data_ ? tx_data_->start_scn_ : tx_commit_data_.start_scn_); return start_log_scn; } -const palf::SCN ObCtxTxData::get_end_log_ts() const +const SCN ObCtxTxData::get_end_log_ts() const { RLockGuard guard(lock_); - palf::SCN end_log_scn = (NULL != tx_data_ ? tx_data_->end_scn_ : tx_commit_data_.end_scn_); + SCN end_log_scn = (NULL != tx_data_ ? tx_data_->end_scn_ : tx_commit_data_.end_scn_); return end_log_scn; } @@ -431,10 +431,11 @@ int ObCtxTxData::add_undo_action(ObUndoAction &undo_action, storage::ObUndoStatu int ObCtxTxData::Guard::get_tx_data(const ObTxData *&tx_data) const { int ret = OB_SUCCESS; - if (NULL == host_.tx_data_) { + auto tmp_tx_data = host_.tx_data_; + if (NULL == tmp_tx_data) { ret = OB_TRANS_CTX_NOT_EXIST; } else { - tx_data = host_.tx_data_; + tx_data = tmp_tx_data; } return ret; } diff --git a/src/storage/tx/ob_ctx_tx_data.h b/src/storage/tx/ob_ctx_tx_data.h index 53143eb4e..e556db9e2 100644 --- a/src/storage/tx/ob_ctx_tx_data.h +++ b/src/storage/tx/ob_ctx_tx_data.h @@ -46,14 +46,14 @@ public: void get_tx_table(storage::ObTxTable *&tx_table); int set_state(int32_t state); - int set_commit_version(const palf::SCN &commit_version); - int set_start_log_ts(const palf::SCN &start_ts); - int set_end_log_ts(const palf::SCN &end_ts); + int set_commit_version(const share::SCN &commit_version); + int set_start_log_ts(const share::SCN &start_ts); + int set_end_log_ts(const share::SCN &end_ts); int32_t get_state() const; - const palf::SCN get_commit_version() const; - const palf::SCN get_start_log_ts() const; - const palf::SCN get_end_log_ts() const; + const share::SCN get_commit_version() const; + const share::SCN get_start_log_ts() const; + const share::SCN get_end_log_ts() const; ObTransID get_tx_id() const; @@ -68,12 +68,12 @@ public: TO_STRING_KV(KP(ctx_mgr_), KPC(tx_data_), K(tx_commit_data_), K(read_only_)); public: - class Guard { + class Guard { // TODO(yunxing.cyx): remove it friend class ObCtxTxData; - Guard(ObCtxTxData &host) : host_(host) { host_.lock_.rdlock(); } + Guard(ObCtxTxData &host) : host_(host) { } ObCtxTxData &host_; public: - ~Guard() { host_.lock_.unlock(); } + ~Guard() { } int get_tx_data(const storage::ObTxData *&tx_data) const; }; Guard get_tx_data() { return Guard(*this); } diff --git a/src/storage/tx/ob_defensive_check_mgr.h b/src/storage/tx/ob_defensive_check_mgr.h index aac49a988..e4a46df3c 100644 --- a/src/storage/tx/ob_defensive_check_mgr.h +++ b/src/storage/tx/ob_defensive_check_mgr.h @@ -13,7 +13,7 @@ #ifndef OB_DEFENSIVE_CHECK_MGR_H_ #define OB_DEFENSIVE_CHECK_MGR_H_ #include "storage/blocksstable/ob_fuse_row_cache.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -42,7 +42,7 @@ struct ObDefensiveCheckRecordExtend K_(is_all_data_from_memtable), K_(query_flag)); public: - palf::SCN fist_access_table_start_scn_; + share::SCN fist_access_table_start_scn_; int64_t total_table_handle_cnt_; int64_t start_access_table_idx_; int64_t end_access_table_idx_; diff --git a/src/storage/tx/ob_gts_task_queue.cpp b/src/storage/tx/ob_gts_task_queue.cpp index 063a4d1fd..94dd38dfb 100644 --- a/src/storage/tx/ob_gts_task_queue.cpp +++ b/src/storage/tx/ob_gts_task_queue.cpp @@ -20,6 +20,7 @@ namespace oceanbase { using namespace common; +using namespace share; namespace transaction { @@ -84,7 +85,7 @@ int ObGTSTaskQueue::foreach_task(const MonotonicTs srr, } } if (OB_SUCC(ret)) { - palf::SCN scn; + SCN scn; scn.convert_for_gts(gts); if (GET_GTS == task_type_) { if (OB_FAIL(task->get_gts_callback(srr, scn, receive_gts_ts))) { diff --git a/src/storage/tx/ob_id_service.cpp b/src/storage/tx/ob_id_service.cpp index 0d3feb7ec..1be18399e 100644 --- a/src/storage/tx/ob_id_service.cpp +++ b/src/storage/tx/ob_id_service.cpp @@ -21,7 +21,7 @@ #include "storage/slog/ob_storage_log_replayer.h" #include "storage/tx_storage/ob_ls_service.h" #include "logservice/ob_log_base_header.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/tx_storage/ob_ls_handle.h" #include "sql/das/ob_das_id_service.h" @@ -128,7 +128,7 @@ int ObIDService::submit_log_(const int64_t last_id, const int64_t limited_id) } else { ObPresistIDLog ls_log(last_id, limited_id); palf::LSN lsn; - palf::SCN log_ts, base_scn; + SCN log_ts, base_scn; int64_t base_ts = 0; if (TimestampService == service_type_) { if (ATOMIC_LOAD(&tmp_last_id_) != 0) { @@ -161,7 +161,7 @@ int ObIDService::submit_log_(const int64_t last_id, const int64_t limited_id) return ret; } -int ObIDService::handle_submit_callback(const bool success, const int64_t limited_id, const palf::SCN log_ts) +int ObIDService::handle_submit_callback(const bool success, const int64_t limited_id, const SCN log_ts) { int ret = OB_SUCCESS; WLockGuard guard(rwlock_); @@ -190,7 +190,7 @@ int ObIDService::handle_submit_callback(const bool success, const int64_t limite } int ObIDService::replay(const void *buffer, const int64_t buf_size, - const palf::LSN &lsn, const palf::SCN &log_scn) + const palf::LSN &lsn, const SCN &log_scn) { //TODO(scn) int ret = OB_SUCCESS; @@ -211,7 +211,7 @@ int ObIDService::replay(const void *buffer, const int64_t buf_size, return ret; } -int ObIDService::handle_replay_result(const int64_t last_id, const int64_t limited_id, const palf::SCN log_ts) +int ObIDService::handle_replay_result(const int64_t last_id, const int64_t limited_id, const SCN log_ts) { int ret = OB_SUCCESS; WLockGuard guard(rwlock_); @@ -257,17 +257,17 @@ int ObIDService::update_ls_id_meta(const bool write_slog) return ret; } -int ObIDService::flush(palf::SCN &rec_scn) +int ObIDService::flush(SCN &rec_scn) { int ret = OB_SUCCESS; WLockGuard guard(rwlock_); - palf::SCN latest_rec_log_ts = rec_log_ts_.atomic_get(); + SCN latest_rec_log_ts = rec_log_ts_.atomic_get(); if (latest_rec_log_ts <= rec_scn) { latest_rec_log_ts = rec_log_ts_.atomic_get(); if (OB_FAIL(update_ls_id_meta(true))) { TRANS_LOG(WARN, "update id meta of ls meta fail", K(ret), K(service_type_)); } else { - rec_log_ts_.atomic_bcas(latest_rec_log_ts, palf::SCN::max_scn()); + rec_log_ts_.atomic_bcas(latest_rec_log_ts, SCN::max_scn()); } TRANS_LOG(INFO, "flush", K(ret), K(service_type_), K(rec_log_ts_), K(limited_id_)); } @@ -294,9 +294,9 @@ int ObIDService::check_leader(bool &leader) return ret; } -palf::SCN ObIDService::get_rec_scn() +SCN ObIDService::get_rec_scn() { - const palf::SCN rec_log_ts = rec_log_ts_.atomic_get(); + const SCN rec_log_ts = rec_log_ts_.atomic_get(); TRANS_LOG(INFO, "get rec log scn", K(service_type_), K(rec_log_ts)); return rec_log_ts; } @@ -385,8 +385,8 @@ int ObIDService::get_number(const int64_t range, const int64_t base_id, int64_t return ret; } -void ObIDService::get_virtual_info(int64_t &last_id, int64_t &limited_id, palf::SCN &rec_log_ts, - palf::SCN &latest_log_ts, int64_t &pre_allocated_range, +void ObIDService::get_virtual_info(int64_t &last_id, int64_t &limited_id, SCN &rec_log_ts, + SCN &latest_log_ts, int64_t &pre_allocated_range, int64_t &submit_log_ts, bool &is_master) { int ret = OB_SUCCESS; @@ -432,7 +432,7 @@ int ObIDService::get_id_service(const int64_t id_service_type, ObIDService *&id_ return ret; } -void ObIDService::update_limited_id(const int64_t limited_id, const palf::SCN latest_log_ts) +void ObIDService::update_limited_id(const int64_t limited_id, const SCN latest_log_ts) { WLockGuard guard(rwlock_); (void)inc_update(&last_id_, limited_id); @@ -670,7 +670,7 @@ int ObAllIDMeta::update_id_service() void ObAllIDMeta::update_id_meta(const int64_t service_type, const int64_t limited_id, - const palf::SCN latest_log_ts) + const SCN latest_log_ts) { (void)inc_update(&id_meta_[service_type].limited_id_, limited_id); id_meta_[service_type].latest_log_ts_.inc_update(latest_log_ts); diff --git a/src/storage/tx/ob_id_service.h b/src/storage/tx/ob_id_service.h index 2e1aeaa3d..aee9fd7aa 100644 --- a/src/storage/tx/ob_id_service.h +++ b/src/storage/tx/ob_id_service.h @@ -18,7 +18,7 @@ #include "logservice/ob_append_callback.h" #include "logservice/ob_log_base_type.h" #include "logservice/ob_log_handler.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -27,10 +27,6 @@ namespace storage { class ObLS; } -namespace palf -{ -class SCN; -} namespace transaction { @@ -64,7 +60,7 @@ public: int serialize_ls_log(ObPresistIDLog &ls_log, int64_t service_type); char *get_log_buf() { return log_buf_; } int64_t get_log_pos() { return pos_; } - void set_log_ts(const palf::SCN &ts) { log_ts_ = ts; } + void set_log_ts(const share::SCN &ts) { log_ts_ = ts; } void set_srv_type(const int64_t srv_type) { id_srv_type_ = srv_type; } void set_limited_id(const int64_t limited_id) { limited_id_ = limited_id; } public: @@ -77,7 +73,7 @@ public: private: int64_t id_srv_type_; int64_t limited_id_; - palf::SCN log_ts_; + share::SCN log_ts_; char log_buf_[MAX_LOG_BUFF_SIZE]; int64_t pos_; }; @@ -107,19 +103,19 @@ public: int get_number(const int64_t range, const int64_t base_id, int64_t &start_id, int64_t &end_id); //日志处理 - int handle_submit_callback(const bool success, const int64_t limited_id, const palf::SCN log_ts); + int handle_submit_callback(const bool success, const int64_t limited_id, const share::SCN log_ts); void test_lock() { WLockGuard guard(rwlock_); } //获取回放数据 - int handle_replay_result(const int64_t last_id, const int64_t limited_id, const palf::SCN log_ts); + int handle_replay_result(const int64_t last_id, const int64_t limited_id, const share::SCN log_ts); // clog checkpoint - int flush(palf::SCN &scn); - palf::SCN get_rec_scn(); + int flush(share::SCN &scn); + share::SCN get_rec_scn(); int64_t get_rec_log_ts(); // for clog replay - int replay(const void *buffer, const int64_t buf_size, const palf::LSN &lsn, const palf::SCN &log_ts); + int replay(const void *buffer, const int64_t buf_size, const palf::LSN &lsn, const share::SCN &log_ts); //切主 int switch_to_follower_gracefully(); @@ -130,12 +126,12 @@ public: int check_leader(bool &leader); int check_and_fill_ls(); void reset_ls(); - void update_limited_id(const int64_t limited_id, const palf::SCN latest_log_ts); + void update_limited_id(const int64_t limited_id, const share::SCN latest_log_ts); int update_ls_id_meta(const bool write_slog); // 虚表 - void get_virtual_info(int64_t &last_id, int64_t &limited_id, palf::SCN &rec_log_ts, - palf::SCN &latest_log_ts, int64_t &pre_allocated_range, + void get_virtual_info(int64_t &last_id, int64_t &limited_id, share::SCN &rec_log_ts, + share::SCN &latest_log_ts, int64_t &pre_allocated_range, int64_t &submit_log_ts, bool &is_master); static const int64_t SUBMIT_LOG_ALARM_INTERVAL = 100 * 1000; static const int64_t MIN_LAST_ID = 1; @@ -156,9 +152,9 @@ protected: int64_t limited_id_; bool is_logging_; //checkpoint ts - palf::SCN rec_log_ts_; + share::SCN rec_log_ts_; //最新持久化成功的ts - palf::SCN latest_log_ts_; + share::SCN latest_log_ts_; // current time when submit log int64_t submit_log_ts_; mutable common::SpinRWLock rwlock_; @@ -174,7 +170,7 @@ public: ObIDMeta() : limited_id_(ObIDService::MIN_LAST_ID) {} ~ObIDMeta() {} int64_t limited_id_; - palf::SCN latest_log_ts_; + share::SCN latest_log_ts_; TO_STRING_KV(K_(limited_id), K_(latest_log_ts)); OB_UNIS_VERSION(1); }; @@ -188,7 +184,7 @@ public: int update_id_service(); void update_id_meta(const int64_t service_type, const int64_t limited_id, - const palf::SCN latest_log_ts); + const share::SCN latest_log_ts); int64_t to_string(char* buf, const int64_t buf_len) const { int64_t pos = 0; diff --git a/src/storage/tx/ob_keep_alive_ls_handler.cpp b/src/storage/tx/ob_keep_alive_ls_handler.cpp index 2620fef8c..95c0977aa 100644 --- a/src/storage/tx/ob_keep_alive_ls_handler.cpp +++ b/src/storage/tx/ob_keep_alive_ls_handler.cpp @@ -100,7 +100,7 @@ int ObKeepAliveLSHandler::try_submit_log() { int ret = OB_SUCCESS; palf::LSN lsn; - palf::SCN scn = palf::SCN::min_scn(); + SCN scn = SCN::min_scn(); if (OB_ISNULL(log_handler_ptr_)) { stat_info_.other_error_cnt += 1; @@ -152,8 +152,8 @@ void ObKeepAliveLSHandler::print_stat_info() bool ObKeepAliveLSHandler::check_gts_() { bool need_submit = true; - palf::SCN gts; - palf::SCN max_scn; + SCN gts; + SCN max_scn; int ret = OB_SUCCESS; if (OB_ISNULL(log_handler_ptr_)) { diff --git a/src/storage/tx/ob_keep_alive_ls_handler.h b/src/storage/tx/ob_keep_alive_ls_handler.h index 0f727b3d5..c386a0335 100644 --- a/src/storage/tx/ob_keep_alive_ls_handler.h +++ b/src/storage/tx/ob_keep_alive_ls_handler.h @@ -65,7 +65,7 @@ public: int64_t near_to_gts_cnt; int64_t other_error_cnt; int64_t submit_succ_cnt; - palf::SCN last_log_ts_; + share::SCN last_log_ts_; palf::LSN last_lsn_; private: @@ -99,7 +99,7 @@ public: int on_success() {ATOMIC_STORE(&is_busy_, false); return OB_SUCCESS;} int on_failure() {ATOMIC_STORE(&is_busy_, false); return OB_SUCCESS;} - int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const palf::SCN &scn) + int replay(const void *buffer, const int64_t nbytes, const palf::LSN &lsn, const share::SCN &scn) { UNUSED(buffer); UNUSED(nbytes); @@ -114,8 +114,8 @@ public: int switch_to_leader() { ATOMIC_STORE(&is_master_,true); return OB_SUCCESS;} int switch_to_follower_gracefully() { ATOMIC_STORE(&is_master_,false); return OB_SUCCESS;} int resume_leader() { ATOMIC_STORE(&is_master_,true);return OB_SUCCESS; } - palf::SCN get_rec_scn() { return palf::SCN::max_scn(); } - int flush(palf::SCN &rec_scn) { return OB_SUCCESS;} + share::SCN get_rec_scn() { return share::SCN::max_scn(); } + int flush(share::SCN &rec_scn) { return OB_SUCCESS;} private: bool check_gts_(); @@ -131,7 +131,7 @@ private : int64_t submit_buf_len_; int64_t submit_buf_pos_; - palf::SCN last_gts_; + share::SCN last_gts_; ObLSKeepAliveStatInfo stat_info_; }; diff --git a/src/storage/tx/ob_standby_timestamp_service.cpp b/src/storage/tx/ob_standby_timestamp_service.cpp index b4d213c3d..28f47a58e 100644 --- a/src/storage/tx/ob_standby_timestamp_service.cpp +++ b/src/storage/tx/ob_standby_timestamp_service.cpp @@ -16,7 +16,7 @@ #include "ob_timestamp_access.h" #include "ob_timestamp_service.h" #include "logservice/ob_log_service.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "observer/ob_server_struct.h" #include "observer/ob_srv_network_frame.h" #include "storage/tx/ob_trans_service.h" diff --git a/src/storage/tx/ob_timestamp_service.cpp b/src/storage/tx/ob_timestamp_service.cpp index 1391ee70f..79591ec20 100644 --- a/src/storage/tx/ob_timestamp_service.cpp +++ b/src/storage/tx/ob_timestamp_service.cpp @@ -18,11 +18,12 @@ #include "ob_timestamp_access.h" #include "storage/tx_storage/ob_ls_map.h" #include "storage/tx_storage/ob_ls_service.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { +using namespace oceanbase::share; namespace transaction { @@ -170,7 +171,7 @@ int ObTimestampService::switch_to_leader() if (OB_FAIL(check_and_fill_ls())) { TRANS_LOG(WARN, "ls set fail", K(ret)); } else { - palf::SCN version; + SCN version; if (OB_FAIL(ls_->get_log_handler()->get_max_scn(version))) { TRANS_LOG(WARN, "get max ts fail", K(ret)); } else { diff --git a/src/storage/tx/ob_trans_ctx.cpp b/src/storage/tx/ob_trans_ctx.cpp index a41892dff..e833d1d7f 100644 --- a/src/storage/tx/ob_trans_ctx.cpp +++ b/src/storage/tx/ob_trans_ctx.cpp @@ -197,7 +197,7 @@ ObITsMgr *ObTransCtx::get_ts_mgr_() return trans_service_->get_ts_mgr(); } -int ObTransCtx::defer_commit_callback_(const int retcode, const palf::SCN commit_version) +int ObTransCtx::defer_commit_callback_(const int retcode, const SCN commit_version) { int ret = OB_SUCCESS; if (!commit_cb_.is_enabled()) { diff --git a/src/storage/tx/ob_trans_ctx.h b/src/storage/tx/ob_trans_ctx.h index 831c1db7e..6e26bc5cb 100644 --- a/src/storage/tx/ob_trans_ctx.h +++ b/src/storage/tx/ob_trans_ctx.h @@ -196,7 +196,7 @@ protected: void set_stc_by_now_(); MonotonicTs get_stc_(); ObITsMgr *get_ts_mgr_(); - int defer_commit_callback_(const int ret, const palf::SCN commit_version); + int defer_commit_callback_(const int ret, const share::SCN commit_version); int64_t get_remaining_wait_interval_us_() { return trans_need_wait_wrap_.get_remaining_wait_interval_us(); diff --git a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp index bdcda7ebf..3067ac44a 100644 --- a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp +++ b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp @@ -581,7 +581,7 @@ int ObLSTxCtxMgr::remove_callback_for_uncommited_tx(ObMemtable* mt) return ret; } -int ObLSTxCtxMgr::replay_start_working_log(const ObTxStartWorkingLog &log, palf::SCN start_working_ts) +int ObLSTxCtxMgr::replay_start_working_log(const ObTxStartWorkingLog &log, SCN start_working_ts) { int ret = OB_SUCCESS; UNUSED(log); @@ -595,7 +595,7 @@ int ObLSTxCtxMgr::replay_start_working_log(const ObTxStartWorkingLog &log, palf: return ret; } -int ObLSTxCtxMgr::on_start_working_log_cb_succ(palf::SCN start_working_ts) +int ObLSTxCtxMgr::on_start_working_log_cb_succ(SCN start_working_ts) { int ret = OB_SUCCESS; WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); @@ -649,7 +649,7 @@ int ObLSTxCtxMgr::on_start_working_log_cb_fail() int ObLSTxCtxMgr::submit_start_working_log_() { int ret = OB_SUCCESS; - palf::SCN scn; + SCN scn; const int64_t fake_epoch = 0xbaba; if (OB_FAIL(ls_log_writer_.submit_start_working_log(fake_epoch, scn))) { TRANS_LOG(WARN, "submit start working log failed", KR(ret), K(*this)); @@ -702,7 +702,7 @@ int ObLSTxCtxMgr::try_wait_gts_and_inc_max_commit_ts_() { int ret = OB_SUCCESS; if (!is_leader_serving_) { - palf::SCN gts; + SCN gts; MonotonicTs receive_gts_ts(0); if (OB_FAIL(ts_mgr_->get_gts(tenant_id_, @@ -968,7 +968,7 @@ int ObLSTxCtxMgr::online() return ret; } -int ObLSTxCtxMgr::get_ls_min_uncommit_tx_prepare_version(palf::SCN &min_prepare_version) +int ObLSTxCtxMgr::get_ls_min_uncommit_tx_prepare_version(SCN &min_prepare_version) { int ret = OB_SUCCESS; @@ -986,7 +986,7 @@ int ObLSTxCtxMgr::get_ls_min_uncommit_tx_prepare_version(palf::SCN &min_prepare_ return ret; } -int ObLSTxCtxMgr::get_min_undecided_scn(palf::SCN &scn) +int ObLSTxCtxMgr::get_min_undecided_scn(SCN &scn) { int ret = OB_SUCCESS; ObGetMinUndecidedLogTsFunctor fn; @@ -1235,7 +1235,7 @@ int ObLSTxCtxMgr::check_with_tx_data(const ObTransID& tx_id, ObITxDataCheckFunct return ret; } -int ObLSTxCtxMgr::get_rec_scn(palf::SCN &rec_scn) +int ObLSTxCtxMgr::get_rec_scn(SCN &rec_scn) { int ret = OB_SUCCESS; @@ -1254,8 +1254,8 @@ int ObLSTxCtxMgr::get_rec_scn(palf::SCN &rec_scn) } else if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { TRANS_LOG(WARN, "for each transaction context error", KR(ret), "manager", *this); } else { - palf::SCN aggre_rec_scn = get_aggre_rec_scn_(); - rec_scn = palf::SCN::min(fn.get_rec_log_ts(), aggre_rec_scn); + SCN aggre_rec_scn = get_aggre_rec_scn_(); + rec_scn = SCN::min(fn.get_rec_log_ts(), aggre_rec_scn); TRANS_LOG(INFO, "succ to get rec scn", K(*this), K(aggre_rec_scn)); } } @@ -1290,7 +1290,7 @@ int ObLSTxCtxMgr::on_tx_ctx_table_flushed() return ret; } -int ObLSTxCtxMgr::get_min_start_scn(palf::SCN &min_start_scn) +int ObLSTxCtxMgr::get_min_start_scn(SCN &min_start_scn) { int ret = OB_SUCCESS; @@ -1304,11 +1304,11 @@ int ObLSTxCtxMgr::get_min_start_scn(palf::SCN &min_start_scn) return ret; } -palf::SCN ObLSTxCtxMgr::get_aggre_rec_scn_() +SCN ObLSTxCtxMgr::get_aggre_rec_scn_() { - palf::SCN ret; - palf::SCN prev_aggre_rec_scn = prev_aggre_rec_scn_.atomic_get(); - palf::SCN aggre_rec_scn = aggre_rec_scn_.atomic_get(); + SCN ret; + SCN prev_aggre_rec_scn = prev_aggre_rec_scn_.atomic_get(); + SCN aggre_rec_scn = aggre_rec_scn_.atomic_get(); // Before the checkpoint of the tx ctx table is succeed, we should still use // the prev_aggre_log_ts. And after successfully checkpointed, we can use the @@ -1335,8 +1335,8 @@ int ObLSTxCtxMgr::refresh_aggre_rec_scn() if (!prev_aggre_rec_scn_.is_valid()) { // We should remember the rec_log_ts before the tx ctx table is successfully // checkpointed - palf::SCN old_v; - palf::SCN new_v; + SCN old_v; + SCN new_v; do { old_v = aggre_rec_scn_; new_v.reset(); @@ -1350,7 +1350,7 @@ int ObLSTxCtxMgr::refresh_aggre_rec_scn() return ret; } -int ObLSTxCtxMgr::update_aggre_log_ts_wo_lock(palf::SCN rec_scn) +int ObLSTxCtxMgr::update_aggre_log_ts_wo_lock(SCN rec_scn) { int ret = OB_SUCCESS; @@ -1358,8 +1358,8 @@ int ObLSTxCtxMgr::update_aggre_log_ts_wo_lock(palf::SCN rec_scn) // we cannot lock here, because the lock order must be // ObLSTxCtxMgr -> ObPartTransCtx, otherwise we may be // deadlocked - palf::SCN old_v; - palf::SCN new_v; + SCN old_v; + SCN new_v; do { old_v = aggre_rec_scn_; if (!old_v.is_valid()) { @@ -1945,7 +1945,7 @@ int ObTxCtxMgr::iterate_tx_ctx_mgr_stat(const ObAddr &addr, return ret; } -int ObTxCtxMgr::get_ls_min_uncommit_tx_prepare_version(const ObLSID &ls_id, palf::SCN &min_prepare_version) +int ObTxCtxMgr::get_ls_min_uncommit_tx_prepare_version(const ObLSID &ls_id, SCN &min_prepare_version) { int ret = OB_SUCCESS; ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; @@ -1972,7 +1972,7 @@ int ObTxCtxMgr::get_ls_min_uncommit_tx_prepare_version(const ObLSID &ls_id, palf return ret; } -int ObTxCtxMgr::get_min_undecided_scn(const ObLSID &ls_id, palf::SCN &scn) +int ObTxCtxMgr::get_min_undecided_scn(const ObLSID &ls_id, SCN &scn) { int ret = OB_SUCCESS; ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; diff --git a/src/storage/tx/ob_trans_ctx_mgr_v4.h b/src/storage/tx/ob_trans_ctx_mgr_v4.h index 129a964c8..9b9f594df 100644 --- a/src/storage/tx/ob_trans_ctx_mgr_v4.h +++ b/src/storage/tx/ob_trans_ctx_mgr_v4.h @@ -237,7 +237,7 @@ public: // Get the min prepare version of transaction module of current observer for slave read // @param [out] min_prepare_version: MIN(uncommitted tx_ctx's prepare version | ObLSTxCtxMgr) - int get_ls_min_uncommit_tx_prepare_version(palf::SCN &min_prepare_version); + int get_ls_min_uncommit_tx_prepare_version(share::SCN &min_prepare_version); // Check the ObLSTxCtxMgr's status; if it's stopped, the new transaction ctx will not be created; // otherwise, the tx_ctx is created normally @@ -355,7 +355,7 @@ public: // Get the minimum value of rec_scn of TxCtx in this ObLSTxCtxMgr; // This value is used as the starting point of the redo replay required by the ObTxCtxTable file; // @param [out] rec_scnMIN(TxCtx's rec_scn in ObLSTxCtxMgr) - int get_rec_scn(palf::SCN &rec_scn); + int get_rec_scn(share::SCN &rec_scn); // Notify that the ObTxCtxTable corresponding to this ObLSTxCtxMgr has completed the flush operation; // Traverse ObLSTxCtxMgr and set the prev_rec_scn of each ObTxCtx to OB_INVALID_TIMESTAMP; @@ -380,7 +380,7 @@ public: int check_with_tx_data(const ObTransID &tx_id, ObITxDataCheckFunctor &fn); // TODO Remove - int get_min_undecided_scn(palf::SCN &scn); + int get_min_undecided_scn(share::SCN &scn); //1. During the minor merge process, the status of uncommitted transactions will be actively // queried; if ObPartTransCtx is released due to Rebuild, and the query cannot be performed, @@ -425,12 +425,12 @@ public: // Replay the start working log entry. Traverse ObLSTxCtxMgr and set the data_complete_ of each // ObTxCtx to false. Make all the preceding ACTIVE_INFO log entry invalid. // @param [in] log: The start working log entry - int replay_start_working_log(const ObTxStartWorkingLog &log, palf::SCN start_working_scn); + int replay_start_working_log(const ObTxStartWorkingLog &log, share::SCN start_working_scn); // START_WORKING log entry is successfully written to the PALF; According to the current // ObLSTxCtxMgr's status, drive ObLSTxCtxMgr to continue to execute the switch_to_leader routine // or resume_leader routine; - int on_start_working_log_cb_succ(palf::SCN start_working_scn); + int on_start_working_log_cb_succ(share::SCN start_working_scn); // START_WORKING log entry failed to written to the PALF; // Break the switch_to_leader routine or resume_leader routine; switch the ObLSTxCtxMgr's state to F_WORKING @@ -450,13 +450,13 @@ public: // When start executing swith_to_leader routine, wait_gts_elapse will be called to ensure that // GTS has exceeded this batch_commit_version; // @param [in] replay_commit_version : replayed single-ls-tx's log entry commit version; - void update_max_replay_commit_version(const palf::SCN &replay_commit_version) + void update_max_replay_commit_version(const share::SCN &replay_commit_version) { max_replay_commit_version_.inc_update(replay_commit_version); } // Iterate all tx ctx in this ls and get the min_start_scn - int get_min_start_scn(palf::SCN &min_start_scn); + int get_min_start_scn(share::SCN &min_start_scn); // Get the trans_service corresponding to this ObLSTxCtxMgr; transaction::ObTransService *get_trans_service() { return txs_; } @@ -474,7 +474,7 @@ public: // Update aggre_log_ts without lock, because we canot lock using the order of // ObPartTransCtx -> ObLSTxCtxMgr, It will be a deadlock with normal order. - int update_aggre_log_ts_wo_lock(palf::SCN rec_log_ts); + int update_aggre_log_ts_wo_lock(share::SCN rec_log_ts); TO_STRING_KV(KP(this), K_(ls_id), @@ -506,7 +506,7 @@ private: int get_tx_ctx_(const ObTransID &tx_id, const bool for_replay, ObPartTransCtx *&tx_ctx); int submit_start_working_log_(); int try_wait_gts_and_inc_max_commit_ts_(); - palf::SCN get_aggre_rec_scn_(); + share::SCN get_aggre_rec_scn_(); public: static const int64_t MAX_HASH_ITEM_PRINT = 16; static const int64_t WAIT_SW_CB_TIMEOUT = 100 * 1000; // 100 ms @@ -700,7 +700,7 @@ private: ObITsMgr *ts_mgr_; // See the ObLSTxCtxMgr's member function update_max_replay_commit_version - palf::SCN max_replay_commit_version_; + share::SCN max_replay_commit_version_; // Recover log timestamp for aggregated tx ctx. For the purpose of Durability, // we need a space to store rec_log_ts for checkpoint[1]. Although tx ctx @@ -709,8 +709,8 @@ private: // remembered while must be released soon, in the ctx_mgr. // // [1]: It you are interested in rec_log_ts, you can see ARIES paper. - palf::SCN aggre_rec_scn_; - palf::SCN prev_aggre_rec_scn_; + share::SCN aggre_rec_scn_; + share::SCN prev_aggre_rec_scn_; // Online timestamp for ObLSTxCtxMgr int64_t online_ts_; @@ -884,10 +884,10 @@ public: // Get the min prepare version of transaction module of current observer for slave read // @param [in] indicates the specified ls; // @param [out] min_prepare_version: MIN(uncommitted transaction ctx's prepare version) - int get_ls_min_uncommit_tx_prepare_version(const share::ObLSID &ls_id, palf::SCN &min_prepare_version); + int get_ls_min_uncommit_tx_prepare_version(const share::ObLSID &ls_id, share::SCN &min_prepare_version); // TODO - int get_min_undecided_scn(const share::ObLSID &ls_id, palf::SCN &scn); + int get_min_undecided_scn(const share::ObLSID &ls_id, share::SCN &scn); // Get all ls ID in the observer // param [out] ls_id_iter: ObLSIDIterator is Used to return all ls id in the observer, diff --git a/src/storage/tx/ob_trans_define.cpp b/src/storage/tx/ob_trans_define.cpp index 831a9ffb1..6a738a28c 100644 --- a/src/storage/tx/ob_trans_define.cpp +++ b/src/storage/tx/ob_trans_define.cpp @@ -30,6 +30,7 @@ namespace oceanbase { using namespace common; +using namespace share; using namespace sql; using namespace storage; using namespace memtable; @@ -292,7 +293,7 @@ void ObElrTransInfo::reset() ctx_id_ = 0; } -int ObElrTransInfo::init(const ObTransID &trans_id, uint32_t ctx_id, const palf::SCN commit_version) +int ObElrTransInfo::init(const ObTransID &trans_id, uint32_t ctx_id, const SCN commit_version) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_trans_define.h b/src/storage/tx/ob_trans_define.h index a476a848d..c666cd5fc 100644 --- a/src/storage/tx/ob_trans_define.h +++ b/src/storage/tx/ob_trans_define.h @@ -31,7 +31,7 @@ #include "storage/tx/ob_trans_result.h" #include "storage/tx/ob_xa_define.h" #include "ob_multi_data_source.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -1259,10 +1259,10 @@ class ObElrTransInfo final public: ObElrTransInfo() { reset(); } void reset(); - int init(const ObTransID &trans_id, uint32_t ctx_id, const palf::SCN commit_version); + int init(const ObTransID &trans_id, uint32_t ctx_id, const share::SCN commit_version); const ObTransID &get_trans_id() const { return trans_id_; } uint32_t get_ctx_id() const { return ctx_id_; } - palf::SCN get_commit_version() const { return commit_version_; } + share::SCN get_commit_version() const { return commit_version_; } int set_result(const int result) { result_ = result; return common::OB_SUCCESS; } uint64_t hash() const { return trans_id_.hash(); } int get_result() const { return ATOMIC_LOAD(&result_); } @@ -1270,7 +1270,7 @@ public: TO_STRING_KV(K_(trans_id), K_(commit_version), K_(result), K_(ctx_id)); private: ObTransID trans_id_; - palf::SCN commit_version_; + share::SCN commit_version_; int result_; uint32_t ctx_id_; }; @@ -1742,15 +1742,15 @@ public: ObTxBufferNodeArray multi_data_source_; // check common::ObAddr scheduler_; - palf::SCN prepare_version_; + share::SCN prepare_version_; int64_t trans_type_; int64_t next_log_entry_no_; - palf::SCN max_applied_log_ts_; - palf::SCN max_applying_log_ts_; + share::SCN max_applied_log_ts_; + share::SCN max_applying_log_ts_; int64_t max_applying_part_log_no_; // start from 0 on follower and always be INT64_MAX on leader int64_t max_submitted_seq_no_; // maintains on Leader and transfer to Follower via ActiveInfoLog uint64_t checksum_; - palf::SCN checksum_scn_; + share::SCN checksum_scn_; palf::LSN max_durable_lsn_; bool data_complete_; bool is_dup_tx_; @@ -1768,9 +1768,9 @@ static const int64_t USEC_PER_SEC = 1000 * 1000; struct ObMulSourceDataNotifyArg { ObTransID tx_id_; - palf::SCN scn_; // the log ts of current notify type + share::SCN scn_; // the log ts of current notify type // in case of abort transaction, trans_version_ is invalid - palf::SCN trans_version_; + share::SCN trans_version_; bool for_replay_; NotifyType notify_type_; diff --git a/src/storage/tx/ob_trans_define_v4.cpp b/src/storage/tx/ob_trans_define_v4.cpp index 5b5004ad1..317b5b46e 100644 --- a/src/storage/tx/ob_trans_define_v4.cpp +++ b/src/storage/tx/ob_trans_define_v4.cpp @@ -27,6 +27,7 @@ #define USING_LOG_PREFIX TRANS namespace oceanbase { +using namespace oceanbase::share; namespace transaction { ObTxIsolationLevel tx_isolation_from_str(const ObString &s) @@ -839,7 +840,7 @@ int ObTxReadSnapshot::assign(const ObTxReadSnapshot &from) return ret; } -void ObTxReadSnapshot::init_weak_read(const palf::SCN snapshot) +void ObTxReadSnapshot::init_weak_read(const SCN snapshot) { core_.version_ = snapshot; core_.tx_id_.reset(); @@ -850,7 +851,7 @@ void ObTxReadSnapshot::init_weak_read(const palf::SCN snapshot) valid_ = true; } -void ObTxReadSnapshot::init_special_read(const palf::SCN snapshot) +void ObTxReadSnapshot::init_special_read(const SCN snapshot) { core_.version_ = snapshot; core_.tx_id_.reset(); diff --git a/src/storage/tx/ob_trans_define_v4.h b/src/storage/tx/ob_trans_define_v4.h index b22f04c9c..87afe6520 100644 --- a/src/storage/tx/ob_trans_define_v4.h +++ b/src/storage/tx/ob_trans_define_v4.h @@ -193,7 +193,7 @@ typedef ObPair ObTxLSEpochPair; // internal core snapshot for read data struct ObTxSnapshot { - palf::SCN version_; + share::SCN version_; ObTransID tx_id_; int64_t scn_; bool elr_; @@ -223,8 +223,8 @@ struct ObTxReadSnapshot int64_t uncertain_bound_; // for source_ GLOBAL ObSEArray parts_; - void init_weak_read(const palf::SCN snapshot); - void init_special_read(const palf::SCN snapshot); + void init_weak_read(const share::SCN snapshot); + void init_special_read(const share::SCN snapshot); void init_none_read() { valid_ = true; source_ = SRC::NONE; } void init_ls_read(const share::ObLSID &ls_id, const ObTxSnapshot &core); void wait_consistency(); @@ -329,7 +329,7 @@ protected: ObXATransID xid_; // xa info if participant in XA ObTxIsolationLevel isolation_; // isolation level ObTxAccessMode access_mode_; // READ_ONLY | READ_WRITE - palf::SCN snapshot_version_; // snapshot for RR | SERIAL Isolation + share::SCN snapshot_version_; // snapshot for RR | SERIAL Isolation int64_t snapshot_uncertain_bound_; // uncertain bound of @snapshot_version_ int64_t snapshot_scn_; // the time of acquire @snapshot_version_ uint32_t sess_id_; // sesssion id @@ -397,7 +397,7 @@ protected: share::ObLSID coord_id_; // coordinator ID int64_t commit_expire_ts_; // commit operation deadline share::ObLSArray commit_parts_; // participants to do commit - palf::SCN commit_version_; // Tx commit version + share::SCN commit_version_; // Tx commit version int commit_out_; // the commit result /* internal abort cause */ int16_t abort_cause_; // Tx Aborted cause @@ -498,7 +498,7 @@ public: bool is_rdonly() const { return access_mode_ == ObTxAccessMode::RD_ONLY; } int64_t get_op_sn() const { return op_sn_; } int inc_op_sn() { return ++op_sn_; } - palf::SCN get_commit_version() const { return commit_version_; } + share::SCN get_commit_version() const { return commit_version_; } bool contain_savepoint(const ObString &sp); bool is_tx_end() { return is_committed() || is_rollbacked(); @@ -625,7 +625,7 @@ protected: ObTransID tx_id_; ObTxIsolationLevel isolation_; ObTxAccessMode access_mode_; - palf::SCN snapshot_version_; + share::SCN snapshot_version_; int64_t snapshot_uncertain_bound_; uint64_t op_sn_; int64_t alloc_ts_; diff --git a/src/storage/tx/ob_trans_end_trans_callback.h b/src/storage/tx/ob_trans_end_trans_callback.h index 0fe42c7bb..3d65d6f9c 100644 --- a/src/storage/tx/ob_trans_end_trans_callback.h +++ b/src/storage/tx/ob_trans_end_trans_callback.h @@ -28,7 +28,7 @@ struct ObTxCommitCallback public: ObTxCommitCallback() { reset(); } ~ObTxCommitCallback() { reset(); } - int init(ObTransService* txs, const ObTransID tx_id, const int ret, const palf::SCN commit_version) + int init(ObTransService* txs, const ObTransID tx_id, const int ret, const share::SCN commit_version) { txs_ = txs; tx_id_ = tx_id; @@ -52,7 +52,7 @@ public: ObTransService* txs_; ObTransID tx_id_; int ret_; - palf::SCN commit_version_; + share::SCN commit_version_; }; class ObTxCommitCallbackTask : public ObTransTask diff --git a/src/storage/tx/ob_trans_functor.h b/src/storage/tx/ob_trans_functor.h index e25af4d57..93028c233 100644 --- a/src/storage/tx/ob_trans_functor.h +++ b/src/storage/tx/ob_trans_functor.h @@ -183,7 +183,7 @@ private: class SwitchToLeaderFunctor { public: - explicit SwitchToLeaderFunctor(palf::SCN &start_working_ts) + explicit SwitchToLeaderFunctor(share::SCN &start_working_ts) { start_working_ts_ = start_working_ts; @@ -206,7 +206,7 @@ public: } private: - palf::SCN start_working_ts_; + share::SCN start_working_ts_; }; class SwitchToFollowerGracefullyFunctor @@ -260,7 +260,7 @@ private: class ResumeLeaderFunctor { public: - ResumeLeaderFunctor(palf::SCN &start_working_ts) + ResumeLeaderFunctor(share::SCN &start_working_ts) { start_working_ts_ = start_working_ts; @@ -282,13 +282,13 @@ public: } private: - palf::SCN start_working_ts_; + share::SCN start_working_ts_; }; class ReplayTxStartWorkingLogFunctor { public: - ReplayTxStartWorkingLogFunctor(palf::SCN &start_working_ts) + ReplayTxStartWorkingLogFunctor(share::SCN &start_working_ts) { start_working_ts_ = start_working_ts; SET_EXPIRED_LIMIT(100 * 1000 /*100ms*/, 3 * 1000 * 1000 /*3s*/); @@ -309,7 +309,7 @@ public: } private: - palf::SCN start_working_ts_; + share::SCN start_working_ts_; }; class KillTxCtxFunctor @@ -597,7 +597,7 @@ public: { min_prepare_version_.set_max(); } - palf::SCN get_min_prepare_version() const { return min_prepare_version_; } + share::SCN get_min_prepare_version() const { return min_prepare_version_; } OPERATOR_V4(IterateMinPrepareVersionFunctor) { int tmp_ret = common::OB_SUCCESS; @@ -607,7 +607,7 @@ public: TRANS_LOG(WARN, "invalid argument", K(tmp_ret), K(tx_id), "ctx", OB_P(tx_ctx)); } else { bool is_prepared = false; - palf::SCN prepare_version; + share::SCN prepare_version; if (OB_TMP_FAIL(tx_ctx->get_prepare_version_if_prepared(is_prepared, prepare_version))) { TRANS_LOG(WARN, "get prepare version if prepared failed", K(tmp_ret), K(*tx_ctx)); } else if (!is_prepared || prepare_version >= min_prepare_version_) { @@ -619,7 +619,7 @@ public: return (OB_SUCCESS == tmp_ret); } private: - palf::SCN min_prepare_version_; + share::SCN min_prepare_version_; }; class ObGetMinUndecidedLogTsFunctor @@ -630,7 +630,7 @@ public: log_ts_.set_max(); } ~ObGetMinUndecidedLogTsFunctor() {} - palf::SCN get_min_undecided_scn() const { return log_ts_; } + share::SCN get_min_undecided_scn() const { return log_ts_; } OPERATOR_V4(ObGetMinUndecidedLogTsFunctor) { int ret = OB_SUCCESS; @@ -638,7 +638,7 @@ public: ret = OB_INVALID_ARGUMENT; TRANS_LOG(WARN, "invalid argument", K(tx_id), "ctx", OB_P(tx_ctx)); } else { - palf::SCN log_ts = tx_ctx->get_min_undecided_log_ts(); + share::SCN log_ts = tx_ctx->get_min_undecided_log_ts(); if (log_ts_ > log_ts) { log_ts_ = log_ts; } @@ -646,7 +646,7 @@ public: return OB_SUCC(ret); } private: - palf::SCN log_ts_; + share::SCN log_ts_; }; class IterateAllLSTxStatFunctor @@ -792,16 +792,16 @@ public: ret = OB_INVALID_ARGUMENT; } else { ObTxCtxTableInfo ctx_info; - rec_log_ts_ = palf::SCN::min(rec_log_ts_, tx_ctx->get_rec_log_ts()); + rec_log_ts_ = share::SCN::min(rec_log_ts_, tx_ctx->get_rec_log_ts()); } if (OB_SUCCESS == ret) { bool_ret = true; } return bool_ret; } - palf::SCN get_rec_log_ts() { return rec_log_ts_; } + share::SCN get_rec_log_ts() { return rec_log_ts_; } private: - palf::SCN rec_log_ts_; + share::SCN rec_log_ts_; }; class OnTxCtxTableFlushedFunctor @@ -1082,7 +1082,7 @@ public: if (!tx_id.is_valid() || OB_ISNULL(tx_ctx)) { TRANS_LOG(WARN, "invalid argument", K(tx_id), KP(tx_ctx)); } else { - palf::SCN start_scn = tx_ctx->get_start_log_ts(); + share::SCN start_scn = tx_ctx->get_start_log_ts(); if (start_scn < min_start_scn_) { min_start_scn_ = start_scn; } @@ -1091,10 +1091,10 @@ public: return bool_ret; } - palf::SCN get_min_start_scn() { return min_start_scn_; } + share::SCN get_min_start_scn() { return min_start_scn_; } private: - palf::SCN min_start_scn_; + share::SCN min_start_scn_; }; class IteratePartCtxAskSchedulerStatusFunctor diff --git a/src/storage/tx/ob_trans_log.h b/src/storage/tx/ob_trans_log.h index 209f19905..c311cf5ab 100644 --- a/src/storage/tx/ob_trans_log.h +++ b/src/storage/tx/ob_trans_log.h @@ -240,12 +240,12 @@ public: ObTransPreCommitLog() : ObTransLog() {} ~ObTransPreCommitLog() {} - palf::SCN get_publish_version() const { return publish_version_; } + share::SCN get_publish_version() const { return publish_version_; } bool is_valid() const; public: TO_STRING_KV(K_(log_type)); private: - palf::SCN publish_version_; + share::SCN publish_version_; }; class ObTransAbortLog : public ObTransLog @@ -381,7 +381,7 @@ private: bool can_elr_; common::ObAddr proposal_leader_; uint64_t cluster_version_; - palf::SCN snapshot_version_; + share::SCN snapshot_version_; int64_t cur_query_start_time_; int64_t stmt_expired_time_; ObXATransID xid_; diff --git a/src/storage/tx/ob_trans_part_ctx.cpp b/src/storage/tx/ob_trans_part_ctx.cpp index 15e434ed8..89a749cf3 100644 --- a/src/storage/tx/ob_trans_part_ctx.cpp +++ b/src/storage/tx/ob_trans_part_ctx.cpp @@ -214,7 +214,7 @@ void ObPartTransCtx::destroy() int ret = OB_TRANS_UNKNOWN; TRANS_LOG(ERROR, "missing callback scheduler, callback with TRANS_UNKNOWN", K(ret), KPC(this)); // NOTE: callback scheduler may introduce deadlock, need take care - trans_service_->handle_tx_commit_result(trans_id_, OB_TRANS_UNKNOWN, palf::SCN()); + trans_service_->handle_tx_commit_result(trans_id_, OB_TRANS_UNKNOWN, SCN()); FORCE_PRINT_TRACE(tlog_, "[missing callback scheduler] "); } @@ -276,7 +276,7 @@ void ObPartTransCtx::default_init_() mds_cache_.reset(); start_replay_ts_.reset(); is_incomplete_replay_ctx_ = false; - start_working_log_ts_ = palf::SCN::min_scn(); + start_working_log_ts_ = SCN::min_scn(); coord_prepare_info_arr_.reset(); reserve_allocator_.reset(); elr_handler_.reset(); @@ -439,6 +439,34 @@ int ObPartTransCtx::handle_timeout(const int64_t delay) } } + // go to preapre state when recover from redo complete + if (!is_follower_() && !exec_info_.is_dup_tx_ && !is_sub2pc()) { + if (ObTxState::REDO_COMPLETE == get_downstream_state()) { + if (is_local_tx_()) { + if (!is_logging_()) { + if (OB_FAIL(one_phase_commit_())) { + TRANS_LOG(WARN, "two phase commit failed", KR(ret), KPC(this)); + } else { + part_trans_action_ = ObPartTransAction::COMMIT; + } + } + } else { + if (ObTxState::PREPARE > get_upstream_state()) { + bool unused = false; + if (is_2pc_logging()) { + TRANS_LOG(INFO, "committer is under logging", K(ret), K(*this)); + } else if (OB_FAIL(do_prepare(unused))) { + TRANS_LOG(WARN, "do prepare failed", K(ret), K(*this)); + } else { + set_upstream_state(ObTxState::PREPARE); + collected_.reset(); + part_trans_action_ = ObPartTransAction::COMMIT; + } + } + } + } + } + // retry commiting for every node if (!is_follower_() && is_committing_()) { if (is_local_tx_()) { @@ -537,7 +565,7 @@ int ObPartTransCtx::kill(const KillTransArg &arg, ObIArray & if (commit_cb_.is_enabled() && !commit_cb_.is_inited()) { if (exec_info_.scheduler_ == addr_) { ObTxCommitCallback cb; - cb.init(trans_service_, trans_id_, cb_param, palf::SCN()); + cb.init(trans_service_, trans_id_, cb_param, SCN()); if (OB_FAIL(cb_array.push_back(cb))) { TRANS_LOG(WARN, "push commit callback fail", K(ret), KPC(this)); } else { @@ -776,7 +804,7 @@ int ObPartTransCtx::iterate_tx_obj_lock_op(ObLockOpIterator &iter) const return ret; } -bool ObPartTransCtx::need_update_schema_version(const int64_t log_id, const palf::SCN) +bool ObPartTransCtx::need_update_schema_version(const int64_t log_id, const SCN) { // const int64_t restore_snapshot_version = ls_tx_ctx_mgr_->get_restore_snapshot_version(); // const int64_t last_restore_log_id = ls_tx_ctx_mgr_->get_last_restore_log_id(); @@ -789,7 +817,7 @@ bool ObPartTransCtx::need_update_schema_version(const int64_t log_id, const palf return need_update; } -int ObPartTransCtx::trans_replay_abort_(const palf::SCN &final_log_ts) +int ObPartTransCtx::trans_replay_abort_(const SCN &final_log_ts) { int ret = OB_SUCCESS; @@ -802,8 +830,8 @@ int ObPartTransCtx::trans_replay_abort_(const palf::SCN &final_log_ts) return ret; } -int ObPartTransCtx::trans_replay_commit_(const palf::SCN &commit_version, - const palf::SCN &final_log_ts, +int ObPartTransCtx::trans_replay_commit_(const SCN &commit_version, + const SCN &final_log_ts, const uint64_t log_cluster_version, const int64_t checksum) { @@ -845,7 +873,7 @@ int ObPartTransCtx::update_publish_version_(const SCN &publish_version, const bo return ret; } -int ObPartTransCtx::replay_start_working_log(const palf::SCN start_working_ts) +int ObPartTransCtx::replay_start_working_log(const SCN start_working_ts) { int ret = OB_SUCCESS; @@ -901,7 +929,7 @@ int ObPartTransCtx::gts_callback_interrupted(const int errcode) } int ObPartTransCtx::get_gts_callback(const MonotonicTs srr, - const palf::SCN >s, + const SCN >s, const MonotonicTs receive_gts_ts) { TRANS_LOG(DEBUG, "ObPartTransCtx get_gts_callback begin", KPC(this)); @@ -935,16 +963,16 @@ int ObPartTransCtx::get_gts_callback(const MonotonicTs srr, ObTxLogType log_type = ObTxLogType::UNKNOWN; // the same as before prepare mt_ctx_.set_trans_version(gts); - const palf::SCN max_read_ts = trans_service_->get_tx_version_mgr().get_max_read_ts(); + const SCN max_read_ts = trans_service_->get_tx_version_mgr().get_max_read_ts(); // TRANS_LOG(INFO, "get_gts_callback mid", K(*this), K(log_type)); if (is_local_tx_()) { - if (OB_FAIL(ctx_tx_data_.set_commit_version(palf::SCN::max(gts, max_read_ts)))) { + if (OB_FAIL(ctx_tx_data_.set_commit_version(SCN::max(gts, max_read_ts)))) { TRANS_LOG(WARN, "set commit_version failed", K(ret)); } log_type = ObTxLogType::TX_COMMIT_LOG; } else { - const palf::SCN local_prepare_version = palf::SCN::max(gts, max_read_ts); - exec_info_.prepare_version_ = palf::SCN::max(local_prepare_version, exec_info_.prepare_version_); + const SCN local_prepare_version = SCN::max(gts, max_read_ts); + exec_info_.prepare_version_ = SCN::max(local_prepare_version, exec_info_.prepare_version_); log_type = ObTxLogType::TX_PREPARE_LOG; } if (OB_FAIL(submit_log_impl_(log_type))) { @@ -967,7 +995,7 @@ int ObPartTransCtx::get_gts_callback(const MonotonicTs srr, return ret; } -int ObPartTransCtx::gts_elapse_callback(const MonotonicTs srr, const palf::SCN >s) +int ObPartTransCtx::gts_elapse_callback(const MonotonicTs srr, const SCN >s) { int ret = OB_SUCCESS; bool need_revert_ctx = false; @@ -1049,7 +1077,7 @@ int ObPartTransCtx::gts_elapse_callback(const MonotonicTs srr, const palf::SCN & return ret; } -int ObPartTransCtx::get_prepare_version_if_prepared(bool &is_prepared, palf::SCN &prepare_version) +int ObPartTransCtx::get_prepare_version_if_prepared(bool &is_prepared, SCN &prepare_version) { int ret = OB_SUCCESS; ObTxState cur_state = exec_info_.state_; @@ -1235,6 +1263,9 @@ int ObPartTransCtx::recover_tx_ctx_table_info(const ObTxCtxTableInfo &ctx_info) trans_id_ = ctx_info.tx_id_; ls_id_ = ctx_info.ls_id_; exec_info_ = ctx_info.exec_info_; + if (ObTxState::REDO_COMPLETE == get_downstream_state()) { + sub_state_.set_info_log_submitted(); + } exec_info_.multi_data_source_.reset(); if (OB_FAIL(ret)) { // do nothing @@ -1311,15 +1342,15 @@ int ObPartTransCtx::get_tx_ctx_table_info(ObTxCtxTableInfo &info) return ret; } -const palf::SCN ObPartTransCtx::get_rec_log_ts() const +const SCN ObPartTransCtx::get_rec_log_ts() const { CtxLockGuard guard(lock_); return get_rec_log_ts_(); } -const palf::SCN ObPartTransCtx::get_rec_log_ts_() const +const SCN ObPartTransCtx::get_rec_log_ts_() const { - palf::SCN log_ts = palf::SCN::max_scn(); + SCN log_ts = SCN::max_scn(); // Before the checkpoint of the tx ctx table is succeed, we should still use // the prev_log_ts. And after successfully checkpointed, we can use the new @@ -1394,6 +1425,9 @@ int ObPartTransCtx::submit_redo_log(const bool is_freeze) if (!is_tx_committing && !final_log_submitting) { (void)mt_ctx_.merge_multi_callback_lists_for_immediate_logging(); ret = submit_log_impl_(ObTxLogType::TX_REDO_LOG); + if (OB_SUCC(ret) || OB_BLOCK_FROZEN == ret) { + ret = submit_multi_data_source_(); + } try_submit = true; } if (try_submit) { @@ -1490,7 +1524,7 @@ int ObPartTransCtx::on_dist_end_(const bool commit) int64_t start_us, end_us; start_us = end_us = 0; - const palf::SCN trans_version = commit ? ctx_tx_data_.get_commit_version() : palf::SCN::invalid_scn(); + const SCN trans_version = commit ? ctx_tx_data_.get_commit_version() : SCN::invalid_scn(); // Distributed transactions need to wait for the commit log majority successfully before // unlocking. If you want to know the reason, it is in the ::do_dist_commit @@ -1523,7 +1557,7 @@ int ObPartTransCtx::on_success(ObTxLogCb *log_cb) { CtxLockGuard guard(lock_); tg.click(); - const palf::SCN log_ts = log_cb->get_log_ts(); + const SCN log_ts = log_cb->get_log_ts(); const ObTxLogType last_log_type = log_cb->get_last_log_type(); if (is_exiting_) { // maybe because commit log callbacked before redo log, and ctx is already released @@ -1605,7 +1639,7 @@ int ObPartTransCtx::on_success_ops_(ObTxLogCb *log_cb) { common::ObTimeGuard tg("part_ctx::on_success_ops", 100 * 1000); int ret = OB_SUCCESS; - const palf::SCN log_ts = log_cb->get_log_ts(); + const SCN log_ts = log_cb->get_log_ts(); const palf::LSN log_lsn = log_cb->get_lsn(); const ObTxCbArgArray &cb_arg_array = log_cb->get_cb_arg_array(); ObTxBufferNodeArray tmp_array; @@ -1810,7 +1844,7 @@ int ObPartTransCtx::common_on_success_(ObTxLogCb *log_cb) int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - const palf::SCN log_ts = log_cb->get_log_ts(); + const SCN log_ts = log_cb->get_log_ts(); const palf::LSN lsn = log_cb->get_lsn(); const ObTxLogType last_log_type = log_cb->get_last_log_type(); if (log_ts > exec_info_.max_applying_log_ts_) { @@ -1879,12 +1913,12 @@ int ObPartTransCtx::on_failure(ObTxLogCb *log_cb) CtxLockGuard guard(lock_); exec_info_.next_log_entry_no_--; const ObTxLogType log_type = log_cb->get_last_log_type(); - const palf::SCN log_ts = log_cb->get_log_ts(); + const SCN log_ts = log_cb->get_log_ts(); // TODO, dingxi mt_ctx_.sync_log_fail(log_cb->get_callbacks()); log_cb->get_mds_range().range_sync_failed(); if (log_ts == ctx_tx_data_.get_start_log_ts()) { - ctx_tx_data_.set_start_log_ts(palf::SCN()); + ctx_tx_data_.set_start_log_ts(SCN()); } if (ObTxLogTypeChecker::is_state_log(log_type)) { if (is_local_tx_()) { @@ -1921,7 +1955,7 @@ int ObPartTransCtx::on_failure(ObTxLogCb *log_cb) } if (need_callback_scheduler_()) { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(defer_commit_callback_(OB_TRANS_KILLED, palf::SCN()))) { + if (OB_TMP_FAIL(defer_commit_callback_(OB_TRANS_KILLED, SCN()))) { TRANS_LOG(WARN, "notify scheduler txn killed fail", K(tmp_ret), K_(trans_id)); } else { commit_cb_.disable(); @@ -1944,7 +1978,7 @@ int ObPartTransCtx::on_failure(ObTxLogCb *log_cb) return ret; } -int ObPartTransCtx::get_local_max_read_version_(palf::SCN &local_max_read_version) +int ObPartTransCtx::get_local_max_read_version_(SCN &local_max_read_version) { int ret = OB_SUCCESS; local_max_read_version = trans_service_->get_tx_version_mgr().get_max_read_ts(); @@ -1954,7 +1988,7 @@ int ObPartTransCtx::get_local_max_read_version_(palf::SCN &local_max_read_versio return ret; } -int ObPartTransCtx::get_gts_(palf::SCN >s) +int ObPartTransCtx::get_gts_(SCN >s) { int ret = OB_SUCCESS; MonotonicTs receive_gts_ts; @@ -2020,8 +2054,8 @@ int ObPartTransCtx::generate_prepare_version_() int ret = OB_SUCCESS; if (ObTxState::PREPARE > upstream_state_) { - palf::SCN gts = palf::SCN::min_scn(); - palf::SCN local_max_read_version = palf::SCN::min_scn(); + SCN gts = SCN::min_scn(); + SCN local_max_read_version = SCN::min_scn(); bool is_gts_ok = false; bool need_gts = need_request_gts_(); @@ -2066,7 +2100,7 @@ int ObPartTransCtx::generate_commit_version_() { int ret = OB_SUCCESS; if (!ctx_tx_data_.get_commit_version().is_valid()) { - palf::SCN gts; + SCN gts; if (OB_FAIL(get_gts_(gts))) { if (OB_EAGAIN != ret) { TRANS_LOG(WARN, "get gts failed", KR(ret), K(*this)); @@ -2074,8 +2108,8 @@ int ObPartTransCtx::generate_commit_version_() } else { // the same as before prepare mt_ctx_.set_trans_version(gts); - const palf::SCN max_read_ts = trans_service_->get_tx_version_mgr().get_max_read_ts(); - if (OB_FAIL(ctx_tx_data_.set_commit_version(palf::SCN::max(gts, max_read_ts)))) { + const SCN max_read_ts = trans_service_->get_tx_version_mgr().get_max_read_ts(); + if (OB_FAIL(ctx_tx_data_.set_commit_version(SCN::max(gts, max_read_ts)))) { TRANS_LOG(WARN, "set tx data commit version", K(ret)); } TRANS_LOG(DEBUG, "generate_commit_version_", KR(ret), K(gts), K(max_read_ts), K(*this)); @@ -2195,7 +2229,7 @@ int ObPartTransCtx::submit_redo_log_(ObTxLogBlock &log_block, if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = NULL; @@ -2256,7 +2290,7 @@ int ObPartTransCtx::submit_redo_log_() } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = NULL; @@ -2299,7 +2333,7 @@ int ObPartTransCtx::submit_redo_commit_info_log_() } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = NULL; @@ -2349,7 +2383,7 @@ int ObPartTransCtx::submit_redo_commit_info_log_(ObTxLogBlock &log_block, } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = NULL; @@ -2415,7 +2449,7 @@ int ObPartTransCtx::submit_redo_active_info_log_() } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = nullptr; @@ -2439,7 +2473,7 @@ int ObPartTransCtx::submit_redo_active_info_log_() } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = nullptr; @@ -2463,7 +2497,7 @@ int ObPartTransCtx::submit_redo_active_info_log_() } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = nullptr; @@ -2530,7 +2564,7 @@ int ObPartTransCtx::submit_prepare_log_() } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = NULL; @@ -2629,7 +2663,7 @@ int ObPartTransCtx::submit_commit_log_() if (OB_SUCC(ret)) { const uint64_t checksum = (exec_info_.need_checksum_ && !is_incomplete_replay_ctx_ ? mt_ctx_.calc_checksum_all() : 0); - palf::SCN log_commit_version; + SCN log_commit_version; if (!is_local_tx_()) { log_commit_version = ctx_tx_data_.get_commit_version(); if (OB_FAIL(get_prev_log_lsn_(log_block, ObTxLogType::TX_PREPARE_LOG, prev_lsn))) { @@ -2677,7 +2711,7 @@ int ObPartTransCtx::submit_commit_log_() } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = NULL; @@ -2821,7 +2855,7 @@ int ObPartTransCtx::submit_abort_log_() } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = NULL; @@ -2863,7 +2897,7 @@ int ObPartTransCtx::submit_clear_log_() } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = NULL; @@ -2905,7 +2939,7 @@ int ObPartTransCtx::submit_record_log_() } else if (OB_FAIL(acquire_ctx_ref_())) { TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); return_log_cb_(log_cb); log_cb = NULL; @@ -3041,7 +3075,7 @@ int ObPartTransCtx::after_submit_log_(ObTxLogBlock &log_block, } } } - if (OB_SUCC(ret) && OB_FAIL(update_rec_log_ts_(false/*for_replay*/, palf::SCN()))) { + if (OB_SUCC(ret) && OB_FAIL(update_rec_log_ts_(false/*for_replay*/, SCN()))) { TRANS_LOG(WARN, "update rec log ts failed", KR(ret), KPC(log_cb), K(*this)); } if (OB_SUCC(ret) && is_contain(cb_arg_array, ObTxLogType::TX_REDO_LOG)) { @@ -3171,7 +3205,7 @@ int ObPartTransCtx::return_log_cb_(ObTxLogCb *log_cb) return ret; } -int ObPartTransCtx::get_max_submitting_log_info_(palf::LSN &lsn, palf::SCN &log_ts) +int ObPartTransCtx::get_max_submitting_log_info_(palf::LSN &lsn, SCN &log_ts) { int ret = OB_SUCCESS; ObTxLogCb *log_cb = NULL; @@ -3199,7 +3233,7 @@ int ObPartTransCtx::get_prev_log_lsn_(const ObTxLogBlock &log_block, { int ret = OB_SUCCESS; palf::LSN tmp_lsn; - palf::SCN tmp_log_ts; + SCN tmp_log_ts; bool in_same_block = false; if (is_contain(log_block.get_cb_arg_array(), prev_log_type)) { @@ -3354,7 +3388,7 @@ bool ObPartTransCtx::is_leaf() const //***************************** for 4.0 int ObPartTransCtx::check_replay_avaliable_(const palf::LSN &offset, - const palf::SCN ×tamp, + const SCN ×tamp, const int64_t &part_log_no, bool &need_replay) { @@ -3397,7 +3431,7 @@ int ObPartTransCtx::check_replay_avaliable_(const palf::LSN &offset, return ret; } -int ObPartTransCtx::push_repalying_log_ts(const palf::SCN log_ts_ns) +int ObPartTransCtx::push_repalying_log_ts(const SCN log_ts_ns) { int ret = OB_SUCCESS; @@ -3415,7 +3449,7 @@ int ObPartTransCtx::push_repalying_log_ts(const palf::SCN log_ts_ns) return ret; } -int ObPartTransCtx::push_replayed_log_ts(palf::SCN log_ts_ns, const palf::LSN &offset) +int ObPartTransCtx::push_replayed_log_ts(SCN log_ts_ns, const palf::LSN &offset) { int ret = OB_SUCCESS; @@ -3436,7 +3470,7 @@ int ObPartTransCtx::push_replayed_log_ts(palf::SCN log_ts_ns, const palf::LSN &o return ret; } -int ObPartTransCtx::update_replaying_log_no_(const palf::SCN &log_ts_ns, int64_t part_log_no) +int ObPartTransCtx::update_replaying_log_no_(const SCN &log_ts_ns, int64_t part_log_no) { int ret = OB_SUCCESS; @@ -3457,7 +3491,7 @@ int ObPartTransCtx::update_replaying_log_no_(const palf::SCN &log_ts_ns, int64_t } int ObPartTransCtx::check_trans_type_for_replay_(const int32_t &trans_type, - const palf::SCN &commit_log_ts) + const SCN &commit_log_ts) { int ret = OB_SUCCESS; @@ -3472,8 +3506,8 @@ int ObPartTransCtx::check_trans_type_for_replay_(const int32_t &trans_type, } int ObPartTransCtx::replay_update_tx_data_(const bool commit, - const palf::SCN &log_ts, - const palf::SCN &commit_version) + const SCN &log_ts, + const SCN &commit_version) { int ret = OB_SUCCESS; if (commit) { @@ -3508,7 +3542,7 @@ int ObPartTransCtx::replay_update_tx_data_(const bool commit, } int ObPartTransCtx::replace_tx_data_with_backup_(const ObTxDataBackup &backup, - palf::SCN log_ts) + SCN log_ts) { int ret = OB_SUCCESS; if (backup.get_start_log_ts().is_valid()) { @@ -3528,7 +3562,7 @@ void ObPartTransCtx::force_no_need_replay_checksum() exec_info_.need_checksum_ = false; } -void ObPartTransCtx::check_no_need_replay_checksum(const palf::SCN &log_ts) +void ObPartTransCtx::check_no_need_replay_checksum(const SCN &log_ts) { // TODO(handora.qc): How to lock the tx_ctx @@ -3543,7 +3577,7 @@ void ObPartTransCtx::check_no_need_replay_checksum(const palf::SCN &log_ts) int ObPartTransCtx::validate_replay_log_entry_no(bool first_created_ctx, int64_t log_entry_no, - const palf::SCN &log_ts) + const SCN &log_ts) { int ret = OB_SUCCESS; @@ -3613,7 +3647,7 @@ int ObPartTransCtx::check_and_merge_redo_lsns_(const palf::LSN &offset) int ObPartTransCtx::replay_redo_in_ctx(const ObTxRedoLog &redo_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const SCN ×tamp, const int64_t &part_log_no) { int ret = OB_SUCCESS; @@ -3665,7 +3699,7 @@ int ObPartTransCtx::replay_redo_in_ctx(const ObTxRedoLog &redo_log, int ObPartTransCtx::replay_rollback_to(const ObTxRollbackToLog &log, const palf::LSN &offset, - const palf::SCN ×tamp, + const SCN ×tamp, const int64_t &part_log_no) { int ret = OB_SUCCESS; @@ -3719,7 +3753,7 @@ int ObPartTransCtx::replay_rollback_to(const ObTxRollbackToLog &log, int ObPartTransCtx::replay_active_info(const ObTxActiveInfoLog &log, const palf::LSN &offset, - const palf::SCN ×tamp, + const SCN ×tamp, const int64_t &part_log_no) { int ret = OB_SUCCESS; @@ -3775,7 +3809,7 @@ int ObPartTransCtx::replay_active_info(const ObTxActiveInfoLog &log, int ObPartTransCtx::replay_commit_info(const ObTxCommitInfoLog &commit_info_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const SCN ×tamp, const int64_t &part_log_no) { int ret = OB_SUCCESS; @@ -3867,7 +3901,7 @@ int ObPartTransCtx::replay_commit_info(const ObTxCommitInfoLog &commit_info_log, int ObPartTransCtx::replay_prepare(const ObTxPrepareLog &prepare_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const SCN ×tamp, const int64_t &part_log_no) { int ret = OB_SUCCESS; @@ -3935,15 +3969,15 @@ int ObPartTransCtx::replay_prepare(const ObTxPrepareLog &prepare_log, int ObPartTransCtx::replay_commit(const ObTxCommitLog &commit_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const SCN ×tamp, const int64_t &part_log_no, - const palf::SCN &replay_compact_version) + const SCN &replay_compact_version) { int ret = OB_SUCCESS; common::ObTimeGuard timeguard("replay_commit", 10 * 1000); // const int64_t start = ObTimeUtility::fast_current_time(); - const palf::SCN commit_version = commit_log.get_commit_version(); + const SCN commit_version = commit_log.get_commit_version(); bool need_replay = true; CtxTransTableLockGuard guard(lock_, trans_table_seqlock_); @@ -4041,7 +4075,7 @@ int ObPartTransCtx::replay_commit(const ObTxCommitLog &commit_log, int ObPartTransCtx::replay_clear(const ObTxClearLog &clear_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const SCN ×tamp, const int64_t &part_log_no) { int ret = OB_SUCCESS; @@ -4113,7 +4147,7 @@ int ObPartTransCtx::replay_clear(const ObTxClearLog &clear_log, int ObPartTransCtx::replay_abort(const ObTxAbortLog &abort_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const SCN ×tamp, const int64_t &part_log_no) { int ret = OB_SUCCESS; @@ -4126,7 +4160,7 @@ int ObPartTransCtx::replay_abort(const ObTxAbortLog &abort_log, if (OB_FAIL(check_replay_avaliable_(offset, timestamp, part_log_no, need_replay))) { TRANS_LOG(WARN, "check replay available failed", KR(ret), K(offset), K(timestamp), K(*this)); - } else if (OB_FAIL(replay_update_tx_data_(false, timestamp, palf::SCN() /*unused*/))) { + } else if (OB_FAIL(replay_update_tx_data_(false, timestamp, SCN() /*unused*/))) { TRANS_LOG(WARN, "replay update tx data failed", KR(ret), K(*this)); } else if (OB_FAIL(replace_tx_data_with_backup_(abort_log.get_tx_data_backup(), timestamp))) { TRANS_LOG(WARN, "replace tx data with backup failed", KR(ret), K(*this)); @@ -4202,7 +4236,7 @@ int ObPartTransCtx::replay_abort(const ObTxAbortLog &abort_log, int ObPartTransCtx::replay_multi_data_source(const ObTxMultiDataSourceLog &log, const palf::LSN &lsn, - const palf::SCN ×tamp, + const SCN ×tamp, const int64_t &part_log_no) { int ret = OB_SUCCESS; @@ -4274,7 +4308,7 @@ int ObPartTransCtx::replay_multi_data_source(const ObTxMultiDataSourceLog &log, int ObPartTransCtx::replay_record(const ObTxRecordLog &log, const palf::LSN &lsn, - const palf::SCN ×tamp, + const SCN ×tamp, const int64_t &part_log_no) { int ret = OB_SUCCESS; @@ -4305,9 +4339,9 @@ int ObPartTransCtx::replay_record(const ObTxRecordLog &log, return ret; } -const palf::SCN ObPartTransCtx::get_min_undecided_log_ts() const +const SCN ObPartTransCtx::get_min_undecided_log_ts() const { - palf::SCN log_ts; + SCN log_ts; CtxLockGuard guard(lock_); if (!busy_cbs_.is_empty()) { const ObTxLogCb *log_cb = busy_cbs_.get_first(); @@ -4327,7 +4361,7 @@ bool ObPartTransCtx::is_committing_() const || ObPartTransAction::ABORT == part_trans_action_; } -int ObPartTransCtx::switch_to_leader(const palf::SCN &start_working_ts) +int ObPartTransCtx::switch_to_leader(const SCN &start_working_ts) { int ret = OB_SUCCESS; @@ -4455,7 +4489,7 @@ int ObPartTransCtx::switch_to_follower_forcedly(ObIArray &cb TRANS_LOG(WARN, "do local tx abort failed", K(ret)); } else if (need_cb_scheduler) { ObTxCommitCallback cb; - cb.init(trans_service_, trans_id_, OB_TRANS_KILLED, palf::SCN()); + cb.init(trans_service_, trans_id_, OB_TRANS_KILLED, SCN()); if (OB_FAIL(cb_array.push_back(cb))) { TRANS_LOG(WARN, "push back callback failed", K(ret), "context", *this); } @@ -4481,10 +4515,10 @@ int ObPartTransCtx::switch_to_follower_forcedly(ObIArray &cb ObTxCommitCallback cb; // no CommitInfoLog has been submitted, txn must abort if (exec_info_.state_ == ObTxState::INIT && !sub_state_.is_info_log_submitted()) { - cb.init(trans_service_, trans_id_, OB_TRANS_KILLED, palf::SCN()); + cb.init(trans_service_, trans_id_, OB_TRANS_KILLED, SCN()); } else { // otherwise, txn either continue commit or abort, need retry to get final result - cb.init(trans_service_, trans_id_, OB_NOT_MASTER, palf::SCN()); + cb.init(trans_service_, trans_id_, OB_NOT_MASTER, SCN()); } TRANS_LOG(INFO, "switch to follower forcely, notify txn commit result to scheduler", "commit_result", cb.ret_, KPC(this)); @@ -4512,7 +4546,7 @@ int ObPartTransCtx::switch_to_follower_forcedly(ObIArray &cb int ObPartTransCtx::switch_to_follower_gracefully(ObIArray &cb_array) { int ret = OB_SUCCESS; - bool need_switch = false; + bool need_submit_log = false; ObTxLogType log_type = ObTxLogType::TX_ACTIVE_INFO_LOG; common::ObTimeGuard timeguard("switch_to_follower_gracefully", 10 * 1000); CtxLockGuard guard(lock_); @@ -4530,9 +4564,9 @@ int ObPartTransCtx::switch_to_follower_gracefully(ObIArray & if (pending_write_) { TRANS_LOG(INFO, "current tx is executing stmt", K(*this)); mt_ctx_.set_replay(); - need_switch = true; + need_submit_log = true; } else if (!is_committing_()) { - need_switch = true; + need_submit_log = true; } else if (ObPartTransAction::COMMIT == part_trans_action_) { /* NOTE: * - If commitInfoLog has submitted, txn can continue on the new Leader @@ -4545,12 +4579,12 @@ int ObPartTransCtx::switch_to_follower_gracefully(ObIArray & * - detect commit timeout */ if (exec_info_.state_ < ObTxState::REDO_COMPLETE && !sub_state_.is_info_log_submitted()) { - need_switch = true; + need_submit_log = true; log_type = ObTxLogType::TX_COMMIT_INFO_LOG; } if (need_callback_scheduler_()) { ObTxCommitCallback cb; - cb.init(trans_service_, trans_id_, OB_SWITCHING_TO_FOLLOWER_GRACEFULLY, palf::SCN()); + cb.init(trans_service_, trans_id_, OB_SWITCHING_TO_FOLLOWER_GRACEFULLY, SCN()); TRANS_LOG(INFO, "swtich to follower gracefully, notify scheduler retry", KPC(this)); if (OB_FAIL(cb_array.push_back(cb))) { TRANS_LOG(WARN, "push back callback failed", K(ret), "context", *this); @@ -4558,7 +4592,7 @@ int ObPartTransCtx::switch_to_follower_gracefully(ObIArray & } } timeguard.click(); - if (OB_SUCC(ret) && need_switch) { + if (OB_SUCC(ret) && need_submit_log) { // We need merge all callbacklists before submitting active info (void)mt_ctx_.merge_multi_callback_lists_for_changing_leader(); if (OB_FAIL(submit_log_impl_(log_type))) { @@ -4566,12 +4600,17 @@ int ObPartTransCtx::switch_to_follower_gracefully(ObIArray & // and resume leader would be called. // TODO dingxi, improve this logic TRANS_LOG(WARN, "submit active/commit info log failed", KR(ret), K(*this)); - } else if (OB_FAIL(mt_ctx_.commit_to_replay())) { + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(mt_ctx_.commit_to_replay())) { TRANS_LOG(WARN, "commit to replay failed", KR(ret), K(*this)); } else if (OB_FAIL(unregister_timeout_task_())) { TRANS_LOG(WARN, "unregister timeout handler error", KR(ret), KPC(this)); } } + timeguard.click(); if (OB_FAIL(ret)) { state_helper.restore_state(); @@ -4600,7 +4639,7 @@ int ObPartTransCtx::switch_to_follower_gracefully(ObIArray & // at this time, there are 50 ctx in leader state and 50 ctx in follower state. // 2. rollback the change, and would traverse all the 100 ctx, including // the 50 ctx in leader state, for such ctx, do nothing. -int ObPartTransCtx::resume_leader(const palf::SCN &start_working_ts) +int ObPartTransCtx::resume_leader(const SCN &start_working_ts) { int ret = OB_SUCCESS; common::ObTimeGuard timeguard("resume_leader", 10 * 1000); @@ -4671,7 +4710,7 @@ int ObPartTransCtx::check_with_tx_data(ObITxDataCheckFunctor &fn) return ret; } -int ObPartTransCtx::update_rec_log_ts_(bool for_replay, const palf::SCN &rec_log_ts) +int ObPartTransCtx::update_rec_log_ts_(bool for_replay, const SCN &rec_log_ts) { int ret = OB_SUCCESS; @@ -5010,7 +5049,7 @@ int ObPartTransCtx::submit_multi_data_source_(ObTxLogBlock &log_block) TRANS_LOG(ERROR, "acquire ctx ref failed", KR(ret), K(*this)); log_cb = nullptr; } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false))) { + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false))) { TRANS_LOG(WARN, "submit log to clog adapter failed", KR(ret), K(*this)); release_ctx_ref_(); } else if (OB_FAIL(after_submit_log_(log_block, log_cb, NULL))) { @@ -5047,7 +5086,7 @@ int ObPartTransCtx::prepare_mul_data_source_tx_end_(bool is_commit) TRANS_LOG(WARN, "submit multi data souce log failed", K(ret)); } else if (OB_FAIL(gen_total_mds_array_(tmp_array))) { TRANS_LOG(WARN, "copy total mds array failed", K(ret)); - } else if (OB_FAIL(notify_data_source_(NotifyType::TX_END, palf::SCN(), false, + } else if (OB_FAIL(notify_data_source_(NotifyType::TX_END, SCN(), false, tmp_array))) { TRANS_LOG(WARN, "notify data source failed", KR(ret), K(*this)); } @@ -5056,7 +5095,7 @@ int ObPartTransCtx::prepare_mul_data_source_tx_end_(bool is_commit) } int ObPartTransCtx::notify_data_source_(const NotifyType notify_type, - const palf::SCN &log_ts, + const SCN &log_ts, const bool for_replay, const ObTxBufferNodeArray ¬ify_array) { @@ -5119,7 +5158,7 @@ int ObPartTransCtx::register_multi_data_source(const ObTxDataSourceType data_sou if (OB_FAIL(ret)) { mtl_free(ptr); - } else if (OB_FAIL(notify_data_source_(NotifyType::REGISTER_SUCC, palf::SCN(), false, + } else if (OB_FAIL(notify_data_source_(NotifyType::REGISTER_SUCC, SCN(), false, tmp_array))) { if (OB_SUCCESS != (tmp_ret = mds_cache_.rollback_last_mds_node())) { ret = OB_ERR_UNEXPECTED; @@ -5658,7 +5697,7 @@ int ObPartTransCtx::submit_rollback_to_log_(const int64_t from_scn, return_log_cb_(log_cb); log_cb = NULL; } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( - log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false /*nonblock on EAGAIN*/ + log_block.get_buf(), log_block.get_size(), SCN::min_scn(), log_cb, false /*nonblock on EAGAIN*/ ))) { TRANS_LOG(WARN, "submit log fail", K(ret), K(log_block), KPC(this)); return_log_cb_(log_cb); @@ -5728,7 +5767,7 @@ int ObPartTransCtx::tx_keepalive_response_(const int64_t status) } int ObPartTransCtx::insert_into_retain_ctx_mgr_(RetainCause cause, - const palf::SCN &log_ts, palf::LSN lsn) + const SCN &log_ts, palf::LSN lsn) { int ret = OB_SUCCESS; ObMDSRetainCtxFunctor *retain_func_ptr = nullptr; @@ -5929,7 +5968,7 @@ int ObPartTransCtx::on_local_abort_tx_() if (!has_persisted_log_() && OB_FAIL(ctx_tx_data_.set_state(ObTxData::ABORT))) { TRANS_LOG(WARN, "set abort state failed", K(ret)); } else if (OB_FALSE_IT(start_us = ObTimeUtility::fast_current_time())) { - } else if (OB_FAIL(mt_ctx_.trans_end(false, palf::SCN::invalid_scn(), ctx_tx_data_.get_end_log_ts()))) { + } else if (OB_FAIL(mt_ctx_.trans_end(false, SCN::invalid_scn(), ctx_tx_data_.get_end_log_ts()))) { TRANS_LOG(WARN, "trans end error", KR(ret), K(commit_version), "context", *this); } else if (FALSE_IT(end_us = ObTimeUtility::fast_current_time())) { diff --git a/src/storage/tx/ob_trans_part_ctx.h b/src/storage/tx/ob_trans_part_ctx.h index 9e222a722..a61ccceba 100644 --- a/src/storage/tx/ob_trans_part_ctx.h +++ b/src/storage/tx/ob_trans_part_ctx.h @@ -85,7 +85,6 @@ const static int64_t OB_TX_MAX_LOG_CBS = 32; // participant transaction context class ObPartTransCtx : public ObTransCtx, public ObTsCbTask, - public ObTxOnePhaseCommitter, public ObTxCycleTwoPhaseCommitter { friend class ObTransService; @@ -136,12 +135,12 @@ public: const int64_t &request_id); int abort(const int reason); int one_phase_commit_(); - int get_prepare_version_if_prepared(bool &is_prepared, palf::SCN &prepare_version); - const palf::SCN get_commit_version() const { return ctx_tx_data_.get_commit_version(); } + int get_prepare_version_if_prepared(bool &is_prepared, share::SCN &prepare_version); + const share::SCN get_commit_version() const { return ctx_tx_data_.get_commit_version(); } uint64_t hash() const { return trans_id_.hash(); } int gts_callback_interrupted(const int errcode); - int get_gts_callback(const MonotonicTs srr, const palf::SCN >s, const MonotonicTs receive_gts_ts); - int gts_elapse_callback(const MonotonicTs srr, const palf::SCN >s); + int get_gts_callback(const MonotonicTs srr, const share::SCN >s, const MonotonicTs receive_gts_ts); + int gts_elapse_callback(const MonotonicTs srr, const share::SCN >s); MonotonicTs get_stc() const { return stc_; } uint64_t get_tenant_id() const { return tenant_id_; } int64_t get_role_state() const { return role_state_; } @@ -160,7 +159,7 @@ public: int dump_2_text(FILE *fd); public: - int replay_start_working_log(const palf::SCN start_working_ts); + int replay_start_working_log(const share::SCN start_working_ts); int set_trans_app_trace_id_str(const ObString &app_trace_id_str); const ObString &get_trans_app_trace_id_str() const { return trace_info_.get_app_trace_id(); } int check_modify_schema_elapsed(const ObTabletID &tablet_id, @@ -181,7 +180,7 @@ public: inc_update(&(exec_info_.max_submitted_seq_no_), seq_no); } int check_with_tx_data(ObITxDataCheckFunctor &fn); - const palf::SCN get_rec_log_ts() const; + const share::SCN get_rec_log_ts() const; int on_tx_ctx_table_flushed(); int64_t get_applying_log_ts() const; @@ -197,7 +196,7 @@ public: { mt_ctx_.set_table_lock_killed(); } bool is_table_lock_killed() const; bool need_update_schema_version(const int64_t log_id, - const palf::SCN log_ts); + const share::SCN log_ts); share::ObLSID get_ls_id() const { return ls_id_; } @@ -243,12 +242,12 @@ private: void set_prev_record_lsn_(const LogOffSet &prev_record_lsn); int trans_clear_(); int trans_kill_(); - int trans_replay_commit_(const palf::SCN &commit_version, - const palf::SCN &final_log_ts, + int trans_replay_commit_(const share::SCN &commit_version, + const share::SCN &final_log_ts, const uint64_t log_cluster_version, const int64_t checksum); - int trans_replay_abort_(const palf::SCN &final_log_ts); - int update_publish_version_(const palf::SCN &publish_version, const bool for_replay); + int trans_replay_abort_(const share::SCN &final_log_ts); + int update_publish_version_(const share::SCN &publish_version, const bool for_replay); bool can_be_recycled_(); bool need_to_ask_scheduler_status_(); int check_rs_scheduler_is_alive_(bool &is_alive); @@ -278,62 +277,62 @@ public: // for instant logging and freezing int submit_redo_log(const bool is_freeze); - int push_repalying_log_ts(const palf::SCN log_ts_ns); - int push_replayed_log_ts(const palf::SCN log_ts_ns, const palf::LSN &offset); + int push_repalying_log_ts(const share::SCN log_ts_ns); + int push_replayed_log_ts(const share::SCN log_ts_ns, const palf::LSN &offset); int replay_redo_in_ctx(const ObTxRedoLog &redo_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const share::SCN ×tamp, const int64_t &part_log_no); int replay_rollback_to(const ObTxRollbackToLog &log, const palf::LSN &offset, - const palf::SCN ×tamp, + const share::SCN ×tamp, const int64_t &part_log_no); int replay_active_info(const ObTxActiveInfoLog &active_info_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const share::SCN ×tamp, const int64_t &part_log_no); int replay_commit_info(const ObTxCommitInfoLog &commit_info_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const share::SCN ×tamp, const int64_t &part_log_no); int replay_prepare(const ObTxPrepareLog &prepare_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const share::SCN ×tamp, const int64_t &part_log_no); int replay_commit(const ObTxCommitLog &commit_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const share::SCN ×tamp, const int64_t &part_log_no, - const palf::SCN &replay_compact_version); + const share::SCN &replay_compact_version); int replay_clear(const ObTxClearLog &clear_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const share::SCN ×tamp, const int64_t &part_log_no); int replay_abort(const ObTxAbortLog &abort_log, const palf::LSN &offset, - const palf::SCN ×tamp, + const share::SCN ×tamp, const int64_t &part_log_no); int replay_multi_data_source(const ObTxMultiDataSourceLog &log, const palf::LSN &lsn, - const palf::SCN ×tamp, + const share::SCN ×tamp, const int64_t &part_log_no); int replay_record(const ObTxRecordLog &log, const palf::LSN &lsn, - const palf::SCN ×tamp, + const share::SCN ×tamp, const int64_t &part_log_no); void force_no_need_replay_checksum(); - void check_no_need_replay_checksum(const palf::SCN &log_ts); + void check_no_need_replay_checksum(const share::SCN &log_ts); - int validate_replay_log_entry_no(bool first_created_ctx, int64_t log_entry_no, const palf::SCN &log_ts); + int validate_replay_log_entry_no(bool first_created_ctx, int64_t log_entry_no, const share::SCN &log_ts); // return the min log ts of those logs which are submitted but // not callbacked yet, if there is no such log return INT64_MAX - const palf::SCN get_min_undecided_log_ts() const; + const share::SCN get_min_undecided_log_ts() const; // dump and recover tx ctx table using the following functions. // get_tx_ctx_table_info returns OB_TRANS_CTX_NOT_EXIST if the tx ctx table need not to be @@ -344,9 +343,9 @@ public: // leader switch related bool need_callback_scheduler_(); int switch_to_follower_forcedly(ObIArray &cb_array); - int switch_to_leader(const palf::SCN &start_working_ts); + int switch_to_leader(const share::SCN &start_working_ts); int switch_to_follower_gracefully(ObIArray &cb_array); - int resume_leader(const palf::SCN &start_working_ts); + int resume_leader(const share::SCN &start_working_ts); int supplement_undo_actions_if_exist_(); void set_role_state(const bool for_replay) @@ -356,12 +355,12 @@ public: int register_multi_data_source(const ObTxDataSourceType type, const char *buf, const int64_t len); - const palf::SCN get_start_log_ts() + const share::SCN get_start_log_ts() { return ctx_tx_data_.get_start_log_ts(); } - const palf::SCN get_tx_end_log_ts() const + const share::SCN get_tx_end_log_ts() const { return ctx_tx_data_.get_end_log_ts(); } @@ -412,7 +411,7 @@ private: int64_t get_redo_log_no_() const; bool has_persisted_log_() const; - int update_replaying_log_no_(const palf::SCN &log_ts_ns, int64_t part_log_no); + int update_replaying_log_no_(const share::SCN &log_ts_ns, int64_t part_log_no); int check_and_merge_redo_lsns_(const palf::LSN &offset); int try_submit_next_log_(); // redo lsns is stored when submit log, when log fails to majority @@ -447,10 +446,10 @@ private: bool is_committing_() const; // int insert_to_tx_table_(ObTxData *tx_data); int replay_update_tx_data_(const bool commit, - const palf::SCN &log_ts, - const palf::SCN &commit_version); - int replace_tx_data_with_backup_(const ObTxDataBackup &backup, palf::SCN log_ts_ns); - int check_trans_type_for_replay_(const int32_t &trans_type, const palf::SCN &commit_log_ts); + const share::SCN &log_ts, + const share::SCN &commit_version); + int replace_tx_data_with_backup_(const ObTxDataBackup &backup, share::SCN log_ts_ns); + int check_trans_type_for_replay_(const int32_t &trans_type, const share::SCN &commit_log_ts); void set_durable_state_(const ObTxState state) { exec_info_.state_ = state; } @@ -458,7 +457,7 @@ private: { return sub_state_.is_state_log_submitting() || sub_state_.is_gts_waiting(); } int notify_data_source_(const NotifyType type, - const palf::SCN &log_ts, + const share::SCN &log_ts, const bool for_replay, const ObTxBufferNodeArray ¬ify_array); int gen_final_mds_array_(ObTxBufferNodeArray &array, bool is_committing = true) const; @@ -472,15 +471,15 @@ private: ATOMIC_STORE(&retain_cause_, static_cast(RetainCause::UNKOWN)); } - int insert_into_retain_ctx_mgr_(RetainCause cause, const palf::SCN &log_ts, palf::LSN lsn); + int insert_into_retain_ctx_mgr_(RetainCause cause, const share::SCN &log_ts, palf::LSN lsn); int prepare_mul_data_source_tx_end_(bool is_commit); protected: - virtual int get_gts_(palf::SCN >s); + virtual int get_gts_(share::SCN >s); virtual int wait_gts_elapse_commit_version_(bool &need_wait); - virtual int get_local_max_read_version_(palf::SCN &local_max_read_version); - virtual int update_local_max_commit_version_(const palf::SCN &commit_version); + virtual int get_local_max_read_version_(share::SCN &local_max_read_version); + virtual int update_local_max_commit_version_(const share::SCN &commit_version); virtual int check_and_response_scheduler_(int result); private: @@ -489,7 +488,7 @@ private: int prepare_log_cb_(const bool need_final_cb, ObTxLogCb *&log_cb); int get_log_cb_(const bool need_final_cb, ObTxLogCb *&log_cb); int return_log_cb_(ObTxLogCb *log_cb); - int get_max_submitting_log_info_(palf::LSN &lsn, palf::SCN &log_ts); + int get_max_submitting_log_info_(palf::LSN &lsn, share::SCN &log_ts); int get_prev_log_lsn_(const ObTxLogBlock &log_block, ObTxLogType prev_log_type, palf::LSN &lsn); // int init_tx_data_(const share::ObLSID&ls_id, const ObTransID &tx_id); @@ -500,15 +499,15 @@ private: int set_scheduler_(const common::ObAddr &scheduler); int check_replay_avaliable_(const palf::LSN &offset, - const palf::SCN ×tamp, + const share::SCN ×tamp, const int64_t &part_log_no, bool &need_replay); bool is_leader_() const { return TxCtxRoleState::LEADER == role_state_; } // inaccurate state when switch leader bool is_follower_() const { return TxCtxRoleState::FOLLOWER == role_state_; } //inaccurate state when switch leader - int update_rec_log_ts_(bool for_replay, const palf::SCN &rec_log_ts); + int update_rec_log_ts_(bool for_replay, const share::SCN &rec_log_ts); int refresh_rec_log_ts_(); int get_tx_ctx_table_info_(ObTxCtxTableInfo &info); - const palf::SCN get_rec_log_ts_() const; + const share::SCN get_rec_log_ts_() const; // ======================================================== // ======================== C2PC MSG HANDLER BEGIN ======================== @@ -540,10 +539,10 @@ private: int set_2pc_participants_(const share::ObLSArray &participants); int set_2pc_incremental_participants_(const share::ObLSArray &participants); int set_2pc_request_id_(const int64_t request_id); - int update_2pc_prepare_version_(const palf::SCN &prepare_version); + int update_2pc_prepare_version_(const share::SCN &prepare_version); int merge_prepare_log_info_(const ObLSLogInfoArray &info_array); int merge_prepare_log_info_(const ObLSLogInfo &prepare_info); - int set_2pc_commit_version_(const palf::SCN &commit_version); + int set_2pc_commit_version_(const share::SCN &commit_version); int find_participant_id_(const share::ObLSID&participant, uint64_t &participant_id); int post_tx_commit_resp_(const int status); @@ -672,7 +671,7 @@ private: bool is_inited_; memtable::ObMemtableCtx mt_ctx_; uint64_t cluster_id_; - palf::SCN end_log_ts_; + share::SCN end_log_ts_; int64_t stmt_expired_time_; int64_t last_ask_scheduler_status_ts_; @@ -725,8 +724,8 @@ private: // otherwise return rec_log_ts // 6. NB(TODO(handora.qc): Should we maintain it): Requirement is neccessary // that the merge of the tx ctx table must be one at a time - palf::SCN rec_log_ts_; - palf::SCN prev_rec_log_ts_; + share::SCN rec_log_ts_; + share::SCN prev_rec_log_ts_; bool is_ctx_table_merged_; // trace_info_ int64_t role_state_; @@ -737,9 +736,9 @@ private: // | end_log_ts = n+10 | ----------------> | | -------------------> | | --> | (0 | | // +-------------------+ +---------------------------------+ +-------+ +-----------------+ +----------------------+ bool is_incomplete_replay_ctx_; - palf::SCN start_replay_ts_; // replay debug + share::SCN start_replay_ts_; // replay debug - palf::SCN start_working_log_ts_; + share::SCN start_working_log_ts_; int16_t retain_cause_; diff --git a/src/storage/tx/ob_trans_rpc.cpp b/src/storage/tx/ob_trans_rpc.cpp index cef97b9ba..35fe233f8 100644 --- a/src/storage/tx/ob_trans_rpc.cpp +++ b/src/storage/tx/ob_trans_rpc.cpp @@ -44,7 +44,7 @@ int handle_trans_msg_callback(const share::ObLSID &sender_ls_id, const int status, const ObAddr &addr, const int64_t request_id, - const palf::SCN &private_data) + const SCN &private_data) { return MTL(ObTransService *)->handle_trans_msg_callback(sender_ls_id, receiver_ls_id, diff --git a/src/storage/tx/ob_trans_rpc.h b/src/storage/tx/ob_trans_rpc.h index af4cdfe52..4f410df74 100644 --- a/src/storage/tx/ob_trans_rpc.h +++ b/src/storage/tx/ob_trans_rpc.h @@ -62,7 +62,7 @@ private: int64_t send_timestamp_; public: // for ObTxCommitReqMsg, it is commit version - palf::SCN private_data_; + share::SCN private_data_; }; struct ObTxRpcRollbackSPResult @@ -203,7 +203,7 @@ int handle_trans_msg_callback(const share::ObLSID &sender_ls_id, const int status, const ObAddr &receiver_addr, const int64_t request_id, - const palf::SCN &private_data); + const share::SCN &private_data); int handle_sp_rollback_resp(const share::ObLSID &receiver_ls_id, const int64_t epoch, diff --git a/src/storage/tx/ob_trans_service.cpp b/src/storage/tx/ob_trans_service.cpp index 3edcea740..2bf76f8e9 100644 --- a/src/storage/tx/ob_trans_service.cpp +++ b/src/storage/tx/ob_trans_service.cpp @@ -305,7 +305,7 @@ void ObTransService::destroy() } int ObTransService::get_gts_( - palf::SCN &snapshot_version, + SCN &snapshot_version, MonotonicTs &receive_gts_ts, const int64_t trans_expired_time, const int64_t stmt_expire_time, @@ -316,7 +316,7 @@ int ObTransService::get_gts_( // and only get gts once during transaction lifecyle const MonotonicTs request_ts = MonotonicTs::current_time(); const int64_t WAIT_GTS_US = 500; - palf::SCN gts; + SCN gts; MonotonicTs tmp_receive_gts_ts; if (OB_ISNULL(ts_mgr_)) { ret = OB_ERR_UNEXPECTED; @@ -339,8 +339,8 @@ int ObTransService::get_gts_( ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "invalid gts, unexpected error", KR(ret), K(gts)); } else { - const palf::SCN max_commit_ts = tx_version_mgr_.get_max_commit_ts(true); - snapshot_version = palf::SCN::max(max_commit_ts, gts); + const SCN max_commit_ts = tx_version_mgr_.get_max_commit_ts(true); + snapshot_version = SCN::max(max_commit_ts, gts); receive_gts_ts = tmp_receive_gts_ts; } } while (OB_EAGAIN == ret); @@ -511,7 +511,7 @@ void ObTransService::handle(void *task) UNUSED(ret); //make compiler happy } -int ObTransService::get_ls_min_uncommit_prepare_version(const ObLSID &ls_id, palf::SCN &min_prepare_version) +int ObTransService::get_ls_min_uncommit_prepare_version(const ObLSID &ls_id, SCN &min_prepare_version) { int ret = OB_SUCCESS; @@ -535,7 +535,7 @@ int ObTransService::get_ls_min_uncommit_prepare_version(const ObLSID &ls_id, pal return ret; } -int ObTransService::get_min_undecided_log_ts(const ObLSID &ls_id, palf::SCN &log_ts) +int ObTransService::get_min_undecided_log_ts(const ObLSID &ls_id, SCN &log_ts) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -597,7 +597,7 @@ int ObTransService::remove_callback_for_uncommited_txn(memtable::ObMemtable* mt) * @pkey : not NULL if this is a single local partition stmt, * will try local publish version */ -int ObTransService::get_weak_read_snapshot(const uint64_t tenant_id, palf::SCN &snapshot_version) +int ObTransService::get_weak_read_snapshot(const uint64_t tenant_id, SCN &snapshot_version) { int ret = OB_SUCCESS; ObIWeakReadService *wrs = GCTX.weak_read_service_; @@ -680,8 +680,8 @@ int ObTransService::register_mds_into_tx(ObTxDesc &tx_desc, TRANS_LOG(WARN, "invalid argument", KR(ret), K(tx_desc), K(ls_id), K(type), KP(buf), K(buf_len)); } else if (!tx_desc.is_tx_active()) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(ERROR, "txn must in active for register", K(ret)); + ret = OB_TRANS_IS_EXITING; + TRANS_LOG(WARN, "txn must in active for register", K(ret)); } else if (OB_ISNULL(rpc_proxy_)) { ret = OB_NOT_INIT; TRANS_LOG(WARN, "rpc proxy not inited", KR(ret), K(tx_desc), K(ls_id), K(type)); @@ -847,7 +847,7 @@ int ObTransService::register_mds_into_ctx(ObTxDesc &tx_desc, return ret; } -int ObTransService::get_max_commit_version(palf::SCN &commit_version) const +int ObTransService::get_max_commit_version(SCN &commit_version) const { int ret = OB_SUCCESS; if (IS_NOT_INIT) { diff --git a/src/storage/tx/ob_trans_service.h b/src/storage/tx/ob_trans_service.h index f34847cb3..1cf3a031a 100644 --- a/src/storage/tx/ob_trans_service.h +++ b/src/storage/tx/ob_trans_service.h @@ -165,10 +165,10 @@ public: virtual void handle(void *task) override; public: int check_trans_partition_leader_unsafe(const share::ObLSID &ls_id, bool &is_leader); - int get_weak_read_snapshot(const uint64_t tenant_id, palf::SCN &snapshot_version); + int get_weak_read_snapshot(const uint64_t tenant_id, share::SCN &snapshot_version); int calculate_trans_cost(const ObTransID &tid, uint64_t &cost); - int get_ls_min_uncommit_prepare_version(const share::ObLSID &ls_id, palf::SCN &min_prepare_version); - int get_min_undecided_log_ts(const share::ObLSID &ls_id, palf::SCN &log_ts); + int get_ls_min_uncommit_prepare_version(const share::ObLSID &ls_id, share::SCN &min_prepare_version); + int get_min_undecided_log_ts(const share::ObLSID &ls_id, share::SCN &log_ts); //get the memory used condition of transaction module int iterate_trans_memory_stat(ObTransMemStatIterator &mem_stat_iter); int dump_elr_statistic(); @@ -207,7 +207,7 @@ private: private: int handle_redo_sync_task_(ObDupTableRedoSyncTask *task, bool &need_release_task); int handle_dup_pre_commit_task_(ObPreCommitTask *task, bool &need_release_task); - int get_gts_(palf::SCN &snapshot_version, + int get_gts_(share::SCN &snapshot_version, MonotonicTs &receive_gts_ts, const int64_t trans_expired_time, const int64_t stmt_expired_time, @@ -221,7 +221,7 @@ public: ObITxCallback *endTransCb, const bool is_rollback, const int64_t expire_ts); - int get_max_commit_version(palf::SCN &commit_version) const; + int get_max_commit_version(share::SCN &commit_version) const; #include "ob_trans_service_v4.h" private: static const int64_t END_STMT_MORE_TIME_US = 100 * 1000; diff --git a/src/storage/tx/ob_trans_service_v4.cpp b/src/storage/tx/ob_trans_service_v4.cpp index 41674972a..bd05ecce4 100644 --- a/src/storage/tx/ob_trans_service_v4.cpp +++ b/src/storage/tx/ob_trans_service_v4.cpp @@ -44,6 +44,7 @@ namespace oceanbase { namespace transaction { using namespace memtable; +using namespace share; int ObTransService::create_ls(const share::ObLSID &ls_id, ObLS &ls, @@ -182,7 +183,7 @@ int ObTransService::acquire_tx(const char* buf, int ObTransService::do_commit_tx_(ObTxDesc &tx, const int64_t expire_ts, ObITxCallback &cb, - palf::SCN &commit_version) + SCN &commit_version) { int ret = OB_SUCCESS; ObTxPart *coord = NULL; @@ -354,7 +355,7 @@ int ObTransService::handle_tx_commit_timeout(ObTxDesc &tx, const int64_t delay) */ int ObTransService::handle_tx_commit_result(const ObTransID &tx_id, const int result, - const palf::SCN commit_version) + const SCN commit_version) { int ret = OB_SUCCESS; ObTxDesc *tx = NULL; @@ -398,7 +399,7 @@ int ObTransService::handle_tx_commit_result(const ObTransID &tx_id, */ int ObTransService::handle_tx_commit_result_(ObTxDesc &tx, const int result, - const palf::SCN commit_version) + const SCN commit_version) { int ret = OB_SUCCESS; bool commit_fin = true; @@ -936,7 +937,8 @@ int ObTransService::get_read_store_ctx(const ObTxReadSnapshot &snapshot, tx_table_guard, snapshot.core_, store_ctx.timeout_, - lock_timeout + lock_timeout, + snapshot.is_weak_read() ); update_max_read_ts_(tenant_id_, ls_id, snapshot.core_.version_); } @@ -947,7 +949,7 @@ int ObTransService::get_read_store_ctx(const ObTxReadSnapshot &snapshot, -int ObTransService::get_read_store_ctx(const palf::SCN snapshot_version, +int ObTransService::get_read_store_ctx(const SCN snapshot_version, const int64_t lock_timeout, ObStoreCtx &store_ctx) { @@ -1108,6 +1110,7 @@ int ObTransService::create_tx_ctx_(const share::ObLSID &ls_id, { int ret = OB_SUCCESS; bool existed = false; + int64_t epoch = 0; ObTxCreateArg arg(tx.can_elr_, /* can_elr */ false, /* for_replay */ tx.tenant_id_, @@ -1201,17 +1204,17 @@ int ObTransService::revert_store_ctx(storage::ObStoreCtx &store_ctx) * used to validate specified snapshot version * precondition: version <= current gts value */ -int ObTransService::validate_snapshot_version_(const palf::SCN snapshot, +int ObTransService::validate_snapshot_version_(const SCN snapshot, const int64_t expire_ts, ObLS &ls) { int ret = OB_SUCCESS; - const palf::SCN ls_weak_read_ts = ls.get_ls_wrs_handler()->get_ls_weak_read_ts(); + const SCN ls_weak_read_ts = ls.get_ls_wrs_handler()->get_ls_weak_read_ts(); if (snapshot <= tx_version_mgr_.get_max_commit_ts(false) || snapshot <= tx_version_mgr_.get_max_read_ts() || snapshot <= ls_weak_read_ts) { } else { - palf::SCN gts; + SCN gts; const MonotonicTs stc_ahead = MonotonicTs::current_time() - MonotonicTs(GCONF._ob_get_gts_ahead_interval); MonotonicTs tmp_receive_gts_ts(0); do { @@ -1261,7 +1264,7 @@ int ObTransService::validate_snapshot_version_(const palf::SCN snapshot, * if so, update replica_readable_snapshot = v and return OK, otherwise * 4. return OB_REPLICA_NOT_READABLE */ -int ObTransService::check_replica_readable_(const palf::SCN snapshot, +int ObTransService::check_replica_readable_(const SCN snapshot, const bool elr, const ObTxReadSnapshot::SRC src, const share::ObLSID &ls_id, @@ -1271,7 +1274,7 @@ int ObTransService::check_replica_readable_(const palf::SCN snapshot, int ret = OB_SUCCESS; bool leader = false; int64_t epoch = 0; - palf::SCN ls_weak_read_ts = ls.get_ls_wrs_handler()->get_ls_weak_read_ts(); + SCN ls_weak_read_ts = ls.get_ls_wrs_handler()->get_ls_weak_read_ts(); bool readable = snapshot <= ls_weak_read_ts; if (!readable) { if (OB_FAIL(ls.get_tx_svr()->get_tx_ls_log_adapter()->get_role(leader, epoch))) { @@ -1296,7 +1299,7 @@ int ObTransService::check_replica_readable_(const palf::SCN snapshot, int ObTransService::wait_follower_readable_(ObLS &ls, const int64_t expire_ts, - const palf::SCN snapshot) + const SCN snapshot) { int ret = OB_REPLICA_NOT_READABLE; int64_t compare_timeout = 0; @@ -1375,11 +1378,12 @@ int ObTransService::abort_participants_(const ObTxDesc &tx_desc) } int ObTransService::acquire_local_snapshot_(const share::ObLSID &ls_id, - palf::SCN &snapshot) + SCN &snapshot) { int ret = OB_SUCCESS; + int64_t epoch = 0; bool leader = false; - palf::SCN snapshot0; + SCN snapshot0; ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; if (OB_FAIL(tx_ctx_mgr_.get_ls_tx_ctx_mgr(ls_id, ls_tx_ctx_mgr))) { TRANS_LOG(WARN, "get ls_tx_ctx_mgr fail", K(ret), K(ls_id)); @@ -1404,7 +1408,7 @@ int ObTransService::acquire_local_snapshot_(const share::ObLSID &ls_id, int ObTransService::sync_acquire_global_snapshot_(ObTxDesc &tx, const int64_t expire_ts, - palf::SCN &snapshot, + SCN &snapshot, int64_t &uncertain_bound) { int ret = OB_SUCCESS; @@ -1435,7 +1439,7 @@ int ObTransService::sync_acquire_global_snapshot_(ObTxDesc &tx, int ObTransService::acquire_global_snapshot__(const int64_t expire_ts, const int64_t gts_ahead, - palf::SCN &snapshot, + SCN &snapshot, int64_t &uncertain_bound, ObFunction interrupt_checker) { @@ -1485,14 +1489,19 @@ int ObTransService::batch_post_tx_msg_(ObTxRollbackSPMsg &msg, { int ret = OB_SUCCESS; int last_ret = OB_SUCCESS; + const ObTxDesc *tx_ptr = msg.tx_ptr_; ARRAY_FOREACH_NORET(list, idx) { auto &p = list.at(idx); msg.receiver_ = p.left_; msg.epoch_ = p.right_; + if (msg.epoch_ > 0) { + msg.tx_ptr_ = NULL; + } if (OB_FAIL(rpc_->post_msg(p.left_, msg))) { TRANS_LOG(WARN, "post msg falied", K(ret), K(msg), K(p)); last_ret = ret; } + msg.tx_ptr_ = tx_ptr; } return last_ret; } @@ -1563,7 +1572,7 @@ int ObTransService::handle_trans_commit_request(ObTxCommitMsg &msg, ObTransRpcResult &result) { int ret = OB_SUCCESS; - palf::SCN commit_version; + SCN commit_version; if (OB_FAIL(local_ls_commit_tx_(msg.tx_id_, msg.receiver_, msg.parts_, @@ -1592,7 +1601,7 @@ int ObTransService::local_ls_commit_tx_(const ObTransID &tx_id, const int64_t &expire_ts, const common::ObString &app_trace_info, const int64_t &request_id, - palf::SCN &commit_version) + SCN &commit_version) { int ret = OB_SUCCESS; MonotonicTs commit_time = MonotonicTs::current_time(); @@ -1635,7 +1644,7 @@ int ObTransService::local_ls_commit_tx_(const ObTransID &tx_id, int ObTransService::get_tx_state_from_tx_table_(const share::ObLSID &lsid, const ObTransID &tx_id, int &state, - palf::SCN &commit_version) + SCN &commit_version) { int ret = OB_SUCCESS; ObTxTableGuard tx_table_guard; @@ -1677,45 +1686,22 @@ int ObTransService::handle_trans_abort_request(ObTxAbortMsg &abort_req, ObTransR return ret; } -int ObTransService::create_tx_ctx_(ObTxRollbackSPMsg &msg, ObPartTransCtx *&ctx) -{ - int ret = OB_SUCCESS; - bool existed = false; - ObTxCreateArg arg(msg.can_elr_, /* can_elr */ - false, /* for_replay */ - msg.tenant_id_, - msg.tx_id_, - msg.receiver_, - msg.cluster_id_, - msg.cluster_version_, - msg.session_id_, /*session_id*/ - msg.tx_addr_, - msg.tx_expire_ts_, - this); - ret = tx_ctx_mgr_.create_tx_ctx(arg, existed, ctx); - if (OB_FAIL(ret)) { - TRANS_LOG(WARN, "create tx ctx fail", K(ret), K(msg), K(arg)); - ctx = NULL; - } - TRANS_LOG(TRACE, "create tx ctx for savepoint rollback", K(ret), K(msg), K(arg)); - return ret; -} - int ObTransService::handle_sp_rollback_request(ObTxRollbackSPMsg &msg, obrpc::ObTxRpcRollbackSPResult &result) { int ret = OB_SUCCESS; int64_t ctx_born_epoch = -1; - ObFunction create_tx_ctx_func = [this, &msg](ObPartTransCtx *&ctx) -> int { - return this->create_tx_ctx_(msg, ctx); - }; ret = ls_rollback_to_savepoint_(msg.tx_id_, msg.receiver_, msg.epoch_, msg.op_sn_, msg.savepoint_, ctx_born_epoch, - create_tx_ctx_func); + msg.tx_ptr_); + if (OB_NOT_NULL(msg.tx_ptr_)) { + ob_free((void*)msg.tx_ptr_); + msg.tx_ptr_ = NULL; + } result.status_ = ret; result.addr_ = self_; result.born_epoch_ = ctx_born_epoch; @@ -1897,7 +1883,7 @@ int ObTransService::handle_trans_msg_callback(const share::ObLSID &sender_ls_id, const int status, const ObAddr &receiver_addr, const int64_t request_id, - const palf::SCN &private_data) + const SCN &private_data) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!is_running_)) { @@ -1966,7 +1952,7 @@ int ObTransService::handle_trans_msg_callback(const share::ObLSID &sender_ls_id, int ObTransService::update_max_read_ts_(const uint64_t tenant_id, const share::ObLSID &lsid, - const palf::SCN ts) + const SCN ts) { int ret = OB_SUCCESS; tx_version_mgr_.update_max_read_ts(ts); @@ -2047,7 +2033,7 @@ bool ObTransService::commit_need_retry_(const int ret) return OB_NOT_MASTER == ret || OB_BLOCK_FROZEN == ret || OB_TX_NOLOGCB == ret || OB_EAGAIN == ret; } -int ObTransService::get_min_uncommit_tx_prepare_version(const share::ObLSID& ls_id, palf::SCN &min_prepare_version) +int ObTransService::get_min_uncommit_tx_prepare_version(const share::ObLSID& ls_id, SCN &min_prepare_version) { int ret = OB_SUCCESS; @@ -2455,7 +2441,7 @@ int ObTransService::sub_prepare_local_ls_(const ObTransID &tx_id, TRANS_LOG(WARN, "get coordinator context fail", K(ret), K(tx_id), K(coord)); if (OB_TRANS_CTX_NOT_EXIST == ret) { int tx_state; - palf::SCN commit_version; + SCN commit_version; if (OB_FAIL(get_tx_state_from_tx_table_(coord, tx_id, tx_state, commit_version))) { TRANS_LOG(WARN, "get tx state from tx table fail", K(ret), K(coord), K(tx_id)); if (OB_ENTRY_NOT_EXIST == ret) { @@ -2641,7 +2627,7 @@ int ObTransService::sub_end_tx_local_ls_(const ObTransID &tx_id, TRANS_LOG(WARN, "fail to get coordinator tx context", K(ret), K(tx_id), K(coord)); if (OB_TRANS_CTX_NOT_EXIST == ret) { int tx_state; - palf::SCN commit_version; + SCN commit_version; if (OB_FAIL(get_tx_state_from_tx_table_(coord, tx_id, tx_state, commit_version))) { TRANS_LOG(WARN, "get tx state from tx table fail", K(ret), K(coord), K(tx_id)); if (OB_ENTRY_NOT_EXIST == ret) { diff --git a/src/storage/tx/ob_trans_service_v4.h b/src/storage/tx/ob_trans_service_v4.h index 5ea334249..a5922486b 100644 --- a/src/storage/tx/ob_trans_service_v4.h +++ b/src/storage/tx/ob_trans_service_v4.h @@ -72,7 +72,7 @@ int get_read_store_ctx(const ObTxReadSnapshot &snapshot, const bool read_latest, const int64_t lock_timeout, ObStoreCtx &store_ctx); -int get_read_store_ctx(const palf::SCN snapshot_version, +int get_read_store_ctx(const share::SCN snapshot_version, const int64_t lock_timeout, ObStoreCtx &store_ctx); int get_write_store_ctx(ObTxDesc &tx, @@ -103,7 +103,7 @@ int handle_trans_msg_callback(const share::ObLSID &sender_ls_id, const int status, const ObAddr &receiver_addr, const int64_t request_id, - const palf::SCN &private_data); + const share::SCN &private_data); int handle_trans_keepalive(const ObTxKeepaliveMsg &msg, obrpc::ObTransRpcResult &result); int handle_trans_keepalive_response(const ObTxKeepaliveRespMsg &msg, obrpc::ObTransRpcResult &result); int handle_tx_batch_req(int type, const char* buf, int32_t size, const bool need_check_leader = true); @@ -111,11 +111,11 @@ int refresh_location_cache(const share::ObLSID ls); int handle_tx_commit_timeout(ObTxDesc &tx, const int64_t delay); int handle_tx_commit_result(const ObTransID &tx_id, const int result, - const palf::SCN commit_version = palf::SCN()); + const share::SCN commit_version = share::SCN()); ObTxCtxMgr &get_tx_ctx_mgr() { return tx_ctx_mgr_; } -int get_min_uncommit_tx_prepare_version(const share::ObLSID& ls_id, palf::SCN &min_prepare_version); +int get_min_uncommit_tx_prepare_version(const share::ObLSID& ls_id, share::SCN &min_prepare_version); int kill_all_tx(const share::ObLSID &ls_id, const KillTransArg &arg, bool &is_all_tx_cleaned_up); @@ -173,7 +173,7 @@ int rollback_savepoint_slowpath_(ObTxDesc &tx, int create_tx_ctx_(const share::ObLSID &ls_id, const ObTxDesc &tx, ObPartTransCtx *&ctx); -int create_tx_ctx_(ObTxRollbackSPMsg &msg, ObPartTransCtx *&ctx); + int create_tx_ctx_(const share::ObLSID &ls_id, ObLS *ls, const ObTxDesc &tx, @@ -190,10 +190,10 @@ int get_tx_ctx_(const share::ObLSID &ls_id, int revert_tx_ctx_(ObLS *ls, ObPartTransCtx *ctx); int revert_tx_ctx_(ObPartTransCtx *ctx); -int validate_snapshot_version_(const palf::SCN snapshot, +int validate_snapshot_version_(const share::SCN snapshot, const int64_t expire_ts, ObLS &ls); -int check_replica_readable_(const palf::SCN snapshot, +int check_replica_readable_(const share::SCN snapshot, const bool elr, const ObTxReadSnapshot::SRC src, const share::ObLSID &ls_id, @@ -201,14 +201,14 @@ int check_replica_readable_(const palf::SCN snapshot, ObLS &ls); int build_tx_commit_msg_(const ObTxDesc &tx, ObTxCommitMsg &msg); int abort_participants_(const ObTxDesc &tx_desc); -int acquire_local_snapshot_(const share::ObLSID &ls_id, palf::SCN &snapshot); +int acquire_local_snapshot_(const share::ObLSID &ls_id, share::SCN &snapshot); int sync_acquire_global_snapshot_(ObTxDesc &tx, const int64_t expire_ts, - palf::SCN &snapshot, + share::SCN &snapshot, int64_t &uncertain_bound); int acquire_global_snapshot__(const int64_t expire_ts, const int64_t gts_ahead, - palf::SCN &snapshot, + share::SCN &snapshot, int64_t &uncertain_bound, ObFunction interrupt_checker); int batch_post_tx_msg_(ObTxRollbackSPMsg &msg, const ObIArray &pl); @@ -223,17 +223,17 @@ void handle_orphan_2pc_msg_(const ObTxMsg &msg, const bool need_check_leader); int update_max_read_ts_(const uint64_t tenant_id, const share::ObLSID &lsid, - const palf::SCN ts); + const share::SCN ts); int do_commit_tx_(ObTxDesc &tx, const int64_t expire_ts, ObITxCallback &cb, - palf::SCN &commit_version); + share::SCN &commit_version); int do_commit_tx_slowpath_(ObTxDesc &tx, const int64_t expire_ts); int register_commit_retry_task_(ObTxDesc &tx, const int64_t max_delay = INT64_MAX); int unregister_commit_retry_task_(ObTxDesc &tx); int handle_tx_commit_result_(ObTxDesc &tx, const int result, - const palf::SCN commit_version = palf::SCN()); + const share::SCN commit_version = share::SCN()); int decide_tx_commit_info_(ObTxDesc &tx, ObTxPart *&coord); int local_ls_commit_tx_(const ObTransID &tx_id, const share::ObLSID &coord, @@ -241,11 +241,11 @@ int local_ls_commit_tx_(const ObTransID &tx_id, const int64_t &expire_ts, const common::ObString &app_trace_info, const int64_t &request_id, - palf::SCN &commit_version); + share::SCN &commit_version); int get_tx_state_from_tx_table_(const share::ObLSID &lsid, const ObTransID &tx_id, int &state, - palf::SCN &commit_version); + share::SCN &commit_version); int gen_trans_id_(ObTransID &trans_id); bool commit_need_retry_(const int ret); // for xa @@ -281,7 +281,7 @@ int ls_rollback_to_savepoint_(const ObTransID &tx_id, const int64_t op_sn, const int64_t savepoint, int64_t &ctx_born_epoch, - ObFunction &func, + const ObTxDesc *tx, int64_t expire_ts = -1); int sync_rollback_savepoint__(ObTxDesc &tx, ObTxRollbackSPMsg &msg, @@ -315,7 +315,7 @@ int get_tx_table_guard_(ObLS *ls, void fetch_cflict_tx_ids_from_mem_ctx_to_desc_(memtable::ObMvccAccessCtx &acc_ctx); int wait_follower_readable_(ObLS &ls, const int64_t expire_ts, - const palf::SCN snapshot); + const share::SCN snapshot); // include tx api refacored for future public: #include "ob_tx_api.h" diff --git a/src/storage/tx/ob_trans_submit_log_cb.h b/src/storage/tx/ob_trans_submit_log_cb.h index 2b09bc303..9f4f49307 100644 --- a/src/storage/tx/ob_trans_submit_log_cb.h +++ b/src/storage/tx/ob_trans_submit_log_cb.h @@ -53,15 +53,15 @@ public: void reset(); void reuse(); public: - int set_log_ts(const palf::SCN &log_ts); - const palf::SCN &get_log_ts() const { return log_ts_; } + int set_log_ts(const share::SCN &log_ts); + const share::SCN &get_log_ts() const { return log_ts_; } int set_lsn(const palf::LSN &lsn); palf::LSN get_lsn() const { return lsn_; } void set_submit_ts(const int64_t submit_ts) { submit_ts_ = submit_ts; } int64_t get_submit_ts() const { return submit_ts_; } TO_STRING_KV(K_(log_ts), K_(lsn), K_(submit_ts)); protected: - palf::SCN log_ts_; + share::SCN log_ts_; palf::LSN lsn_; int64_t submit_ts_; }; diff --git a/src/storage/tx/ob_ts_mgr.cpp b/src/storage/tx/ob_ts_mgr.cpp index faf1b7350..4cf7ac23a 100644 --- a/src/storage/tx/ob_ts_mgr.cpp +++ b/src/storage/tx/ob_ts_mgr.cpp @@ -14,6 +14,7 @@ #include "share/ob_errno.h" #include "share/ob_define.h" #include "share/ob_cluster_version.h" +#include "share/scn.h" #include "ob_trans_event.h" #include "share/schema/ob_multi_version_schema_service.h" #include "share/schema/ob_schema_getter_guard.h" @@ -724,7 +725,7 @@ int ObTsMgr::update_gts(const uint64_t tenant_id, const int64_t gts, bool &updat return ret; } -int ObTsMgr::get_gts(const uint64_t tenant_id, ObTsCbTask *task, palf::SCN &scn) +int ObTsMgr::get_gts(const uint64_t tenant_id, ObTsCbTask *task, SCN &scn) { int ret = OB_SUCCESS; int64_t gts = 0;//need be invalid value for SCN @@ -778,7 +779,7 @@ int ObTsMgr::get_gts(const uint64_t tenant_id, ObTsCbTask *task, palf::SCN &scn) int ObTsMgr::get_gts(const uint64_t tenant_id, const MonotonicTs stc, ObTsCbTask *task, - palf::SCN &scn, + SCN &scn, MonotonicTs &receive_gts_ts) { int ret = OB_SUCCESS; @@ -834,7 +835,7 @@ int ObTsMgr::get_gts(const uint64_t tenant_id, int ObTsMgr::get_ts_sync(const uint64_t tenant_id, const int64_t timeout_us, - palf::SCN &scn, + SCN &scn, bool &is_external_consistent) { int ret = OB_SUCCESS; @@ -993,7 +994,7 @@ bool ObTsMgr::is_external_consistent(const uint64_t tenant_id) return bool_ret; } -int ObTsMgr::wait_gts_elapse(const uint64_t tenant_id, const palf::SCN &scn, +int ObTsMgr::wait_gts_elapse(const uint64_t tenant_id, const SCN &scn, ObTsCbTask *task, bool &need_wait) { const int64_t start = ObTimeUtility::fast_current_time(); @@ -1045,7 +1046,7 @@ int ObTsMgr::wait_gts_elapse(const uint64_t tenant_id, const palf::SCN &scn, return ret; } -int ObTsMgr::wait_gts_elapse(const uint64_t tenant_id, const palf::SCN &scn) +int ObTsMgr::wait_gts_elapse(const uint64_t tenant_id, const SCN &scn) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_ts_mgr.h b/src/storage/tx/ob_ts_mgr.h index 3d40a6a4a..5b8d1afd1 100644 --- a/src/storage/tx/ob_ts_mgr.h +++ b/src/storage/tx/ob_ts_mgr.h @@ -24,7 +24,6 @@ #include "share/ob_errno.h" #include "share/ob_thread_pool.h" #include "share/lock/ob_qsync_lock.h" -#include "logservice/palf/scn.h" #include "ob_gts_source.h" #include "ob_gts_define.h" #include "ob_ts_worker.h" @@ -54,7 +53,7 @@ namespace frame class ObReqTransport; } } -namespace palf +namespace share { class SCN; } @@ -71,8 +70,8 @@ public: ObTsCbTask() {} virtual ~ObTsCbTask() {} virtual int gts_callback_interrupted(const int errcode) = 0; - virtual int get_gts_callback(const MonotonicTs srr, const palf::SCN >s, const MonotonicTs receive_gts_ts) = 0; - virtual int gts_elapse_callback(const MonotonicTs srr, const palf::SCN >s) = 0; + virtual int get_gts_callback(const MonotonicTs srr, const share::SCN >s, const MonotonicTs receive_gts_ts) = 0; + virtual int gts_elapse_callback(const MonotonicTs srr, const share::SCN >s) = 0; virtual MonotonicTs get_stc() const = 0; virtual uint64_t hash() const = 0; virtual uint64_t get_tenant_id() const = 0; @@ -89,11 +88,11 @@ public: virtual int get_gts(const uint64_t tenant_id, const MonotonicTs stc, ObTsCbTask *task, - palf::SCN &scn, + share::SCN &scn, MonotonicTs &receive_gts_ts) = 0; - virtual int get_gts(const uint64_t tenant_id, ObTsCbTask *task, palf::SCN &scn) = 0; + virtual int get_gts(const uint64_t tenant_id, ObTsCbTask *task, share::SCN &scn) = 0; virtual int get_ts_sync(const uint64_t tenant_id, const int64_t timeout_ts, - palf::SCN &scn, bool &is_external_consistent) = 0; + share::SCN &scn, bool &is_external_consistent) = 0; /* virtual int get_local_trans_version(const uint64_t tenant_id, const MonotonicTs stc, @@ -104,9 +103,9 @@ public: ObTsCbTask *task, int64_t >s) = 0; */ - virtual int wait_gts_elapse(const uint64_t tenant_id, const palf::SCN &scn, ObTsCbTask *task, + virtual int wait_gts_elapse(const uint64_t tenant_id, const share::SCN &scn, ObTsCbTask *task, bool &need_wait) = 0; - virtual int wait_gts_elapse(const uint64_t tenant_id, const palf::SCN &scn) = 0; + virtual int wait_gts_elapse(const uint64_t tenant_id, const share::SCN &scn) = 0; virtual int update_base_ts(const int64_t base_ts) = 0; virtual int get_base_ts(int64_t &base_ts) = 0; virtual bool is_external_consistent(const uint64_t tenant_id) = 0; @@ -356,14 +355,14 @@ public: int get_gts(const uint64_t tenant_id, const MonotonicTs stc, ObTsCbTask *task, - palf::SCN &scn, + share::SCN &scn, MonotonicTs &receive_gts_ts); //仅仅获取本地gts cache的最新值，但可能会失败，失败之后处理逻辑如下: //1. 如果task == NULL，说明调用者不需要异步回调，直接返回报错，由调用者处理 //2. 如果task != NULL，需要注册异步回调任务 - int get_gts(const uint64_t tenant_id, ObTsCbTask *task, palf::SCN &scn); + int get_gts(const uint64_t tenant_id, ObTsCbTask *task, share::SCN &scn); int get_ts_sync(const uint64_t tenant_id, const int64_t timeout_ts, - palf::SCN &scn, bool &is_external_consistent); + share::SCN &scn, bool &is_external_consistent); /* int get_local_trans_version(const uint64_t tenant_id, const MonotonicTs stc, @@ -374,9 +373,9 @@ public: ObTsCbTask *task, int64_t >s); */ - int wait_gts_elapse(const uint64_t tenant_id, const palf::SCN &scn, ObTsCbTask *task, + int wait_gts_elapse(const uint64_t tenant_id, const share::SCN &scn, ObTsCbTask *task, bool &need_wait); - int wait_gts_elapse(const uint64_t tenant_id, const palf::SCN &scn); + int wait_gts_elapse(const uint64_t tenant_id, const share::SCN &scn); int update_base_ts(const int64_t base_ts); int get_base_ts(int64_t &base_ts); bool is_external_consistent(const uint64_t tenant_id); diff --git a/src/storage/tx/ob_two_phase_committer.h b/src/storage/tx/ob_two_phase_committer.h index 963319f90..943ab4b4e 100644 --- a/src/storage/tx/ob_two_phase_committer.h +++ b/src/storage/tx/ob_two_phase_committer.h @@ -340,7 +340,7 @@ private: // exception. // // NB: We should take both upstream and downstream into consideration. - int decide_downstream_msg_type_(ObTwoPhaseCommitMsgType &msg_type); + int decide_downstream_msg_type_(bool &need_submit, ObTwoPhaseCommitMsgType &msg_type); int retransmit_downstream_msg_(); int retransmit_downstream_msg_(const uint8_t participant); int retransmit_upstream_msg_(const ObTxState state); diff --git a/src/storage/tx/ob_two_phase_downstream_committer.cpp b/src/storage/tx/ob_two_phase_downstream_committer.cpp index cfb33344d..55d219e19 100644 --- a/src/storage/tx/ob_two_phase_downstream_committer.cpp +++ b/src/storage/tx/ob_two_phase_downstream_committer.cpp @@ -260,7 +260,7 @@ int ObTxCycleTwoPhaseCommitter::retransmit_upstream_msg_(const ObTxState state) // if xa trans, prepare redo response is required msg_type = ObTwoPhaseCommitMsgType::OB_MSG_TX_PREPARE_REDO_RESP; } else { - // do nothing + need_respond = false; } } case ObTxState::PREPARE: { @@ -754,15 +754,19 @@ int ObTxCycleTwoPhaseCommitter::apply_abort_log() const ObTxState state = get_downstream_state(); const ObTxState upstream_state = get_upstream_state(); + if (ObTxState::ABORT != upstream_state) { + TRANS_LOG(WARN, "meet tx whose upstrean state is not abort", K(ret), KPC(this)); + } + if (ObTxState::INIT != state && ObTxState::REDO_COMPLETE != state && ObTxState::PREPARE != state) { // We will never apply abort under commit and clear state ret = OB_TRANS_INVALID_STATE; TRANS_LOG(ERROR, "apply abort with wrong state", K(state)); - } else if (ObTxState::ABORT != upstream_state) { - ret = OB_TRANS_INVALID_STATE; - TRANS_LOG(ERROR, "apply invalid log", K(ret), K(*this), K(upstream_state)); + // } else if (ObTxState::ABORT != upstream_state) { + // ret = OB_TRANS_INVALID_STATE; + // TRANS_LOG(ERROR, "apply invalid log", K(ret), K(*this), K(upstream_state)); } else if (OB_FAIL(on_abort())) { TRANS_LOG(ERROR, "on abort failed", K(ret), K(*this), K(state)); } else if (OB_FAIL(set_downstream_state(ObTxState::ABORT))) { diff --git a/src/storage/tx/ob_two_phase_upstream_committer.cpp b/src/storage/tx/ob_two_phase_upstream_committer.cpp index 29a93e9d1..3fbe0a4ef 100644 --- a/src/storage/tx/ob_two_phase_upstream_committer.cpp +++ b/src/storage/tx/ob_two_phase_upstream_committer.cpp @@ -35,7 +35,7 @@ int ObTxCycleTwoPhaseCommitter::two_phase_commit() bool no_need_submit_log = false; //start 2pc from root - if (ObTxState::INIT != get_upstream_state()) { + if (ObTxState::PREPARE <= get_upstream_state()) { TRANS_LOG(INFO, "already enter two phase commit", K(ret), K(*this)); } else if (is_2pc_logging()) { TRANS_LOG(INFO, "committer is under logging", K(ret), K(*this)); @@ -185,12 +185,13 @@ int ObTxCycleTwoPhaseCommitter::retransmit_downstream_msg_() int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; ObTwoPhaseCommitMsgType msg_type; + bool need_submit = true; if (!is_leaf()) { int this_part_id = get_participant_id(); - if (OB_FAIL(decide_downstream_msg_type_(msg_type))) { + if (OB_FAIL(decide_downstream_msg_type_(need_submit, msg_type))) { TRANS_LOG(WARN, "deecide downstream msg_type fail", K(ret), KPC(this)); - } else { + } else if (need_submit) { for (int64_t i = 0; i < get_participants_size(); ++i) { if (!collected_.has_member(i) && this_part_id != i) { TRANS_LOG(INFO, "unresponded participant", K(i), K(*this)); @@ -204,38 +205,48 @@ int ObTxCycleTwoPhaseCommitter::retransmit_downstream_msg_() return ret; } -int ObTxCycleTwoPhaseCommitter::decide_downstream_msg_type_(ObTwoPhaseCommitMsgType &msg_type) +int ObTxCycleTwoPhaseCommitter::decide_downstream_msg_type_(bool &need_submit, + ObTwoPhaseCommitMsgType &msg_type) { int ret = OB_SUCCESS; msg_type = ObTwoPhaseCommitMsgType::OB_MSG_TX_UNKNOWN; + need_submit = true; switch (get_upstream_state()) { case ObTxState::REDO_COMPLETE: { if (is_sub2pc()) { + need_submit = true; msg_type = ObTwoPhaseCommitMsgType::OB_MSG_TX_PREPARE_REDO_REQ; } else { - ret = OB_TRANS_INVALID_STATE; - TRANS_LOG(WARN, "invalid coord state", KR(ret), K(get_upstream_state())); + need_submit = false; + if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) { + TRANS_LOG(WARN, "handle timeout when redo complete", KR(ret), KPC(this)); + } } break; } case ObTxState::PREPARE: { + need_submit = true; msg_type = ObTwoPhaseCommitMsgType::OB_MSG_TX_PREPARE_REQ; break; } case ObTxState::PRE_COMMIT: { + need_submit = true; msg_type = ObTwoPhaseCommitMsgType::OB_MSG_TX_PRE_COMMIT_REQ; break; } case ObTxState::COMMIT: { + need_submit = true; msg_type = ObTwoPhaseCommitMsgType::OB_MSG_TX_COMMIT_REQ; break; } case ObTxState::ABORT: { + need_submit = true; msg_type = ObTwoPhaseCommitMsgType::OB_MSG_TX_ABORT_REQ; break; } case ObTxState::CLEAR: { + need_submit = true; msg_type = ObTwoPhaseCommitMsgType::OB_MSG_TX_CLEAR_REQ; break; } @@ -251,11 +262,12 @@ int ObTxCycleTwoPhaseCommitter::retransmit_downstream_msg_(const uint8_t partici { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; + bool need_submit = true; ObTwoPhaseCommitMsgType msg_type; if (is_leaf()) { - } else if (OB_FAIL(decide_downstream_msg_type_(msg_type))) { + } else if (OB_FAIL(decide_downstream_msg_type_(need_submit, msg_type))) { TRANS_LOG(WARN, "decide downstream msg type fail", K(ret), KPC(this)); - } else if (OB_TMP_FAIL(post_msg(msg_type, participant))) { + } else if (need_submit && OB_TMP_FAIL(post_msg(msg_type, participant))) { TRANS_LOG(WARN, "post prepare msg failed", KR(tmp_ret), KPC(this)); } diff --git a/src/storage/tx/ob_tx_2pc_ctx_impl.cpp b/src/storage/tx/ob_tx_2pc_ctx_impl.cpp index 986f0f644..6fbe143ad 100644 --- a/src/storage/tx/ob_tx_2pc_ctx_impl.cpp +++ b/src/storage/tx/ob_tx_2pc_ctx_impl.cpp @@ -16,6 +16,7 @@ namespace oceanbase { using namespace common; +using namespace share; namespace transaction { @@ -139,7 +140,7 @@ int ObPartTransCtx::do_pre_commit(bool &need_wait) int ObPartTransCtx::do_commit() { int ret = OB_SUCCESS; - palf::SCN gts, local_max_read_version; + SCN gts, local_max_read_version; if (is_local_tx_()) { if (OB_FAIL(get_gts_(gts))) { @@ -157,7 +158,7 @@ int ObPartTransCtx::do_commit() mt_ctx_.before_prepare(gts); if (OB_FAIL(get_local_max_read_version_(local_max_read_version))) { TRANS_LOG(ERROR, "get local max read version failed", KR(ret), K(*this)); - } else if (OB_FAIL(ctx_tx_data_.set_commit_version(palf::SCN::max(gts, local_max_read_version)))) { + } else if (OB_FAIL(ctx_tx_data_.set_commit_version(SCN::max(gts, local_max_read_version)))) { TRANS_LOG(ERROR, "set tx data commit version failed", K(ret)); } } @@ -231,7 +232,7 @@ int ObPartTransCtx::check_and_response_scheduler_(int result) } -int ObPartTransCtx::update_local_max_commit_version_(const palf::SCN &commit_version) +int ObPartTransCtx::update_local_max_commit_version_(const SCN &commit_version) { int ret = OB_SUCCESS; trans_service_->get_tx_version_mgr().update_max_commit_ts(commit_version, false); diff --git a/src/storage/tx/ob_tx_2pc_msg_handler.cpp b/src/storage/tx/ob_tx_2pc_msg_handler.cpp index ee0eca569..c6f9489b3 100644 --- a/src/storage/tx/ob_tx_2pc_msg_handler.cpp +++ b/src/storage/tx/ob_tx_2pc_msg_handler.cpp @@ -445,16 +445,16 @@ int ObPartTransCtx::merge_prepare_log_info_(const ObLSLogInfo &prepare_info) return ret; } -int ObPartTransCtx::update_2pc_prepare_version_(const palf::SCN &prepare_version) +int ObPartTransCtx::update_2pc_prepare_version_(const SCN &prepare_version) { int ret = OB_SUCCESS; - exec_info_.prepare_version_ = palf::SCN::max(prepare_version, exec_info_.prepare_version_); + exec_info_.prepare_version_ = SCN::max(prepare_version, exec_info_.prepare_version_); return ret; } -int ObPartTransCtx::set_2pc_commit_version_(const palf::SCN &commit_version) +int ObPartTransCtx::set_2pc_commit_version_(const SCN &commit_version) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_tx_api.cpp b/src/storage/tx/ob_tx_api.cpp index 28713d874..9a280c469 100644 --- a/src/storage/tx/ob_tx_api.cpp +++ b/src/storage/tx/ob_tx_api.cpp @@ -458,7 +458,7 @@ int ObTransService::submit_commit_tx(ObTxDesc &tx, OB_FAIL(tx.trace_info_.set_app_trace_info(*trace_info))) { TRANS_LOG(WARN, "set trace_info failed", K(ret), KPC(trace_info)); } - palf::SCN commit_version; + SCN commit_version; if (OB_SUCC(ret) && OB_FAIL(do_commit_tx_(tx, expire_ts, cb, commit_version))) { TRANS_LOG(WARN, "try to commit tx fail, tx will be aborted", @@ -531,7 +531,7 @@ int ObTransService::get_read_snapshot(ObTxDesc &tx, // only acquire snapshot once in these isolation level if (tx.isolation_ != isolation /*change isolation*/ || !tx.snapshot_version_.is_valid()/*version invalid*/) { - palf::SCN version; + SCN version; int64_t uncertain_bound = 0; if (OB_FAIL(sync_acquire_global_snapshot_(tx, expire_ts, version, uncertain_bound))) { TRANS_LOG(WARN, "acquire global snapshot fail", K(ret), K(tx)); @@ -650,7 +650,7 @@ int ObTransService::get_ls_read_snapshot(ObTxDesc &tx, } int ObTransService::get_read_snapshot_version(const int64_t expire_ts, - palf::SCN &snapshot_version) + SCN &snapshot_version) { int ret = OB_SUCCESS; int64_t uncertain_bound = 0; @@ -664,14 +664,14 @@ int ObTransService::get_read_snapshot_version(const int64_t expire_ts, } int ObTransService::get_ls_read_snapshot_version(const share::ObLSID &local_ls_id, - palf::SCN &snapshot_version) + SCN &snapshot_version) { int ret = OB_SUCCESS; ret = acquire_local_snapshot_(local_ls_id, snapshot_version); return ret; } -int ObTransService::get_weak_read_snapshot_version(palf::SCN &snapshot) +int ObTransService::get_weak_read_snapshot_version(SCN &snapshot) { int ret = OB_SUCCESS; bool monotinic_read = true;; @@ -700,7 +700,7 @@ int ObTransService::get_weak_read_snapshot_version(palf::SCN &snapshot) int ObTransService::release_snapshot(ObTxDesc &tx) { int ret = OB_SUCCESS; - palf::SCN snapshot; + SCN snapshot; ObSpinLockGuard guard(tx.lock_); tx.inc_op_sn(); if (tx.state_ != ObTxDesc::State::IDLE) { @@ -1279,16 +1279,13 @@ int ObTransService::rollback_savepoint_(ObTxDesc &tx, slowpath = false; ObTxPart &p = parts[0]; int64_t born_epoch = 0; - ObFunction create_tx_ctx_func = [this, &p, &tx](ObPartTransCtx *&ctx) -> int { - return this->create_tx_ctx_(p.id_, tx, ctx); - }; if (OB_FAIL(ls_rollback_to_savepoint_(tx.tx_id_, p.id_, p.epoch_, tx.op_sn_, savepoint, born_epoch, - create_tx_ctx_func, + &tx, expire_ts))) { if (OB_NOT_MASTER == ret) { slowpath = true; @@ -1342,7 +1339,7 @@ int ObTransService::ls_rollback_to_savepoint_(const ObTransID &tx_id, const int64_t op_sn, const int64_t savepoint, int64_t &ctx_born_epoch, - ObFunction &func, + const ObTxDesc *tx, int64_t expire_ts) { int ret = OB_SUCCESS; @@ -1351,8 +1348,8 @@ int ObTransService::ls_rollback_to_savepoint_(const ObTransID &tx_id, if (OB_FAIL(get_tx_ctx_(ls, tx_id, ctx))) { if (OB_NOT_MASTER == ret) { } else if (OB_TRANS_CTX_NOT_EXIST == ret && verify_epoch <= 0) { - if (OB_FAIL(func(ctx))) { - TRANS_LOG(WARN, "create tx ctx fail", K(ret), K(ls), K(tx_id)); + if (OB_FAIL(create_tx_ctx_(ls, *tx, ctx))) { + TRANS_LOG(WARN, "create tx ctx fail", K(ret), K(ls), KPC(tx)); } } else { TRANS_LOG(WARN, "get transaction context error", K(ret), K(tx_id), K(ls)); @@ -1403,12 +1400,32 @@ inline int ObTransService::rollback_savepoint_slowpath_(ObTxDesc &tx, msg.tx_id_ = tx.tx_id_; msg.savepoint_ = savepoint; msg.op_sn_ = tx.op_sn_; - msg.can_elr_ = tx.can_elr_; - msg.session_id_ = tx.sess_id_; - msg.tx_addr_ = tx.addr_; - msg.tx_expire_ts_ = tx.get_expire_ts(); msg.epoch_ = -1; msg.request_id_ = tx.op_sn_; + // prepare msg.tx_ptr_ if required + // TODO(yunxing.cyx) : in 4.1 rework here, won't serialize txDesc + ObTxDesc *tmp_tx_desc = NULL; + ARRAY_FOREACH_NORET(parts, i) { + if (parts[i].epoch_ <= 0) { + int64_t len = tx.get_serialize_size() + sizeof(ObTxDesc); + char *buf = (char*)ob_malloc(len); + int64_t pos = sizeof(ObTxDesc); + if (OB_FAIL(tx.serialize(buf, len, pos))) { + TRANS_LOG(WARN, "serialize tx fail", KR(ret), K(tx)); + ob_free(buf); + } else { + tmp_tx_desc = new(buf)ObTxDesc(); + pos = sizeof(ObTxDesc); + if (OB_FAIL(tmp_tx_desc->deserialize(buf, len, pos))) { + TRANS_LOG(WARN, "deserialize tx fail", KR(ret)); + } else { + tmp_tx_desc->parts_.reset(); + msg.tx_ptr_ = tmp_tx_desc; + } + } + break; + } + } int64_t start_ts = ObTimeUtility::current_time(); int retries = 0; if (OB_SUCC(ret)) { @@ -1429,6 +1446,11 @@ inline int ObTransService::rollback_savepoint_slowpath_(ObTxDesc &tx, // clear interrupt flag tx.flags_.INTERRUPTED_ = false; } + if (OB_NOT_NULL(tmp_tx_desc)) { + msg.tx_ptr_ = NULL; + tmp_tx_desc->~ObTxDesc(); + ob_free(tmp_tx_desc); + } auto elapsed_us = ObTimeUtility::current_time() - start_ts; TRANS_LOG(INFO, "rollback savepoint slowpath", K(ret), K_(tx.tx_id), K(start_ts), K(retries), diff --git a/src/storage/tx/ob_tx_api.h b/src/storage/tx/ob_tx_api.h index f53ff7138..82353957a 100644 --- a/src/storage/tx/ob_tx_api.h +++ b/src/storage/tx/ob_tx_api.h @@ -202,7 +202,7 @@ int get_ls_read_snapshot(ObTxDesc &tx, * OB_TIMEOUT - if expire_ts hit */ int get_read_snapshot_version(const int64_t expire_ts, - palf::SCN &snapshot_version); + share::SCN &snapshot_version); /** * get_ls_read_snapshot_version - get a read snapshot of specified @@ -222,7 +222,7 @@ int get_read_snapshot_version(const int64_t expire_ts, * OB_TIMEOUT - if expire_ts hit */ int get_ls_read_snapshot_version(const share::ObLSID &local_ls_id, - palf::SCN &snapshot_version); + share::SCN &snapshot_version); /** * get_weak_read_snapshot_version - get snapshot version for weak read * @@ -232,7 +232,7 @@ int get_ls_read_snapshot_version(const share::ObLSID &local_ls_id, * OB_SUCCESS - OK * OB_REPLICA_NOT_READABLE - snapshot is too stale */ -int get_weak_read_snapshot_version(palf::SCN &snapshot_version); +int get_weak_read_snapshot_version(share::SCN &snapshot_version); /* * release_snapshot - release snapshot * diff --git a/src/storage/tx/ob_tx_data_define.h b/src/storage/tx/ob_tx_data_define.h index f2bc63788..6185061ed 100644 --- a/src/storage/tx/ob_tx_data_define.h +++ b/src/storage/tx/ob_tx_data_define.h @@ -14,7 +14,7 @@ #define OCEANBASE_STORAGE_OB_TX_DATA_DEFINE #include "lib/allocator/ob_slice_alloc.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/tx/ob_committer_define.h" #include "storage/tx/ob_trans_define.h" #include "storage/ob_i_table.h" @@ -195,14 +195,14 @@ class ObTxCCCtx { public: // For Tx Ctx Table - ObTxCCCtx(transaction::ObTxState state, palf::SCN prepare_version) + ObTxCCCtx(transaction::ObTxState state, share::SCN prepare_version) : state_(state), prepare_version_(prepare_version) {} // For Tx Data Table ObTxCCCtx() : state_(transaction::ObTxState::MAX), prepare_version_() {} TO_STRING_KV(K_(state), K_(prepare_version)); public: transaction::ObTxState state_; - palf::SCN prepare_version_; + share::SCN prepare_version_; }; class ObTxCommitData @@ -232,9 +232,9 @@ public: transaction::ObTransID tx_id_; int32_t state_; bool is_in_tx_data_table_; - palf::SCN commit_version_; - palf::SCN start_scn_; - palf::SCN end_scn_; + share::SCN commit_version_; + share::SCN start_scn_; + share::SCN end_scn_; }; class ObTxData : public ObTxCommitData, public TxDataHashValue diff --git a/src/storage/tx/ob_tx_data_functor.cpp b/src/storage/tx/ob_tx_data_functor.cpp index 1451e7b99..1fc60734f 100644 --- a/src/storage/tx/ob_tx_data_functor.cpp +++ b/src/storage/tx/ob_tx_data_functor.cpp @@ -21,7 +21,7 @@ namespace oceanbase { -using namespace palf; +using namespace share; namespace storage { diff --git a/src/storage/tx/ob_tx_data_functor.h b/src/storage/tx/ob_tx_data_functor.h index 6164f7c3b..f0f23c19b 100644 --- a/src/storage/tx/ob_tx_data_functor.h +++ b/src/storage/tx/ob_tx_data_functor.h @@ -33,7 +33,7 @@ class ObReCheckTxNodeForLockForReadOperation public: ObReCheckTxNodeForLockForReadOperation(memtable::ObMvccTransNode &tnode, bool &can_read, - palf::SCN &trans_version, + share::SCN &trans_version, bool &is_determined_state) : tnode_(tnode), can_read_(can_read), @@ -45,7 +45,7 @@ private: memtable::ObMvccTransNode &tnode_; bool &can_read_; bool &is_determined_state_; - palf::SCN &trans_version_; + share::SCN &trans_version_; }; class ObReCheckNothingOperation @@ -134,17 +134,17 @@ public: class GetTxStateWithSCNFunctor : public ObITxDataCheckFunctor { public: - GetTxStateWithSCNFunctor(const palf::SCN scn, + GetTxStateWithSCNFunctor(const share::SCN scn, int64_t &state, - palf::SCN &trans_version) + share::SCN &trans_version) : scn_(scn), state_(state), trans_version_(trans_version) {} virtual ~GetTxStateWithSCNFunctor() {} virtual int operator()(const ObTxData &tx_data, ObTxCCCtx *tx_cc_ctx = nullptr) override; TO_STRING_KV(K(scn_), K(state_), K(trans_version_)); public: - const palf::SCN scn_; + const share::SCN scn_; int64_t &state_; - palf::SCN &trans_version_; + share::SCN &trans_version_; }; // the txn READ_TRANS_ID use SNAPSHOT_VERSION to read the data, @@ -158,7 +158,7 @@ class LockForReadFunctor : public ObITxDataCheckFunctor public: LockForReadFunctor(const transaction::ObLockForReadArg &lock_for_read_arg, bool &can_read, - palf::SCN &trans_version, + share::SCN &trans_version, bool &is_determined_state, const ObCleanoutOp &cleanout_op = ObCleanoutNothingOperation(), const ObReCheckOp &recheck_op = ObReCheckNothingOperation()) @@ -180,7 +180,7 @@ public: const transaction::ObLockForReadArg &lock_for_read_arg_; bool &can_read_; bool &is_determined_state_; - palf::SCN &trans_version_; + share::SCN &trans_version_; // Cleanout the tx node if necessary ObCleanoutOp cleanout_op_; // ReCheck whether tx node is valid. diff --git a/src/storage/tx/ob_tx_log.cpp b/src/storage/tx/ob_tx_log.cpp index f5c124d73..ed2cb43ed 100644 --- a/src/storage/tx/ob_tx_log.cpp +++ b/src/storage/tx/ob_tx_log.cpp @@ -459,7 +459,7 @@ int ObTxRedoLog::ob_admin_dump(memtable::ObMemtableMutatorIterator *iter_ptr, palf::block_id_t block_id, palf::LSN lsn, int64_t tx_id, - palf::SCN scn, + SCN scn, bool &has_dumped_tx_id) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_tx_log.h b/src/storage/tx/ob_tx_log.h index 400556e8c..f4407ed14 100644 --- a/src/storage/tx/ob_tx_log.h +++ b/src/storage/tx/ob_tx_log.h @@ -24,7 +24,7 @@ #include "logservice/ob_log_base_type.h" #include "logservice/ob_log_base_header.h" #include "logservice/palf/lsn.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" //#include namespace oceanbase @@ -241,7 +241,7 @@ public: palf::block_id_t block_id, palf::LSN lsn, int64_t tx_id, - palf::SCN scn, + share::SCN scn, bool &has_output); //------------ @@ -587,11 +587,11 @@ public: int init(const storage::ObTxData * const tx_data); void reset(); - palf::SCN get_start_log_ts() const { return start_log_ts_; } + share::SCN get_start_log_ts() const { return start_log_ts_; } TO_STRING_KV(K(start_log_ts_)); private: - palf::SCN start_log_ts_; + share::SCN start_log_ts_; }; class ObTxCommitLogTempRef @@ -621,7 +621,7 @@ public: { before_serialize(); } - ObTxCommitLog(palf::SCN commit_version, + ObTxCommitLog(share::SCN commit_version, uint64_t checksum, share::ObLSArray &incremental_participants, ObTxBufferNodeArray &multi_source_data, @@ -637,7 +637,7 @@ public: } int init_tx_data_backup(const storage::ObTxData *const tx_data); const ObTxDataBackup &get_tx_data_backup() const { return tx_data_backup_; } - palf::SCN get_commit_version() const { return commit_version_; } + share::SCN get_commit_version() const { return commit_version_; } uint64_t get_checksum() const { return checksum_; } const share::ObLSArray &get_incremental_participants() const { return incremental_participants_; } const ObTxBufferNodeArray &get_multi_source_data() const { return multi_source_data_; } @@ -664,7 +664,7 @@ private: private: ObTxSerCompatByte compat_bytes_; - palf::SCN commit_version_; // equal to INVALID_COMMIT_VERSION in Single LS Transaction + share::SCN commit_version_; // equal to INVALID_COMMIT_VERSION in Single LS Transaction uint64_t checksum_; share::ObLSArray &incremental_participants_; ObTxBufferNodeArray &multi_source_data_; diff --git a/src/storage/tx/ob_tx_log_adapter.cpp b/src/storage/tx/ob_tx_log_adapter.cpp index 87d9e77da..b054b9801 100644 --- a/src/storage/tx/ob_tx_log_adapter.cpp +++ b/src/storage/tx/ob_tx_log_adapter.cpp @@ -16,7 +16,7 @@ namespace oceanbase { -using namespace palf; +using namespace share; namespace transaction { @@ -35,13 +35,13 @@ int ObLSTxLogAdapter::init(ObITxLogParam *param) int ObLSTxLogAdapter::submit_log(const char *buf, const int64_t size, - const palf::SCN base_ts, + const SCN base_ts, ObTxBaseLogCb *cb, const bool need_nonblock) { int ret = OB_SUCCESS; palf::LSN lsn; - palf::SCN ts; + SCN ts; if (OB_ISNULL(log_handler_) || !log_handler_->is_valid() || NULL == buf || 0 == size || base_ts.convert_to_ts() > ObTimeUtility::current_time() + 86400000000L) { diff --git a/src/storage/tx/ob_tx_log_adapter.h b/src/storage/tx/ob_tx_log_adapter.h index 87c5bc685..b90fe8078 100644 --- a/src/storage/tx/ob_tx_log_adapter.h +++ b/src/storage/tx/ob_tx_log_adapter.h @@ -48,7 +48,7 @@ class ObITxLogAdapter public: virtual int submit_log(const char *buf, const int64_t size, - const palf::SCN base_ts, + const share::SCN base_ts, ObTxBaseLogCb *cb, const bool need_nonblock) = 0; @@ -65,7 +65,7 @@ public: int init(ObITxLogParam *param); int submit_log(const char *buf, const int64_t size, - const palf::SCN base_ts, + const share::SCN base_ts, ObTxBaseLogCb *cb, const bool need_nonblock); int get_role(bool &is_leader, int64_t &epoch); diff --git a/src/storage/tx/ob_tx_loop_worker.cpp b/src/storage/tx/ob_tx_loop_worker.cpp index 47a944b4b..9a1064100 100644 --- a/src/storage/tx/ob_tx_loop_worker.cpp +++ b/src/storage/tx/ob_tx_loop_worker.cpp @@ -203,7 +203,7 @@ void ObTxLoopWorker::do_tx_gc_(ObLS *ls_ptr) void ObTxLoopWorker::update_max_commit_ts_(ObLS *ls_ptr) { int ret = OB_SUCCESS; - palf::SCN snapshot; + SCN snapshot; const int64_t expire_ts = ObClockGenerator::getClock() + 1000000; // 1s do { diff --git a/src/storage/tx/ob_tx_ls_log_writer.cpp b/src/storage/tx/ob_tx_ls_log_writer.cpp index 5a81452e6..b91c8510e 100644 --- a/src/storage/tx/ob_tx_ls_log_writer.cpp +++ b/src/storage/tx/ob_tx_ls_log_writer.cpp @@ -235,7 +235,7 @@ void ObTxLSLogWriter::destroy() destroy_cbs_(free_cbs_); } -int ObTxLSLogWriter::submit_start_working_log(const int64_t &leader_epoch, palf::SCN &log_ts) +int ObTxLSLogWriter::submit_start_working_log(const int64_t &leader_epoch, SCN &log_ts) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_tx_ls_log_writer.h b/src/storage/tx/ob_tx_ls_log_writer.h index 21a2726ef..1e57452b0 100644 --- a/src/storage/tx/ob_tx_ls_log_writer.h +++ b/src/storage/tx/ob_tx_ls_log_writer.h @@ -160,7 +160,7 @@ public: void destroy(); public: - int submit_start_working_log(const int64_t &leader_epoch, palf::SCN &log_ts); + int submit_start_working_log(const int64_t &leader_epoch, share::SCN &log_ts); public: int on_success(ObTxLSLogCb *cb); @@ -177,7 +177,7 @@ private: int submit_ls_log_(T &ls_log, const enum logservice::ObReplayBarrierType barrier_type, bool need_nonblock, - palf::SCN &log_ts); + share::SCN &log_ts); int get_log_cb_(const ObTxLogType &log_type, ObTxLSLogCb *&cb); int return_log_cb_(ObTxLSLogCb *cb); bool enough_log_cb_(); @@ -207,7 +207,7 @@ template int ObTxLSLogWriter::submit_ls_log_(T &ls_log, const enum logservice::ObReplayBarrierType barrier_type, bool need_nonblock, - palf::SCN &log_ts) + share::SCN &log_ts) { int ret = OB_SUCCESS; ObSpinLockGuard cb_guard(cbs_lock_); @@ -221,7 +221,7 @@ int ObTxLSLogWriter::submit_ls_log_(T &ls_log, } } else if (nullptr == cb || OB_FAIL(cb->serialize_ls_log(ls_log, replay_hint, barrier_type))) { TRANS_LOG(WARN, "[TxLsLogWriter] serialize ls log error", KR(ret), K(T::LOG_TYPE), KP(cb)); - } else if (OB_FAIL(tx_log_adapter_->submit_log(cb->get_log_buf(), cb->get_log_pos(), palf::SCN::min_scn(), cb, need_nonblock))) { + } else if (OB_FAIL(tx_log_adapter_->submit_log(cb->get_log_buf(), cb->get_log_pos(), share::SCN::min_scn(), cb, need_nonblock))) { return_log_cb_(cb); cb = nullptr; TRANS_LOG(WARN, "[TxLsLogWriter] submit ls log failed", KR(ret), K(T::LOG_TYPE), K(ls_id_)); diff --git a/src/storage/tx/ob_tx_msg.cpp b/src/storage/tx/ob_tx_msg.cpp index 3dff62735..efedff76f 100644 --- a/src/storage/tx/ob_tx_msg.cpp +++ b/src/storage/tx/ob_tx_msg.cpp @@ -53,7 +53,56 @@ OB_SERIALIZE_MEMBER_INHERIT(Ob2pcPrepareRedoReqMsg, ObTxMsg, xid_, upstream_, ap OB_SERIALIZE_MEMBER_INHERIT(Ob2pcPrepareRedoRespMsg, ObTxMsg); OB_SERIALIZE_MEMBER_INHERIT(Ob2pcPrepareVersionReqMsg, ObTxMsg); OB_SERIALIZE_MEMBER_INHERIT(Ob2pcPrepareVersionRespMsg, ObTxMsg, prepare_version_, prepare_info_array_); -OB_SERIALIZE_MEMBER_INHERIT(ObTxRollbackSPMsg, ObTxMsg, savepoint_, op_sn_, can_elr_, session_id_, tx_addr_, tx_expire_ts_); + +OB_DEF_SERIALIZE_SIZE(ObTxRollbackSPMsg) +{ + int len = 0; + len += ObTxMsg::get_serialize_size(); + LST_DO_CODE(OB_UNIS_ADD_LEN, savepoint_, op_sn_, branch_id_); + if (OB_NOT_NULL(tx_ptr_)) { + OB_UNIS_ADD_LEN(true); + OB_UNIS_ADD_LEN(*tx_ptr_); + } else { + OB_UNIS_ADD_LEN(false); + } + return len; +} + +OB_DEF_SERIALIZE(ObTxRollbackSPMsg) +{ + int ret = ObTxMsg::serialize(buf, buf_len, pos); + if (OB_SUCC(ret)) { + LST_DO_CODE(OB_UNIS_ENCODE, savepoint_, op_sn_, branch_id_); + if (OB_NOT_NULL(tx_ptr_)) { + OB_UNIS_ENCODE(true); + OB_UNIS_ENCODE(*tx_ptr_); + } else { + OB_UNIS_ENCODE(false); + } + } + return ret; +} + +OB_DEF_DESERIALIZE(ObTxRollbackSPMsg) +{ + int ret = ObTxMsg::deserialize(buf, data_len, pos); + if (OB_SUCC(ret)) { + LST_DO_CODE(OB_UNIS_DECODE, savepoint_, op_sn_, branch_id_); + bool has_tx_ptr = false; + OB_UNIS_DECODE(has_tx_ptr); + if (has_tx_ptr) { + void *buffer = ob_malloc(sizeof(ObTxDesc)); + if (OB_ISNULL(buffer)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else { + ObTxDesc *tmp = new(buffer)ObTxDesc(); + OB_UNIS_DECODE(*tmp); + tx_ptr_ = tmp; + } + } + } + return ret; +} bool ObTxMsg::is_valid() const { @@ -159,9 +208,7 @@ bool ObTxRollbackSPMsg::is_valid() const { bool ret = false; if (ObTxMsg::is_valid() && type_ == ROLLBACK_SAVEPOINT - && savepoint_ > -1 && op_sn_ > -1 - && session_id_ > 0 && tx_addr_.is_valid() - && tx_expire_ts_ > 0) { + && savepoint_ > -1 && op_sn_ > -1) { ret = true; } return ret; diff --git a/src/storage/tx/ob_tx_msg.h b/src/storage/tx/ob_tx_msg.h index 4af616194..0a8938667 100644 --- a/src/storage/tx/ob_tx_msg.h +++ b/src/storage/tx/ob_tx_msg.h @@ -13,7 +13,7 @@ #ifndef OCEANBASE_TRANSACTION_OB_TX_MSG_ #define OCEANBASE_TRANSACTION_OB_TX_MSG_ -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_define.h" #include "ob_trans_define.h" #include "share/rpc/ob_batch_proxy.h" @@ -207,7 +207,7 @@ namespace transaction ObTxCommitRespMsg() : ObTxMsg(TX_COMMIT_RESP) {} - palf::SCN commit_version_; + share::SCN commit_version_; int ret_; bool is_valid() const; INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(ret), K_(commit_version)); @@ -229,21 +229,26 @@ namespace transaction ObTxMsg(ROLLBACK_SAVEPOINT), savepoint_(-1), op_sn_(-1), - can_elr_(false), - session_id_(0), - tx_addr_(), - tx_expire_ts_(-1) + //todo:后续branch_id使用方式确定后，需要相应修改 + branch_id_(-1), + tx_ptr_(NULL) {} - ~ObTxRollbackSPMsg() {} + ~ObTxRollbackSPMsg() { + if (OB_NOT_NULL(tx_ptr_)) { + tx_ptr_->~ObTxDesc(); + ob_free((void*)tx_ptr_); + tx_ptr_ = NULL; + } + } int64_t savepoint_; int64_t op_sn_; - bool can_elr_; - uint32_t session_id_; - ObAddr tx_addr_; - int64_t tx_expire_ts_; + //todo:后期设计中操作编号是否等于branch_id + int64_t branch_id_; + const ObTxDesc *tx_ptr_; bool is_valid() const; INHERIT_TO_STRING_KV("txMsg", ObTxMsg, - K_(savepoint), K_(op_sn), K_(can_elr), K_(session_id), K_(tx_addr), K_(tx_expire_ts)); + K_(savepoint), K_(op_sn), K_(branch_id), + KP_(tx_ptr)); OB_UNIS_VERSION(1); }; @@ -290,7 +295,7 @@ namespace transaction prepare_info_array_() {} public: - palf::SCN prepare_version_; + share::SCN prepare_version_; ObLSLogInfoArray prepare_info_array_; bool is_valid() const; INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(prepare_version), K_(prepare_info_array)); @@ -304,7 +309,7 @@ namespace transaction ObTxMsg(TX_2PC_PRE_COMMIT_REQ) {} public: - palf::SCN commit_version_; + share::SCN commit_version_; bool is_valid() const; INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(commit_version)); OB_UNIS_VERSION(1); @@ -320,7 +325,7 @@ namespace transaction //set commit_version when the root participant //which recover from prepare log recive a pre_commit response //because the coord_state_ will be set as pre_commit - palf::SCN commit_version_; + share::SCN commit_version_; bool is_valid() const; INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(commit_version)); OB_UNIS_VERSION(1); @@ -334,7 +339,7 @@ namespace transaction prepare_info_array_() {} public: - palf::SCN commit_version_; + share::SCN commit_version_; ObLSLogInfoArray prepare_info_array_; bool is_valid() const; INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(commit_version), K_(prepare_info_array)); @@ -349,7 +354,7 @@ namespace transaction {} public: bool is_valid() const; - palf::SCN commit_version_; + share::SCN commit_version_; INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(commit_version)); OB_UNIS_VERSION(1); }; @@ -446,7 +451,7 @@ namespace transaction prepare_info_array_() {} public: - palf::SCN prepare_version_; + share::SCN prepare_version_; ObLSLogInfoArray prepare_info_array_; bool is_valid() const; INHERIT_TO_STRING_KV("txMsg", ObTxMsg, K_(prepare_version), K_(prepare_info_array)); diff --git a/src/storage/tx/ob_tx_replay_executor.cpp b/src/storage/tx/ob_tx_replay_executor.cpp index b1f897a6a..480571961 100644 --- a/src/storage/tx/ob_tx_replay_executor.cpp +++ b/src/storage/tx/ob_tx_replay_executor.cpp @@ -37,7 +37,7 @@ int ObTxReplayExecutor::execute(storage::ObLS *ls, const int64_t size, const int skip_pos, const palf::LSN &lsn, - const palf::SCN &log_timestamp, + const SCN &log_timestamp, const int64_t &replay_hint, const ObLSID &ls_id, const int64_t &tenant_id) @@ -424,7 +424,7 @@ int ObTxReplayExecutor::replay_commit_() int ret = OB_SUCCESS; ObTxCommitLogTempRef temp_ref; ObTxCommitLog commit_log(temp_ref); - palf::SCN replay_compact_version = ls_tx_srv_->get_ls_weak_read_ts(); + SCN replay_compact_version = ls_tx_srv_->get_ls_weak_read_ts(); if (OB_FAIL(log_block_.deserialize_log_body(commit_log))) { TRANS_LOG(WARN, "[Replay Tx] deserialize log body error", K(ret), K(commit_log), K(lsn_), K(log_ts_ns_)); diff --git a/src/storage/tx/ob_tx_replay_executor.h b/src/storage/tx/ob_tx_replay_executor.h index 20bb2dae9..91eadc8f1 100644 --- a/src/storage/tx/ob_tx_replay_executor.h +++ b/src/storage/tx/ob_tx_replay_executor.h @@ -56,7 +56,7 @@ public: const int64_t size, const int skip_pos, const palf::LSN &lsn, - const palf::SCN &log_timestamp, + const share::SCN &log_timestamp, const int64_t &replay_hint, const share::ObLSID &ls_id, const int64_t &tenant_id); @@ -74,7 +74,7 @@ private: ObTxReplayExecutor(storage::ObLS *ls, storage::ObLSTxService *ls_tx_srv, const palf::LSN &lsn, - const palf::SCN &log_timestamp) + const share::SCN &log_timestamp) : ctx_(nullptr), ls_(ls), ls_tx_srv_(ls_tx_srv), lsn_(lsn), log_ts_ns_(log_timestamp), mmi_ptr_(nullptr), mt_ctx_(nullptr), first_created_ctx_(false), has_redo_(false), tx_part_log_no_(0), mvcc_row_count_(0), table_lock_row_count_(0) @@ -135,7 +135,7 @@ private: ObTxLogBlockHeader log_block_header_; ObTxLogBlock log_block_; palf::LSN lsn_; - palf::SCN log_ts_ns_; + share::SCN log_ts_ns_; memtable::ObMemtableMutatorIterator * mmi_ptr_; memtable::ObMemtableCtx *mt_ctx_; diff --git a/src/storage/tx/ob_tx_retain_ctx_mgr.cpp b/src/storage/tx/ob_tx_retain_ctx_mgr.cpp index ddea51e87..0c4576f8a 100644 --- a/src/storage/tx/ob_tx_retain_ctx_mgr.cpp +++ b/src/storage/tx/ob_tx_retain_ctx_mgr.cpp @@ -20,11 +20,12 @@ namespace oceanbase { +using namespace share; namespace transaction { -ObAdvanceLSCkptTask::ObAdvanceLSCkptTask(share::ObLSID ls_id, palf::SCN ckpt_ts) +ObAdvanceLSCkptTask::ObAdvanceLSCkptTask(share::ObLSID ls_id, SCN ckpt_ts) {; task_type_ = ObTransRetryTaskType::ADVANCE_LS_CKPT_TASK; ls_id_ = ls_id; @@ -116,7 +117,7 @@ int ObIRetainCtxCheckFunctor::del_retain_ctx() int ObMDSRetainCtxFunctor::init(ObPartTransCtx *ctx, RetainCause cause, - const palf::SCN &final_log_ts, + const SCN &final_log_ts, palf::LSN final_log_lsn) { int ret = OB_SUCCESS; diff --git a/src/storage/tx/ob_tx_retain_ctx_mgr.h b/src/storage/tx/ob_tx_retain_ctx_mgr.h index d9f1ca557..54552c50e 100644 --- a/src/storage/tx/ob_tx_retain_ctx_mgr.h +++ b/src/storage/tx/ob_tx_retain_ctx_mgr.h @@ -36,7 +36,7 @@ class ObTxRetainCtxMgr; class ObAdvanceLSCkptTask : public ObTransTask { public: - ObAdvanceLSCkptTask(share::ObLSID ls_id, palf::SCN target_ts); + ObAdvanceLSCkptTask(share::ObLSID ls_id, share::SCN target_ts); ~ObAdvanceLSCkptTask() { reset(); } void reset(); @@ -44,7 +44,7 @@ public: private: share::ObLSID ls_id_; - palf::SCN target_ckpt_ts_; + share::SCN target_ckpt_ts_; }; /** @@ -82,13 +82,13 @@ public: final_log_ts_.reset(); final_log_lsn_.reset(); } - int init(ObPartTransCtx *ctx, RetainCause cause, const palf::SCN &final_log_ts, palf::LSN final_log_lsn); + int init(ObPartTransCtx *ctx, RetainCause cause, const share::SCN &final_log_ts, palf::LSN final_log_lsn); virtual int operator()(storage::ObLS *ls, ObTxRetainCtxMgr * retain_mgr) override; virtual bool is_valid() override; private: - palf::SCN final_log_ts_; + share::SCN final_log_ts_; palf::LSN final_log_lsn_; }; @@ -116,8 +116,8 @@ public: int print_retain_ctx_info(share::ObLSID ls_id); int force_gc_retain_ctx(); - void set_max_ckpt_ts(palf::SCN ckpt_ts) { max_wait_ckpt_ts_.inc_update(ckpt_ts); } - //palf::SCN get_max_ckpt_ts() { return ATOMIC_LOAD(&max_wait_ckpt_ts_); } + void set_max_ckpt_ts(share::SCN ckpt_ts) { max_wait_ckpt_ts_.inc_update(ckpt_ts); } + //share::SCN get_max_ckpt_ts() { return ATOMIC_LOAD(&max_wait_ckpt_ts_); } int64_t get_retain_ctx_cnt() { return retain_ctx_list_.size(); } void try_advance_retain_ctx_gc(share::ObLSID ls_id); @@ -134,7 +134,7 @@ private: common::SpinRWLock retain_ctx_lock_; RetainCtxList retain_ctx_list_; - palf::SCN max_wait_ckpt_ts_; + share::SCN max_wait_ckpt_ts_; int64_t last_push_gc_task_ts_; TransModulePageAllocator reserve_allocator_; diff --git a/src/storage/tx/ob_tx_version_mgr.h b/src/storage/tx/ob_tx_version_mgr.h index 12ec92a8a..becc198a0 100644 --- a/src/storage/tx/ob_tx_version_mgr.h +++ b/src/storage/tx/ob_tx_version_mgr.h @@ -33,7 +33,7 @@ public: } ~ObTxVersionMgr() {} public: - void update_max_commit_ts(const palf::SCN &ts, const bool elr) + void update_max_commit_ts(const share::SCN &ts, const bool elr) { if (!elr) { max_commit_ts_.inc_update(ts); @@ -42,31 +42,31 @@ public: } TRANS_LOG(TRACE, "update max commit ts", K(ts), K(elr)); } - void update_max_read_ts(const palf::SCN &ts) + void update_max_read_ts(const share::SCN &ts) { max_read_ts_.inc_update(ts); TRANS_LOG(TRACE, "update max read ts", K(ts)); } - palf::SCN get_max_commit_ts(const bool elr) const + share::SCN get_max_commit_ts(const bool elr) const { - palf::SCN max_commit_ts = max_commit_ts_.atomic_get(); + share::SCN max_commit_ts = max_commit_ts_.atomic_get(); if (elr) { - const palf::SCN max_elr_commit_ts = max_elr_commit_ts_.atomic_get(); - max_commit_ts = palf::SCN::max(max_commit_ts, max_elr_commit_ts); + const share::SCN max_elr_commit_ts = max_elr_commit_ts_.atomic_get(); + max_commit_ts = share::SCN::max(max_commit_ts, max_elr_commit_ts); } TRANS_LOG(TRACE, "get max commit ts", K(max_commit_ts), K(elr)); return max_commit_ts; } - palf::SCN get_max_read_ts() const + share::SCN get_max_read_ts() const { - const palf::SCN max_read_ts = palf::SCN::scn_inc(max_read_ts_); + const share::SCN max_read_ts = share::SCN::scn_inc(max_read_ts_); TRANS_LOG(TRACE, "get max read ts", K(max_read_ts)); return max_read_ts; } private: - palf::SCN max_commit_ts_ CACHE_ALIGNED; - palf::SCN max_elr_commit_ts_ CACHE_ALIGNED; - palf::SCN max_read_ts_ CACHE_ALIGNED; + share::SCN max_commit_ts_ CACHE_ALIGNED; + share::SCN max_elr_commit_ts_ CACHE_ALIGNED; + share::SCN max_read_ts_ CACHE_ALIGNED; }; } diff --git a/src/storage/tx/wrs/ob_i_weak_read_service.h b/src/storage/tx/wrs/ob_i_weak_read_service.h index fe73f78bf..083e49b17 100644 --- a/src/storage/tx/wrs/ob_i_weak_read_service.h +++ b/src/storage/tx/wrs/ob_i_weak_read_service.h @@ -11,7 +11,7 @@ */ #include -#include "logservice/palf/scn.h" +#include "share/scn.h" #ifndef OCEANBASE_TRANSACTION_OB_I_WRS_SERVICE_ #define OCEANBASE_TRANSACTION_OB_I_WRS_SERVICE_ @@ -69,13 +69,13 @@ public: /// @param [in] tenant_id target tenant ID /// @param [in] is_inner is inner table or not /// @param [out] version wrs version - virtual int get_server_version(const uint64_t tenant_id, palf::SCN &version) const = 0; + virtual int get_server_version(const uint64_t tenant_id, share::SCN &version) const = 0; /// get CLUSTER level wrs version /// /// @param [in] tenant_id target tenant ID /// @param [out] version wrs version - virtual int get_cluster_version(const uint64_t tenant_id, palf::SCN &version) = 0; + virtual int get_cluster_version(const uint64_t tenant_id, share::SCN &version) = 0; /// check server can start service or not virtual void check_server_can_start_service(bool &can_start_service, diff --git a/src/storage/tx/wrs/ob_ls_wrs_handler.cpp b/src/storage/tx/wrs/ob_ls_wrs_handler.cpp index c7ab01fff..2215fdee1 100644 --- a/src/storage/tx/wrs/ob_ls_wrs_handler.cpp +++ b/src/storage/tx/wrs/ob_ls_wrs_handler.cpp @@ -21,6 +21,7 @@ namespace oceanbase using namespace common; using namespace transaction; using namespace clog; +using namespace share; namespace storage { @@ -75,11 +76,11 @@ int ObLSWRSHandler::online() int ObLSWRSHandler::generate_ls_weak_read_snapshot_version(ObLS &ls, bool &need_skip, bool &is_user_ls, - palf::SCN &wrs_version, + SCN &wrs_version, const int64_t max_stale_time) { int ret = OB_SUCCESS; - palf::SCN timestamp; + SCN timestamp; const ObLSID &ls_id = ls.get_ls_id(); need_skip = false; ObMigrationStatus status = ObMigrationStatus::OB_MIGRATION_STATUS_NONE; @@ -101,7 +102,7 @@ int ObLSWRSHandler::generate_ls_weak_read_snapshot_version(ObLS &ls, } else if (OB_FAIL(ls.get_migration_status(status)) || ObMigrationStatus::OB_MIGRATION_STATUS_NONE == status ) { // check the weak read timestamp of the migrated ls - if (timestamp.get_val_for_lsn_allocator() > ObTimeUtility::current_time() - 500 * 1000) { + if (timestamp.get_val_for_logservice() > ObTimeUtility::current_time() - 500 * 1000) { STORAGE_LOG(TRACE, "ls received the latest log", K(timestamp)); // clog chases within 500ms, then clear the mark need_skip = false; @@ -110,7 +111,7 @@ int ObLSWRSHandler::generate_ls_weak_read_snapshot_version(ObLS &ls, } } else { int64_t snapshot_version_barrier = ObTimeUtility::current_time() - max_stale_time; - if (timestamp.get_val_for_lsn_allocator() <= snapshot_version_barrier) { + if (timestamp.get_val_for_logservice() <= snapshot_version_barrier) { // rule out these ls to avoid too old weak read timestamp need_skip = true; } else { @@ -136,10 +137,10 @@ int ObLSWRSHandler::generate_ls_weak_read_snapshot_version(ObLS &ls, return ret; } -int ObLSWRSHandler::generate_weak_read_timestamp_(ObLS &ls, const int64_t max_stale_time, palf::SCN ×tamp) +int ObLSWRSHandler::generate_weak_read_timestamp_(ObLS &ls, const int64_t max_stale_time, SCN ×tamp) { int ret = OB_SUCCESS; - palf::SCN min_log_service_scn, min_tx_service_ts; + SCN min_log_service_scn, min_tx_service_ts; const ObLSID &ls_id = ls.get_ls_id(); //the order of apply service、trx should not be changed @@ -164,7 +165,7 @@ int ObLSWRSHandler::generate_weak_read_timestamp_(ObLS &ls, const int64_t max_st STORAGE_LOG(WARN, "get_min_uncommit_prepare_version error", K(ret), K(ls_id)); } } else { - timestamp = palf::SCN::min(min_log_service_scn, min_tx_service_ts); + timestamp = SCN::min(min_log_service_scn, min_tx_service_ts); const int64_t current_us = ObClockGenerator::getClock(); if (current_us - timestamp.convert_to_ts() > 3000 * 1000L /*3s*/ || REACH_TIME_INTERVAL(10 * 1000 * 1000)) { diff --git a/src/storage/tx/wrs/ob_ls_wrs_handler.h b/src/storage/tx/wrs/ob_ls_wrs_handler.h index e45afa5f9..c412c78c4 100644 --- a/src/storage/tx/wrs/ob_ls_wrs_handler.h +++ b/src/storage/tx/wrs/ob_ls_wrs_handler.h @@ -17,7 +17,7 @@ #include #include "lib/lock/ob_spin_lock.h" #include "lib/utility/ob_macro_utils.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { namespace clog @@ -41,11 +41,11 @@ public: int generate_ls_weak_read_snapshot_version(oceanbase::storage::ObLS &ls, bool &need_skip, bool &is_user_ls, - palf::SCN &wrs_version, + share::SCN &wrs_version, const int64_t max_stale_time); - palf::SCN get_ls_weak_read_ts() const { return ls_weak_read_ts_; } + share::SCN get_ls_weak_read_ts() const { return ls_weak_read_ts_; } private: - int generate_weak_read_timestamp_(oceanbase::storage::ObLS &ls, const int64_t max_stale_time, palf::SCN ×tamp); + int generate_weak_read_timestamp_(oceanbase::storage::ObLS &ls, const int64_t max_stale_time, share::SCN ×tamp); private: DISALLOW_COPY_AND_ASSIGN(ObLSWRSHandler); @@ -55,7 +55,7 @@ protected: common::ObSpinLock lock_; bool is_inited_; bool is_enabled_; - palf::SCN ls_weak_read_ts_; + share::SCN ls_weak_read_ts_; }; } diff --git a/src/storage/tx/wrs/ob_tenant_weak_read_cluster_service.cpp b/src/storage/tx/wrs/ob_tenant_weak_read_cluster_service.cpp index bea363add..bc79dc787 100644 --- a/src/storage/tx/wrs/ob_tenant_weak_read_cluster_service.cpp +++ b/src/storage/tx/wrs/ob_tenant_weak_read_cluster_service.cpp @@ -171,8 +171,8 @@ void ObTenantWeakReadClusterService::update_valid_server_count_() #define QUERY_CLUSTER_VERSION_SQL "select min_version, max_version from %s where tenant_id = %lu and level_id = %d and level_value = ''" -int ObTenantWeakReadClusterService::query_cluster_version_range_(palf::SCN &cur_min_version, - palf::SCN &cur_max_version, +int ObTenantWeakReadClusterService::query_cluster_version_range_(SCN &cur_min_version, + SCN &cur_max_version, bool &record_exist) { int ret = OB_SUCCESS; @@ -234,10 +234,10 @@ int ObTenantWeakReadClusterService::query_cluster_version_range_(palf::SCN &cur_ where tenant_id = %lu and level_id = %d and level_value = '' and min_version = %lu and max_version = %lu \ " -int ObTenantWeakReadClusterService::build_update_version_sql_(const palf::SCN last_min_version, - const palf::SCN last_max_version, - const palf::SCN new_min_version, - const palf::SCN new_max_version, +int ObTenantWeakReadClusterService::build_update_version_sql_(const SCN last_min_version, + const SCN last_max_version, + const SCN new_min_version, + const SCN new_max_version, const bool record_exist, ObSqlString &sql) { @@ -265,10 +265,10 @@ int ObTenantWeakReadClusterService::build_update_version_sql_(const palf::SCN la return ret; } -int ObTenantWeakReadClusterService::persist_version_if_need_(const palf::SCN last_min_version, - const palf::SCN last_max_version, - const palf::SCN new_min_version, - const palf::SCN new_max_version, +int ObTenantWeakReadClusterService::persist_version_if_need_(const SCN last_min_version, + const SCN last_max_version, + const SCN new_min_version, + const SCN new_max_version, const bool record_exist, int64_t &affected_rows) { @@ -364,7 +364,7 @@ int ObTenantWeakReadClusterService::start_service() ret = OB_ERR_UNEXPECTED; LOG_ERROR("WRS leader epoch is invalid", KR(ret), K(leader_epoch)); } else { - palf::SCN cur_min_version, cur_max_version; + SCN cur_min_version, cur_max_version; bool record_exist = false; begin_query_ts = ObTimeUtility::current_time(); @@ -377,12 +377,12 @@ int ObTenantWeakReadClusterService::start_service() // new weak read version delay should smaller than max_stale_time int64_t new_version = MAX(cur_max_version.get_val_for_gts(), ObTimeUtility::current_time_ns() - max_stale_time); - palf::SCN new_version_scn; + SCN new_version_scn; if (OB_FAIL(new_version_scn.convert_for_gts(new_version))) { LOG_ERROR("convert for gts fail", KR(ret)); } else { - palf::SCN new_min_version = new_version_scn; - palf::SCN new_max_version = generate_max_version_(new_min_version); + SCN new_min_version = new_version_scn; + SCN new_max_version = generate_max_version_(new_min_version); int64_t affected_rows = 0; // do persist @@ -463,18 +463,18 @@ void ObTenantWeakReadClusterService::get_serve_info(bool &in_service, int64_t &l leader_epoch = leader_epoch_; } -int ObTenantWeakReadClusterService::get_version(palf::SCN &version) const +int ObTenantWeakReadClusterService::get_version(SCN &version) const { - palf::SCN min_version, max_version; + SCN min_version, max_version; return get_version(version, min_version, max_version); } -int ObTenantWeakReadClusterService::get_version(palf::SCN &version, palf::SCN &min_version, - palf::SCN &max_version) const +int ObTenantWeakReadClusterService::get_version(SCN &version, SCN &min_version, + SCN &max_version) const { int ret = OB_SUCCESS; static const int64_t GET_CLUSTER_VERSION_RDLOCK_TIMEOUT = 100; - palf::SCN ret_version; + SCN ret_version; int64_t rdlock_wait_time = GET_CLUSTER_VERSION_RDLOCK_TIMEOUT; // Lock wait time is necessary to prevent 'deadlock' @@ -610,7 +610,7 @@ bool ObTenantWeakReadClusterService::check_can_update_version_() int ObTenantWeakReadClusterService::update_version(int64_t &affected_rows) { int ret = OB_SUCCESS; - palf::SCN new_version; + SCN new_version; int64_t cur_leader_epoch = 0; int64_t skipped_server_count = 0; bool need_print = false; @@ -645,8 +645,8 @@ int ObTenantWeakReadClusterService::update_version(int64_t &affected_rows) // if new version exceeds the weak read version range([min, max)), do PERSIST // NOTE：Note that min_version maybe equal to max_version if (new_version > min_version_ && new_version >= max_version_) { - palf::SCN new_min_version = new_version; - palf::SCN new_max_version = generate_max_version_(new_min_version); + SCN new_min_version = new_version; + SCN new_max_version = generate_max_version_(new_min_version); // weak read record should be inserted in start service, and following update version only include UPDATE SQL bool record_exist = true; @@ -677,12 +677,12 @@ int ObTenantWeakReadClusterService::update_version(int64_t &affected_rows) return ret; } -palf::SCN ObTenantWeakReadClusterService::compute_version_(int64_t &skipped_servers, bool need_print) const +SCN ObTenantWeakReadClusterService::compute_version_(int64_t &skipped_servers, bool need_print) const { /// min weak read version delay should not smaller than max_stale_time - palf::SCN base_version = ObWeakReadUtil::generate_min_weak_read_version(MTL_ID()); + SCN base_version = ObWeakReadUtil::generate_min_weak_read_version(MTL_ID()); // weak read version should increase monotonically - base_version = palf::SCN::max(current_version_, base_version); + base_version = SCN::max(current_version_, base_version); return cluster_version_mgr_.get_version(base_version, skipped_servers, need_print); } @@ -976,7 +976,7 @@ void ObTenantWeakReadClusterService::stop_service_impl_() } int ObTenantWeakReadClusterService::update_server_version(const common::ObAddr &addr, - const palf::SCN version, + const SCN version, const int64_t valid_part_count, const int64_t total_part_count, const int64_t generate_timestamp) diff --git a/src/storage/tx/wrs/ob_tenant_weak_read_cluster_service.h b/src/storage/tx/wrs/ob_tenant_weak_read_cluster_service.h index 5ce82b2a5..ea80aa50f 100644 --- a/src/storage/tx/wrs/ob_tenant_weak_read_cluster_service.h +++ b/src/storage/tx/wrs/ob_tenant_weak_read_cluster_service.h @@ -80,8 +80,8 @@ public: /// @retval OB_NOT_MASTER self is in service, but not wrs leader, should stop service /// @retval OB_NEED_RETRY wrs not ready，need retry /// @retval OTHER CODE fail - int get_version(palf::SCN &version) const; - int get_version(palf::SCN &version, palf::SCN &min_version, palf::SCN &max_version) const; + int get_version(share::SCN &version) const; + int get_version(share::SCN &version, share::SCN &min_version, share::SCN &max_version) const; /// update wrs version /// @@ -114,7 +114,7 @@ public: /// update server level weak read version int update_server_version(const common::ObAddr &addr, - const palf::SCN version, + const share::SCN version, const int64_t valid_part_count, const int64_t total_part_count, const int64_t generate_timestamp); @@ -122,28 +122,28 @@ public: private: int check_leader_info_(int64_t &leader_epoch) const; void update_valid_server_count_(); - int query_cluster_version_range_(palf::SCN &cur_min_version, palf::SCN &cur_max_version, + int query_cluster_version_range_(share::SCN &cur_min_version, share::SCN &cur_max_version, bool &record_exist); - int persist_version_if_need_(const palf::SCN last_min_version, - const palf::SCN last_max_version, - const palf::SCN new_min_version, - const palf::SCN new_max_version, + int persist_version_if_need_(const share::SCN last_min_version, + const share::SCN last_max_version, + const share::SCN new_min_version, + const share::SCN new_max_version, const bool record_exist, int64_t &affected_rows); - int build_update_version_sql_(const palf::SCN last_min_version, - const palf::SCN last_max_version, - const palf::SCN new_min_version, - const palf::SCN new_max_version, + int build_update_version_sql_(const share::SCN last_min_version, + const share::SCN last_max_version, + const share::SCN new_min_version, + const share::SCN new_max_version, const bool record_exist, common::ObSqlString &sql); bool check_can_update_version_(); - palf::SCN generate_max_version_(const palf::SCN min_version) const + share::SCN generate_max_version_(const share::SCN min_version) const { - return palf::SCN::plus(min_version, WRS_VERSION_GAP_FOR_PERSISTENCE * 1000); + return share::SCN::plus(min_version, WRS_VERSION_GAP_FOR_PERSISTENCE * 1000); } void stop_service_impl_(); - palf::SCN compute_version_(int64_t &skipped_servers, bool need_print) const; - int get_version_(palf::SCN &version, palf::SCN &min_version, palf::SCN &max_version) const; + share::SCN compute_version_(int64_t &skipped_servers, bool need_print) const; + int get_version_(share::SCN &version, share::SCN &min_version, share::SCN &max_version) const; bool need_force_change_leader_(); int force_change_leader_() const; int verify_candidate_server_(const common::ObAddr &server) const; @@ -187,9 +187,9 @@ private: int64_t all_valid_server_count_; // current version cached in local - palf::SCN current_version_ CACHE_ALIGNED; - palf::SCN min_version_ CACHE_ALIGNED; - palf::SCN max_version_; + share::SCN current_version_ CACHE_ALIGNED; + share::SCN min_version_ CACHE_ALIGNED; + share::SCN max_version_; ClusterVersionMgr cluster_version_mgr_; diff --git a/src/storage/tx/wrs/ob_tenant_weak_read_cluster_version_mgr.cpp b/src/storage/tx/wrs/ob_tenant_weak_read_cluster_version_mgr.cpp index 381abc394..7d9dff3fa 100644 --- a/src/storage/tx/wrs/ob_tenant_weak_read_cluster_version_mgr.cpp +++ b/src/storage/tx/wrs/ob_tenant_weak_read_cluster_version_mgr.cpp @@ -18,6 +18,7 @@ namespace oceanbase { using namespace common; +using namespace share; namespace transaction { @@ -64,7 +65,7 @@ bool ObTenantWeakReadClusterVersionMgr::find_match_server(int64_t &pre_count, } int ObTenantWeakReadClusterVersionMgr::update_server_version(const common::ObAddr &addr, - const palf::SCN version, + const SCN version, const int64_t valid_part_count, const int64_t total_part_count, const int64_t generate_tstamp, @@ -119,11 +120,11 @@ int ObTenantWeakReadClusterVersionMgr::update_server_version(const common::ObAdd return ret; } -palf::SCN ObTenantWeakReadClusterVersionMgr::get_version(const palf::SCN base_version, +SCN ObTenantWeakReadClusterVersionMgr::get_version(const SCN base_version, int64_t &skip_server_count, const bool force_print) const { - palf::SCN min_version; + SCN min_version; ObSEArray skip_servers; bool need_print = force_print; @@ -133,7 +134,7 @@ palf::SCN ObTenantWeakReadClusterVersionMgr::get_version(const palf::SCN base_ve for (int64_t i = 0; i < svr_array_.count(); i++) { bool need_skip = false; bool is_first_skipped = false; - palf::SCN version = svr_array_.at(i).get_version(need_skip, is_first_skipped); + SCN version = svr_array_.at(i).get_version(need_skip, is_first_skipped); if (need_skip) { skip_servers.push_back(svr_array_.at(i).addr_); @@ -185,7 +186,7 @@ ObTenantWeakReadClusterVersionMgr::ServerInfo::ServerInfo() : } ObTenantWeakReadClusterVersionMgr::ServerInfo::ServerInfo(const ObAddr &addr, - const palf::SCN version, + const SCN version, const int64_t valid_part_count, const int64_t total_part_count) { @@ -203,7 +204,7 @@ bool ObTenantWeakReadClusterVersionMgr::ServerInfo::match(const common::ObAddr & return addr_ == addr; } -void ObTenantWeakReadClusterVersionMgr::ServerInfo::update(const palf::SCN version, +void ObTenantWeakReadClusterVersionMgr::ServerInfo::update(const SCN version, const int64_t valid_part_count, const int64_t total_part_count, const int64_t generate_tstamp) @@ -219,11 +220,11 @@ void ObTenantWeakReadClusterVersionMgr::ServerInfo::update(const palf::SCN versi } } -palf::SCN ObTenantWeakReadClusterVersionMgr::ServerInfo::get_version(bool &need_skip, +SCN ObTenantWeakReadClusterVersionMgr::ServerInfo::get_version(bool &need_skip, bool &is_first_skipped) const { int64_t cur_tstamp = ObTimeUtility::current_time(); - palf::SCN ret_version; + SCN ret_version; SpinLockGuard guard(lock_); diff --git a/src/storage/tx/wrs/ob_tenant_weak_read_cluster_version_mgr.h b/src/storage/tx/wrs/ob_tenant_weak_read_cluster_version_mgr.h index dce7dfa0e..23bba5a10 100644 --- a/src/storage/tx/wrs/ob_tenant_weak_read_cluster_version_mgr.h +++ b/src/storage/tx/wrs/ob_tenant_weak_read_cluster_version_mgr.h @@ -17,14 +17,10 @@ #include "lib/lock/ob_spin_rwlock.h" // SpinRWLock #include "lib/lock/ob_small_spin_lock.h" // ObByteLock #include "lib/container/ob_se_array.h" // ObSEArray -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { -namespace palf -{ -class SCN; -} namespace transaction { @@ -52,7 +48,7 @@ public: /// @retval OB_SUCCESS success /// @retval OTHER CODE fail int update_server_version(const common::ObAddr &addr, - const palf::SCN version, + const share::SCN version, const int64_t valid_part_count, const int64_t total_part_count, const int64_t generate_tstamp, @@ -60,7 +56,7 @@ public: /// get min server version which not smaller than base_version /// if no statisfied server version, return base_version - palf::SCN get_version(const palf::SCN base_version, int64_t &skip_server_count, const bool need_print_server_info) const; + share::SCN get_version(const share::SCN base_version, int64_t &skip_server_count, const bool need_print_server_info) const; // get server count in cluster master cached registered servers int64_t get_server_count() const; @@ -80,7 +76,7 @@ private: struct ServerInfo { common::ObAddr addr_; // server addr - palf::SCN version_; // server weak read version + share::SCN version_; // server weak read version int64_t valid_part_count_; // valid partition count int64_t total_part_count_; // total partition count int64_t generate_tstamp_; // server weak read version generation timestamp @@ -90,15 +86,15 @@ private: ServerInfo(); ServerInfo(const common::ObAddr &addr, - const palf::SCN verison, + const share::SCN verison, const int64_t valid_part_count, const int64_t total_part_count); bool match(const common::ObAddr &addr) const; - void update(const palf::SCN version, + void update(const share::SCN version, const int64_t valid_part_count, const int64_t total_part_count, const int64_t generate_tstamp); - palf::SCN get_version(bool &need_skip, bool &is_first_skipped) const; + share::SCN get_version(bool &need_skip, bool &is_first_skipped) const; TO_STRING_KV(K_(addr), K_(version), K_(valid_part_count), K_(total_part_count), K_(generate_tstamp), K_(is_skipped)); diff --git a/src/storage/tx/wrs/ob_tenant_weak_read_server_version_mgr.cpp b/src/storage/tx/wrs/ob_tenant_weak_read_server_version_mgr.cpp index fbeeb00c3..181595c14 100644 --- a/src/storage/tx/wrs/ob_tenant_weak_read_server_version_mgr.cpp +++ b/src/storage/tx/wrs/ob_tenant_weak_read_server_version_mgr.cpp @@ -17,6 +17,7 @@ #include "ob_tenant_weak_read_server_version_mgr.h" using namespace oceanbase::common; +using namespace oceanbase::share; namespace oceanbase { namespace transaction @@ -31,9 +32,9 @@ ObTenantWeakReadServerVersionMgr::ObTenantWeakReadServerVersionMgr() : ObTenantWeakReadServerVersionMgr::~ObTenantWeakReadServerVersionMgr() {} -palf::SCN ObTenantWeakReadServerVersionMgr::get_version() const +SCN ObTenantWeakReadServerVersionMgr::get_version() const { - palf::SCN ret_version; + SCN ret_version; ServerVersion sv; get_version(sv); @@ -42,7 +43,7 @@ palf::SCN ObTenantWeakReadServerVersionMgr::get_version() const return ret_version; } -palf::SCN ObTenantWeakReadServerVersionMgr::get_version(int64_t &total_part_count, +SCN ObTenantWeakReadServerVersionMgr::get_version(int64_t &total_part_count, int64_t &valid_part_count) const { ServerVersion sv; @@ -68,7 +69,7 @@ int ObTenantWeakReadServerVersionMgr::update_with_part_info(const uint64_t tenan const int64_t epoch_tstamp, const bool need_skip, const bool is_user_part, - const palf::SCN version) + const SCN version) { int ret = OB_SUCCESS; if (OB_UNLIKELY(epoch_tstamp <= 0) @@ -85,7 +86,7 @@ int ObTenantWeakReadServerVersionMgr::update_with_part_info(const uint64_t tenan int ObTenantWeakReadServerVersionMgr::generate_new_version(const uint64_t tenant_id, const int64_t epoch_tstamp, - const palf::SCN base_version_when_no_valid_partition, + const SCN base_version_when_no_valid_partition, const bool need_print_status) { int ret = OB_SUCCESS; @@ -128,7 +129,7 @@ int ObTenantWeakReadServerVersionMgr::generate_new_version(const uint64_t tenant void ObTenantWeakReadServerVersionMgr::ServerVersionInner::update_with_part_info(const int64_t epoch_tstamp, const bool need_skip, const bool is_user_part, - const palf::SCN version) + const SCN version) { if (epoch_tstamp != epoch_tstamp_) { // new epoch statistic start @@ -137,7 +138,7 @@ void ObTenantWeakReadServerVersionMgr::ServerVersionInner::update_with_part_info } if (! need_skip) { - palf::SCN &target_version = version_; + SCN &target_version = version_; int64_t &target_valid_count = is_user_part ? valid_user_part_count_ : valid_inner_part_count_; // update target version if (!target_version.is_valid()) { @@ -159,7 +160,7 @@ void ObTenantWeakReadServerVersionMgr::ServerVersionInner::update_with_part_info // generate weak read version for tenant without partitions in local server int ObTenantWeakReadServerVersionMgr::ServerVersionInner::amend(const uint64_t tenant_id, const int64_t new_epoch_tstamp, - const palf::SCN base_version_when_no_valid_partition) + const SCN base_version_when_no_valid_partition) { int ret = OB_SUCCESS; static const int64_t PRINT_INTERVAL = 10 * 1000 * 1000; @@ -214,7 +215,7 @@ int ObTenantWeakReadServerVersionMgr::ServerVersionInner::update(const ServerVer valid_user_part_count_ = new_version.valid_user_part_count_; // version should incease monotonically - version_ = palf::SCN::max(new_version.version_, version_); + version_ = SCN::max(new_version.version_, version_); } return ret; } diff --git a/src/storage/tx/wrs/ob_tenant_weak_read_server_version_mgr.h b/src/storage/tx/wrs/ob_tenant_weak_read_server_version_mgr.h index 4d9e36b31..dddb22cf2 100644 --- a/src/storage/tx/wrs/ob_tenant_weak_read_server_version_mgr.h +++ b/src/storage/tx/wrs/ob_tenant_weak_read_server_version_mgr.h @@ -15,7 +15,7 @@ #include "lib/lock/ob_spin_rwlock.h" // SpinRWLock #include "lib/utility/ob_print_utils.h" // TO_STRING_KV -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -28,8 +28,8 @@ public: ObTenantWeakReadServerVersionMgr(); ~ObTenantWeakReadServerVersionMgr(); - palf::SCN get_version() const; - palf::SCN get_version(int64_t &total_part_count, int64_t &valid_part_count) const; + share::SCN get_version() const; + share::SCN get_version(int64_t &total_part_count, int64_t &valid_part_count) const; class ServerVersion; void get_version(ServerVersion &sv) const; @@ -41,12 +41,12 @@ public: const int64_t epoch_tstamp, const bool need_skip, const bool is_user_part, - const palf::SCN version); + const share::SCN version); // generate new SERVER level weak read version after scan all partitions int generate_new_version(const uint64_t tenant_id, const int64_t epoch_tstamp, - const palf::SCN base_version_when_no_valid_partition, + const share::SCN base_version_when_no_valid_partition, const bool need_print_status); public: @@ -55,7 +55,7 @@ public: public: struct ServerVersion { - palf::SCN version_; // server version, including inner and user partitions + share::SCN version_; // server version, including inner and user partitions int64_t total_part_count_; // total partition count int64_t valid_inner_part_count_; // valid inner partition count int64_t valid_user_part_count_; // valid user partition count @@ -68,7 +68,7 @@ public: valid_inner_part_count_ = 0; valid_user_part_count_ = 0; } - void reset(const palf::SCN version, + void reset(const share::SCN version, const int64_t total_part_count, const int64_t valid_inner_part_count, const int64_t valid_user_part_count) @@ -104,14 +104,14 @@ private: void update_with_part_info(const int64_t epoch_tstamp, const bool need_skip, const bool is_user_part, - const palf::SCN version); + const share::SCN version); // update based on all partition statistic result int update(const ServerVersionInner &new_version); // amend weak read version based on epoch timestamp int amend(const uint64_t tenant_id, const int64_t new_epoch_tstamp, - const palf::SCN base_version_when_no_valid_partition); + const share::SCN base_version_when_no_valid_partition); TO_STRING_KV(K_(version), K_(total_part_count), diff --git a/src/storage/tx/wrs/ob_tenant_weak_read_service.cpp b/src/storage/tx/wrs/ob_tenant_weak_read_service.cpp index 2303312aa..f0505d47e 100644 --- a/src/storage/tx/wrs/ob_tenant_weak_read_service.cpp +++ b/src/storage/tx/wrs/ob_tenant_weak_read_service.cpp @@ -167,13 +167,13 @@ void ObTenantWeakReadService::destroy() TG_DESTROY(tg_id_); } -palf::SCN ObTenantWeakReadService::get_server_version() const +SCN ObTenantWeakReadService::get_server_version() const { return svr_version_mgr_.get_version(); } // if weak_read_refresh_interval is 0, wrs close, and return OB_NOT_SUPPORTED -int ObTenantWeakReadService::get_cluster_version(palf::SCN &cur_version) +int ObTenantWeakReadService::get_cluster_version(SCN &cur_version) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!ObWeakReadUtil::check_weak_read_service_available())) { @@ -191,7 +191,7 @@ int ObTenantWeakReadService::get_cluster_version(palf::SCN &cur_version) ret = OB_TRANS_WEAK_READ_VERSION_NOT_READY; LOG_WARN("get cluster version fail, need retry", K(old_ret), KR(ret), K(tenant_id_)); } else { - palf::SCN last_local_cluster_version = local_cluster_version_.atomic_get(); + SCN last_local_cluster_version = local_cluster_version_.atomic_get(); local_cluster_version_.atomic_set(cur_version); LOG_TRACE("get cluster version", K(cur_version), K(last_local_cluster_version)); } @@ -199,7 +199,7 @@ int ObTenantWeakReadService::get_cluster_version(palf::SCN &cur_version) return ret; } -int ObTenantWeakReadService::get_cluster_version_internal_(palf::SCN &version, +int ObTenantWeakReadService::get_cluster_version_internal_(SCN &version, const bool only_request_local) { int ret = OB_SUCCESS; @@ -228,7 +228,7 @@ int ObTenantWeakReadService::get_cluster_version_internal_(palf::SCN &version, return ret; } -int ObTenantWeakReadService::get_cluster_version_by_rpc_(palf::SCN &version) +int ObTenantWeakReadService::get_cluster_version_by_rpc_(SCN &version) { int ret = OB_SUCCESS; ObAddr cluster_service_master; @@ -298,7 +298,7 @@ int ObTenantWeakReadService::get_cluster_service_master_(common::ObAddr &cluster int ObTenantWeakReadService::update_server_version_with_part_info(const int64_t epoch_tstamp, const bool need_skip, const bool is_user_part, - const palf::SCN version) + const SCN version) { return svr_version_mgr_.update_with_part_info(tenant_id_, epoch_tstamp, need_skip, is_user_part, version); } @@ -306,7 +306,7 @@ int ObTenantWeakReadService::update_server_version_with_part_info(const int64_t int ObTenantWeakReadService::generate_server_version(const int64_t epoch_tstamp, const bool need_print_status) { - palf::SCN base_version_when_no_valid_partition = ObWeakReadUtil::generate_min_weak_read_version(tenant_id_); + SCN base_version_when_no_valid_partition = ObWeakReadUtil::generate_min_weak_read_version(tenant_id_); return svr_version_mgr_.generate_new_version(tenant_id_, epoch_tstamp, base_version_when_no_valid_partition, need_print_status); } @@ -316,9 +316,9 @@ void ObTenantWeakReadService::get_weak_read_stat(ObTenantWeakReadStat &wrs_stat) bool in_service = 0; int64_t master_epoch = 0; - palf::SCN current_cluster_version; - palf::SCN min_cluster_version; - palf::SCN max_cluster_version; + SCN current_cluster_version; + SCN min_cluster_version; + SCN max_cluster_version; int64_t cur_tstamp = ObTimeUtility::current_time(); cluster_service_.get_serve_info(in_service, master_epoch); @@ -401,7 +401,7 @@ void ObTenantWeakReadService::set_force_self_check_(bool need_stop_service) thread_cond_.signal(); } -int ObTenantWeakReadService::process_get_cluster_version_rpc(palf::SCN &version) +int ObTenantWeakReadService::process_get_cluster_version_rpc(SCN &version) { int ret = OB_SUCCESS; bool only_request_local = true; @@ -442,7 +442,7 @@ void ObTenantWeakReadService::process_cluster_heartbeat_rpc_cb( } int ObTenantWeakReadService::process_cluster_heartbeat_rpc(const common::ObAddr &svr, - const palf::SCN version, + const SCN version, const int64_t valid_part_count, const int64_t total_part_count, const int64_t generate_timestamp) @@ -494,9 +494,9 @@ void ObTenantWeakReadService::print_stat_() int get_cluster_version_err = 0; int64_t cur_tstamp = ObTimeUtility::current_time(); ObTenantWeakReadServerVersionMgr::ServerVersion sv; - palf::SCN cluster_version; - palf::SCN min_cluster_version; - palf::SCN max_cluster_version; + SCN cluster_version; + SCN min_cluster_version; + SCN max_cluster_version; bool in_cluster_service = cluster_service_.is_in_service(); int64_t weak_read_refresh_interval = GCONF.weak_read_version_refresh_interval; @@ -650,7 +650,7 @@ int ObTenantWeakReadService::scan_all_ls_(storage::ObLSService *ls_svr) int ObTenantWeakReadService::handle_ls_(ObLS &ls) { int ret = OB_SUCCESS; - palf::SCN version; + SCN version; bool need_skip = false; bool is_user_ls = false; const ObLSID &ls_id = ls.get_ls_id(); @@ -726,7 +726,7 @@ void ObTenantWeakReadService::do_cluster_heartbeat_() DEFINE_TIME_GUARD("do_cluster_heartbeat", 100 * 1000L); // get local server version - palf::SCN local_server_version = svr_version_mgr_.get_version(total_part_count, valid_part_count); + SCN local_server_version = svr_version_mgr_.get_version(total_part_count, valid_part_count); time_guard.click("get_server_version"); @@ -784,7 +784,7 @@ void ObTenantWeakReadService::do_cluster_heartbeat_() ATOMIC_SET(&last_post_cluster_heartbeat_tstamp_, ObTimeUtility::current_time()); } -int ObTenantWeakReadService::post_cluster_heartbeat_rpc_(const palf::SCN version, +int ObTenantWeakReadService::post_cluster_heartbeat_rpc_(const SCN version, const int64_t valid_part_count, const int64_t total_part_count, const int64_t generate_timestamp) diff --git a/src/storage/tx/wrs/ob_tenant_weak_read_service.h b/src/storage/tx/wrs/ob_tenant_weak_read_service.h index bec78c083..18da7af97 100644 --- a/src/storage/tx/wrs/ob_tenant_weak_read_service.h +++ b/src/storage/tx/wrs/ob_tenant_weak_read_service.h @@ -66,13 +66,13 @@ public: void destroy(); /// get SERVER level weak read version - palf::SCN get_server_version() const; + share::SCN get_server_version() const; /// get CLUSTER level weak read version /// /// @retval OB_SUCCESS success /// @retval OB_TRANS_WEAK_READ_VERSION_NOT_READY fail - int get_cluster_version(palf::SCN &version); + int get_cluster_version(share::SCN &version); // update SERVER level weak read version based on partition readable snapshot version in partition iteration // @@ -80,7 +80,7 @@ public: int update_server_version_with_part_info(const int64_t epoch_tstamp, const bool need_skip, const bool is_user_part, - const palf::SCN version); + const share::SCN version); // generate new SERVER level weak read version after scan all partitions int generate_server_version(const int64_t epoch_tstamp, @@ -92,7 +92,7 @@ public: /// process Cluster level RPC int process_cluster_heartbeat_rpc(const common::ObAddr &svr, - const palf::SCN version, + const share::SCN version, const int64_t valid_part_count, const int64_t total_part_count, const int64_t generate_timestamp); @@ -104,7 +104,7 @@ public: /// @retval OB_NOT_IN_SERVICE self not in service /// @retval OB_NOT_MASTER self is not MASTER /// @retval OTHER CODE fail - int process_get_cluster_version_rpc(palf::SCN &version); + int process_get_cluster_version_rpc(share::SCN &version); // get weak read info stat void get_weak_read_stat(ObTenantWeakReadStat &wrs_stat) const; @@ -124,13 +124,13 @@ private: /// @retval OB_NOT_IN_SERVICE self not in service /// @retval OB_NOT_MASTER self is not MASTER /// @retval OTHER CODE fail - int get_cluster_version_internal_(palf::SCN &version, const bool only_request_local); - int get_cluster_version_by_rpc_(palf::SCN &version); + int get_cluster_version_internal_(share::SCN &version, const bool only_request_local); + int get_cluster_version_by_rpc_(share::SCN &version); int get_cluster_service_master_(common::ObAddr &cluster_service_master); void refresh_cluster_service_master_(); /// do Cluster Heartbeat request，report self's server version void do_cluster_heartbeat_(); - int post_cluster_heartbeat_rpc_(const palf::SCN version, + int post_cluster_heartbeat_rpc_(const share::SCN version, const int64_t valid_part_count, const int64_t total_part_count, const int64_t generate_timestamp); @@ -187,7 +187,7 @@ private: // last cluster version generation timestamp int64_t last_generate_cluster_version_tstamp_; // cluster version the last get_version request get - palf::SCN local_cluster_version_; + share::SCN local_cluster_version_; // need force self check or not bool force_self_check_; diff --git a/src/storage/tx/wrs/ob_tenant_weak_read_stat.h b/src/storage/tx/wrs/ob_tenant_weak_read_stat.h index 4b260e3b1..08b496f9f 100644 --- a/src/storage/tx/wrs/ob_tenant_weak_read_stat.h +++ b/src/storage/tx/wrs/ob_tenant_weak_read_stat.h @@ -33,12 +33,12 @@ public: public: uint64_t tenant_id_; //server level weak read info stat - palf::SCN server_version_; // server level weak read version + share::SCN server_version_; // server level weak read version int64_t total_part_count_; // total partition count int64_t valid_inner_part_count_; // valid inner partition count int64_t valid_user_part_count_; int64_t server_version_delta_; - palf::SCN local_cluster_version_; + share::SCN local_cluster_version_; int64_t local_cluster_version_delta_; common::ObAddr self_; @@ -60,10 +60,10 @@ public: int64_t cluster_servers_count_; // server count int64_t cluster_skipped_servers_count_; // skipped server count int64_t cluster_version_gen_tstamp_; // last cluster version generation timestamp - palf::SCN cluster_version_; // cluster level weak read version + share::SCN cluster_version_; // cluster level weak read version int64_t cluster_version_delta_; - palf::SCN min_cluster_version_; - palf::SCN max_cluster_version_; + share::SCN min_cluster_version_; + share::SCN max_cluster_version_; }; }// transaction diff --git a/src/storage/tx/wrs/ob_weak_read_service.cpp b/src/storage/tx/wrs/ob_weak_read_service.cpp index 1fbf3b01e..39ed2f86d 100644 --- a/src/storage/tx/wrs/ob_weak_read_service.cpp +++ b/src/storage/tx/wrs/ob_weak_read_service.cpp @@ -76,7 +76,7 @@ void ObWeakReadService::wait() LOG_INFO("[WRS] weak read service thread wait"); } -int ObWeakReadService::get_server_version(const uint64_t tenant_id, palf::SCN &version) const +int ObWeakReadService::get_server_version(const uint64_t tenant_id, SCN &version) const { int ret = OB_SUCCESS; // switch tenant @@ -104,7 +104,7 @@ int ObWeakReadService::get_server_version(const uint64_t tenant_id, palf::SCN &v return ret; } -int ObWeakReadService::get_cluster_version(const uint64_t tenant_id, palf::SCN &version) +int ObWeakReadService::get_cluster_version(const uint64_t tenant_id, SCN &version) { int ret = OB_SUCCESS; // switch tenant @@ -164,7 +164,7 @@ void ObWeakReadService::process_get_cluster_version_rpc(const uint64_t tenant_id obrpc::ObWrsGetClusterVersionResponse &res) { int ret = OB_SUCCESS; - palf::SCN version; + SCN version; MTL_SWITCH(tenant_id) { ObTenantWeakReadService *twrs = NULL; diff --git a/src/storage/tx/wrs/ob_weak_read_service.h b/src/storage/tx/wrs/ob_weak_read_service.h index c8e0287e8..3530360c2 100644 --- a/src/storage/tx/wrs/ob_weak_read_service.h +++ b/src/storage/tx/wrs/ob_weak_read_service.h @@ -46,9 +46,9 @@ public: void wait(); public: /// get SERVER level weak read version - int get_server_version(const uint64_t tenant_id, palf::SCN &version) const; + int get_server_version(const uint64_t tenant_id, share::SCN &version) const; /// get CLUSTER level weak read version - int get_cluster_version(const uint64_t tenant_id, palf::SCN &version); + int get_cluster_version(const uint64_t tenant_id, share::SCN &version); void check_server_can_start_service(bool &can_start_service, int64_t &min_wrs) const; diff --git a/src/storage/tx/wrs/ob_weak_read_service_rpc_define.h b/src/storage/tx/wrs/ob_weak_read_service_rpc_define.h index 240f40daf..cc94d68e0 100644 --- a/src/storage/tx/wrs/ob_weak_read_service_rpc_define.h +++ b/src/storage/tx/wrs/ob_weak_read_service_rpc_define.h @@ -44,11 +44,11 @@ struct ObWrsGetClusterVersionRequest struct ObWrsGetClusterVersionResponse { int err_code_; // error code - palf::SCN version_; // weak read version + share::SCN version_; // weak read version int64_t version_duration_us_; // weak read version duration time, for future 'version on Lease' compat ObWrsGetClusterVersionResponse() : err_code_(0), version_(), version_duration_us_(0) {} - void set(const int err, const palf::SCN version, const int64_t version_duration_us = 0) + void set(const int err, const share::SCN version, const int64_t version_duration_us = 0) { err_code_ = err; version_ = version; @@ -64,7 +64,7 @@ struct ObWrsGetClusterVersionResponse struct ObWrsClusterHeartbeatRequest { common::ObAddr req_server_; // Who I am - palf::SCN version_; // What my weak read version is + share::SCN version_; // What my weak read version is int64_t valid_part_count_; // How many valid partition I have int64_t total_part_count_; // How many partition I have int64_t generate_timestamp_; // server version generation timestamp @@ -78,7 +78,7 @@ struct ObWrsClusterHeartbeatRequest {} void set(const common::ObAddr &svr, - const palf::SCN version, + const share::SCN version, const int64_t valid_part_count, const int64_t total_part_count, const int64_t generate_timestamp) diff --git a/src/storage/tx/wrs/ob_weak_read_util.cpp b/src/storage/tx/wrs/ob_weak_read_util.cpp index 365529670..393cb8eb8 100644 --- a/src/storage/tx/wrs/ob_weak_read_util.cpp +++ b/src/storage/tx/wrs/ob_weak_read_util.cpp @@ -19,6 +19,7 @@ namespace oceanbase { using namespace common; +using namespace share; namespace transaction { // 1. follower if readable depend on follower readable snapshot version and weak read cluster version @@ -44,9 +45,9 @@ int64_t ObWeakReadUtil::replica_keepalive_interval() // 2. all partitions offline // 3. all partitions delay too much or in invalid status // 4. all partitions in migrating and readable snapshot version delay more than 500ms -palf::SCN ObWeakReadUtil::generate_min_weak_read_version(const uint64_t tenant_id) +SCN ObWeakReadUtil::generate_min_weak_read_version(const uint64_t tenant_id) { - palf::SCN base_version_when_no_valid_partition; + SCN base_version_when_no_valid_partition; int64_t max_stale_time = 0; bool tenant_config_exist = false; // generating min weak version version should statisfy following constraint diff --git a/src/storage/tx/wrs/ob_weak_read_util.h b/src/storage/tx/wrs/ob_weak_read_util.h index fa6135ffd..5480415bc 100644 --- a/src/storage/tx/wrs/ob_weak_read_util.h +++ b/src/storage/tx/wrs/ob_weak_read_util.h @@ -16,7 +16,7 @@ #include "stdint.h" namespace oceanbase { -namespace palf +namespace share { class SCN; } @@ -31,7 +31,7 @@ public: static const int64_t DEFAULT_REPLICA_KEEPALIVE_INTERVAL = 3000 * 1000L; static const int64_t IGNORE_TENANT_EXIST_WARN = 1; static int64_t replica_keepalive_interval(); - static palf::SCN generate_min_weak_read_version(const uint64_t tenant_id); + static share::SCN generate_min_weak_read_version(const uint64_t tenant_id); static bool enable_monotonic_weak_read(const uint64_t tenant_id); static int64_t max_stale_time_for_weak_consistency(const uint64_t tenant_id, int64_t ignore_warn = 0); static bool check_weak_read_service_available(); diff --git a/src/storage/tx_storage/ob_access_service.cpp b/src/storage/tx_storage/ob_access_service.cpp index a7ce0dd2e..224fe61d3 100644 --- a/src/storage/tx_storage/ob_access_service.cpp +++ b/src/storage/tx_storage/ob_access_service.cpp @@ -27,6 +27,7 @@ namespace oceanbase { using namespace common; +using namespace share; namespace storage { @@ -212,7 +213,7 @@ int ObAccessService::table_scan( ObTableScanParam ¶m = static_cast(vparam); ObStoreAccessType access_type = param.scan_flag_.is_read_latest() ? ObStoreAccessType::READ_LATEST : ObStoreAccessType::READ; - palf::SCN user_specified_snapshot_scn; + SCN user_specified_snapshot_scn; if (ObAccessTypeCheck::is_read_access_type(access_type) && param.fb_snapshot_.is_valid()) { //todo lixinze:后续会判断是否有效 user_specified_snapshot_scn = param.fb_snapshot_; @@ -291,7 +292,7 @@ int ObAccessService::table_rescan( ObLSTabletService *tablet_service = nullptr; ObStoreAccessType access_type = param.scan_flag_.is_read_latest() ? ObStoreAccessType::READ_LATEST : ObStoreAccessType::READ; - palf::SCN user_specified_snapshot_scn; + SCN user_specified_snapshot_scn; if (ObAccessTypeCheck::is_read_access_type(access_type) && param.fb_snapshot_.is_valid()) { //todo lixinze:后续会判断是否有效 user_specified_snapshot_scn = param.fb_snapshot_; @@ -384,7 +385,7 @@ int ObAccessService::check_read_allowed_( const ObStoreAccessType access_type, const ObTableScanParam &scan_param, ObStoreCtxGuard &ctx_guard, - palf::SCN user_specified_snapshot) + SCN user_specified_snapshot) { int ret = OB_SUCCESS; ObLS *ls = nullptr; diff --git a/src/storage/tx_storage/ob_access_service.h b/src/storage/tx_storage/ob_access_service.h index 7d1ca4074..c819ba656 100644 --- a/src/storage/tx_storage/ob_access_service.h +++ b/src/storage/tx_storage/ob_access_service.h @@ -211,7 +211,7 @@ protected: const ObStoreAccessType access_type, const ObTableScanParam &scan_param, ObStoreCtxGuard &ctx_guard, - palf::SCN user_specified_snapshot); + share::SCN user_specified_snapshot); int check_read_allowed_( const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, diff --git a/src/storage/tx_storage/ob_checkpoint_service.cpp b/src/storage/tx_storage/ob_checkpoint_service.cpp index 1dc3fc121..d953811cc 100644 --- a/src/storage/tx_storage/ob_checkpoint_service.cpp +++ b/src/storage/tx_storage/ob_checkpoint_service.cpp @@ -28,6 +28,7 @@ namespace oceanbase { +using namespace share; using namespace palf; namespace storage { @@ -106,7 +107,7 @@ void ObCheckPointService::wait() int ObCheckPointService::add_ls_freeze_task( ObDataCheckpoint *data_checkpoint, - palf::SCN rec_scn) + SCN rec_scn) { int ret = OB_SUCCESS; if (OB_FAIL(freeze_thread_.add_task(data_checkpoint, rec_scn))) { @@ -358,7 +359,7 @@ int ObCheckPointService::do_minor_freeze() ObCheckpointExecutor *checkpoint_executor = nullptr; if (OB_ISNULL(checkpoint_executor = ls->get_checkpoint_executor())) { STORAGE_LOG(WARN, "checkpoint_executor should not be null", K(ls->get_ls_id())); - } else if (OB_SUCCESS != (tmp_ret = (checkpoint_executor->advance_checkpoint_by_flush(palf::SCN::max_scn())))) { + } else if (OB_SUCCESS != (tmp_ret = (checkpoint_executor->advance_checkpoint_by_flush(SCN::max_scn())))) { STORAGE_LOG(WARN, "advance_checkpoint_by_flush failed", K(tmp_ret), K(ls->get_ls_id())); } } diff --git a/src/storage/tx_storage/ob_checkpoint_service.h b/src/storage/tx_storage/ob_checkpoint_service.h index 3cb377bff..abe50006c 100644 --- a/src/storage/tx_storage/ob_checkpoint_service.h +++ b/src/storage/tx_storage/ob_checkpoint_service.h @@ -15,7 +15,7 @@ #include "storage/tx_storage/ob_ls_freeze_thread.h" #include "lib/lock/ob_spin_lock.h" #include "lib/task/ob_timer.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -51,7 +51,7 @@ public: // smaller than this one. int add_ls_freeze_task( ObDataCheckpoint *data_checkpoint, - palf::SCN rec_scn); + share::SCN rec_scn); int do_minor_freeze(); diff --git a/src/storage/tx_storage/ob_ls_freeze_thread.cpp b/src/storage/tx_storage/ob_ls_freeze_thread.cpp index b08d0348c..0de88df39 100644 --- a/src/storage/tx_storage/ob_ls_freeze_thread.cpp +++ b/src/storage/tx_storage/ob_ls_freeze_thread.cpp @@ -24,10 +24,11 @@ namespace storage { using namespace checkpoint; +using namespace share; void ObLSFreezeTask::set_task(ObLSFreezeThread *host, ObDataCheckpoint *data_checkpoint, - palf::SCN rec_scn) + SCN rec_scn) { host_ = host; rec_scn_ = rec_scn; @@ -109,7 +110,7 @@ int ObLSFreezeThread::init(int tg_id) } int ObLSFreezeThread::add_task(ObDataCheckpoint *data_checkpoint, - palf::SCN rec_scn) + SCN rec_scn) { int ret = OB_SUCCESS; ObLSFreezeTask *task = NULL; diff --git a/src/storage/tx_storage/ob_ls_freeze_thread.h b/src/storage/tx_storage/ob_ls_freeze_thread.h index 6297a867c..897a88664 100644 --- a/src/storage/tx_storage/ob_ls_freeze_thread.h +++ b/src/storage/tx_storage/ob_ls_freeze_thread.h @@ -15,7 +15,7 @@ #include "lib/lock/ob_spin_lock.h" #include "lib/thread/thread_mgr_interface.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -35,12 +35,12 @@ class ObLSFreezeTask public: void set_task(ObLSFreezeThread *host, checkpoint::ObDataCheckpoint *data_checkpoint, - palf::SCN rec_scn); + share::SCN rec_scn); void handle(); private: - palf::SCN rec_scn_; + share::SCN rec_scn_; ObLSFreezeThread *host_; checkpoint::ObDataCheckpoint *data_checkpoint_; }; @@ -60,7 +60,7 @@ public: int init(int tg_id); void destroy(); - int add_task(checkpoint::ObDataCheckpoint *data_checkpoint, palf::SCN rec_scn); + int add_task(checkpoint::ObDataCheckpoint *data_checkpoint, share::SCN rec_scn); void handle(void *task); int get_tg_id() { return tg_id_; } diff --git a/src/storage/tx_storage/ob_ls_service.cpp b/src/storage/tx_storage/ob_ls_service.cpp index 600d0a8cd..9f22620ec 100644 --- a/src/storage/tx_storage/ob_ls_service.cpp +++ b/src/storage/tx_storage/ob_ls_service.cpp @@ -245,7 +245,7 @@ int ObLSService::inner_create_ls_(const share::ObLSID &lsid, const ObReplicaType replica_type, const ObMigrationStatus &migration_status, const ObLSRestoreStatus &restore_status, - const palf::SCN &create_scn, + const SCN &create_scn, ObLS *&ls) { int ret = OB_SUCCESS; @@ -385,7 +385,7 @@ int ObLSService::create_ls(const obrpc::ObCreateLSArg &arg) bool need_retry = true; bool ls_exist = false; bool waiting_destroy = false; - const palf::SCN create_scn = arg.get_create_scn(); + const SCN create_scn = arg.get_create_scn(); palf::PalfBaseInfo palf_base_info; const ObMigrationStatus migration_status = ObMigrationStatus::OB_MIGRATION_STATUS_NONE; lib::ObMutexGuard change_guard(change_lock_); @@ -1202,7 +1202,7 @@ int ObLSService::create_tablet(const obrpc::ObBatchCreateTabletArg &batch_arg, } else if (OB_ISNULL(ls = handle.get_ls())) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("log stream is null, unexpected error", K(ret), K(ls_id)); - } else if (OB_FAIL(ls->batch_create_tablets(batch_arg, palf::SCN(), is_replay))) { + } else if (OB_FAIL(ls->batch_create_tablets(batch_arg, SCN(), is_replay))) { LOG_WARN("batch create tablet failed", K(ret), K(batch_arg)); } else { // do nothing diff --git a/src/storage/tx_storage/ob_ls_service.h b/src/storage/tx_storage/ob_ls_service.h index c23f2681f..f574bec46 100644 --- a/src/storage/tx_storage/ob_ls_service.h +++ b/src/storage/tx_storage/ob_ls_service.h @@ -157,7 +157,7 @@ private: const ObReplicaType replica_type, const ObMigrationStatus &migration_status, const share::ObLSRestoreStatus &restore_status, - const palf::SCN &create_scn, + const share::SCN &create_scn, ObLS *&ls); int inner_del_ls_(ObLS *&ls); int add_ls_to_map_(ObLS *ls); diff --git a/src/storage/tx_storage/ob_tablet_gc_service.cpp b/src/storage/tx_storage/ob_tablet_gc_service.cpp index 9128ad0c2..a2729bc14 100644 --- a/src/storage/tx_storage/ob_tablet_gc_service.cpp +++ b/src/storage/tx_storage/ob_tablet_gc_service.cpp @@ -26,7 +26,7 @@ namespace oceanbase { -using namespace palf; +using namespace share; namespace storage { namespace checkpoint @@ -294,7 +294,7 @@ int ObTabletGCHandler::get_unpersist_tablet_ids(common::ObTabletIDArray &unpersi } else { const ObTabletMeta &tablet_meta = tablet_handle.get_obj()->get_tablet_meta(); const ObTabletID &tablet_id = tablet_meta.tablet_id_; - palf::SCN tmp_scn; + SCN tmp_scn; if (OB_FAIL(tablet_handle.get_obj()->get_tx_data(tx_data))) { LOG_WARN("failed to get tx data", K(ret), K(tablet_id)); } else if (only_deleted) { diff --git a/src/storage/tx_storage/ob_tablet_gc_service.h b/src/storage/tx_storage/ob_tablet_gc_service.h index f7f3bddc5..49ab10b3a 100644 --- a/src/storage/tx_storage/ob_tablet_gc_service.h +++ b/src/storage/tx_storage/ob_tablet_gc_service.h @@ -16,7 +16,7 @@ #include "lib/lock/ob_spin_lock.h" #include "lib/task/ob_timer.h" #include "common/ob_tablet_id.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -52,10 +52,10 @@ public: void set_tablet_gc_trigger(); uint8_t get_tablet_persist_trigger_and_reset(); int get_unpersist_tablet_ids(common::ObTabletIDArray &unpersist_create_tablet_ids, - const palf::SCN checkpoint_scn, + const share::SCN checkpoint_scn, bool only_deleted = false); int flush_unpersist_tablet_ids(const common::ObTabletIDArray &unpersist_tablet_ids, - const palf::SCN checkpoint_scn); + const share::SCN checkpoint_scn); int gc_tablets(const common::ObTabletIDArray &tablet_ids); TO_STRING_KV(K_(tablet_persist_trigger), K_(is_inited)); @@ -64,7 +64,7 @@ private: static const int64_t FLUSH_CHECK_INTERVAL; int freeze_unpersist_tablet_ids(const common::ObTabletIDArray &unpersist_tablet_ids); int wait_unpersist_tablet_ids_flushed(const common::ObTabletIDArray &unpersist_tablet_ids, - const palf::SCN checkpoint_scn); + const share::SCN checkpoint_scn); private: storage::ObLS *ls_; diff --git a/src/storage/tx_storage/ob_tenant_freezer.cpp b/src/storage/tx_storage/ob_tenant_freezer.cpp index 8e0c1c5f0..fa214bcef 100644 --- a/src/storage/tx_storage/ob_tenant_freezer.cpp +++ b/src/storage/tx_storage/ob_tenant_freezer.cpp @@ -27,6 +27,7 @@ namespace oceanbase { +using namespace share; namespace storage { @@ -186,6 +187,24 @@ bool ObTenantFreezer::exist_ls_freezing() return exist_ls_freezing_; } +int ObTenantFreezer::ls_freeze_(ObLS *ls) +{ + int ret = OB_SUCCESS; + const int64_t SLEEP_TS = 1000 * 1000; // 1s + int64_t retry_times = 0; + // wait if there is a freeze is doing + do { + retry_times++; + if (OB_FAIL(ls->logstream_freeze()) && OB_ENTRY_EXIST == ret) { + ob_usleep(SLEEP_TS); + } + if (retry_times % 10 == 0) { + LOG_WARN("wait ls freeze finished cost too much time", K(retry_times)); + } + } while (ret == OB_ENTRY_EXIST); + return ret; +} + int ObTenantFreezer::tenant_freeze() { int ret = OB_SUCCESS; @@ -203,7 +222,9 @@ int ObTenantFreezer::tenant_freeze() ObLS *ls = nullptr; int ls_cnt = 0; for (; OB_SUCC(iter->get_next(ls)); ++ls_cnt) { - if (OB_FAIL(ls->logstream_freeze())) { + // wait until this ls freeze finished to make sure not freeze frequently because + // of this ls freeze stuck. + if (OB_FAIL(ls_freeze_(ls))) { if (OB_SUCCESS == first_fail_ret) { first_fail_ret = ret; } @@ -656,10 +677,19 @@ int ObTenantFreezer::get_tenant_memstore_cond( int64_t &total_memstore_used, int64_t &memstore_freeze_trigger, int64_t &memstore_limit, - int64_t &freeze_cnt) + int64_t &freeze_cnt, + const bool force_refresh) { int ret = OB_SUCCESS; int64_t unused = 0; + const int64_t refresh_interval = 100 * 1000; // 100 ms + int64_t current_time = OB_TSC_TIMESTAMP.current_time(); + RLOCAL_INIT(int64_t, last_refresh_timestamp, 0); + RLOCAL(int64_t, last_active_memstore_used); + RLOCAL(int64_t, last_total_memstore_used); + RLOCAL(int64_t, last_memstore_freeze_trigger); + RLOCAL(int64_t, last_memstore_limit); + RLOCAL(int64_t, last_freeze_cnt); active_memstore_used = 0; total_memstore_used = 0; @@ -669,6 +699,13 @@ int ObTenantFreezer::get_tenant_memstore_cond( if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("[TenantFreezer] tenant manager not init", KR(ret)); + } else if (!force_refresh && + current_time - last_refresh_timestamp < refresh_interval) { + active_memstore_used = last_active_memstore_used; + total_memstore_used = last_total_memstore_used; + memstore_freeze_trigger = last_memstore_freeze_trigger; + memstore_limit = last_memstore_limit; + freeze_cnt = last_freeze_cnt; } else { const uint64_t tenant_id = tenant_info_.tenant_id_; SpinRLockGuard guard(lock_); @@ -679,12 +716,19 @@ int ObTenantFreezer::get_tenant_memstore_cond( total_memstore_used, unused))) { LOG_WARN("[TenantFreezer] failed to get tenant mem usage", KR(ret), K(tenant_id)); - } else { - if (OB_FAIL(get_freeze_trigger_(memstore_freeze_trigger))) { + } else if (OB_FAIL(get_freeze_trigger_(memstore_freeze_trigger))) { LOG_WARN("[TenantFreezer] fail to get minor freeze trigger", KR(ret), K(tenant_id)); - } + } else { memstore_limit = tenant_info_.mem_memstore_limit_; freeze_cnt = tenant_info_.freeze_cnt_; + + // cache the result + last_refresh_timestamp = current_time; + last_active_memstore_used = active_memstore_used; + last_total_memstore_used = total_memstore_used; + last_memstore_freeze_trigger = memstore_freeze_trigger; + last_memstore_limit = memstore_limit; + last_freeze_cnt = freeze_cnt; } } return ret; @@ -1081,7 +1125,7 @@ int ObTenantFreezer::get_global_frozen_scn_(int64_t &frozen_scn) int ret = OB_SUCCESS; const int64_t tenant_id = tenant_info_.tenant_id_; - palf::SCN tmp_frozen_scn; + SCN tmp_frozen_scn; if (OB_FAIL(rootserver::ObMajorFreezeHelper::get_frozen_scn(tenant_id, tmp_frozen_scn))) { LOG_WARN("get_frozen_scn failed", KR(ret), K(tenant_id)); } else { diff --git a/src/storage/tx_storage/ob_tenant_freezer.h b/src/storage/tx_storage/ob_tenant_freezer.h index 474894ec4..df0067b52 100644 --- a/src/storage/tx_storage/ob_tenant_freezer.h +++ b/src/storage/tx_storage/ob_tenant_freezer.h @@ -99,7 +99,8 @@ public: int64_t &total_memstore_used, int64_t &memstore_freeze_trigger, int64_t &memstore_limit, - int64_t &freeze_cnt); + int64_t &freeze_cnt, + const bool force_refresh = true); // get the tenant memstore limit. int get_tenant_memstore_limit(int64_t &mem_limit); // this is used to check if the tenant's memstore is out. @@ -127,6 +128,7 @@ public: ObServerConfig *get_config() { return config_; } bool exist_ls_freezing(); private: + int ls_freeze_(ObLS *ls); int64_t get_freeze_trigger_percentage_() const; int post_freeze_request_(const storage::ObFreezeType freeze_type, const int64_t try_frozen_version); diff --git a/src/storage/tx_storage/ob_tenant_freezer_rpc.cpp b/src/storage/tx_storage/ob_tenant_freezer_rpc.cpp index ee96b20fb..27a341ba5 100644 --- a/src/storage/tx_storage/ob_tenant_freezer_rpc.cpp +++ b/src/storage/tx_storage/ob_tenant_freezer_rpc.cpp @@ -24,6 +24,7 @@ namespace oceanbase { using namespace storage; +using namespace share; using namespace rootserver; namespace obrpc { @@ -121,7 +122,7 @@ int ObTenantFreezerP::do_major_freeze_() int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; uint64_t tenant_id = MTL_ID(); - palf::SCN frozen_scn; + SCN frozen_scn; if (OB_FAIL(ObMajorFreezeHelper::get_frozen_scn(tenant_id, frozen_scn))) { LOG_WARN("get_frozen_scn failed", KR(ret)); diff --git a/src/storage/tx_table/ob_tx_ctx_memtable.cpp b/src/storage/tx_table/ob_tx_ctx_memtable.cpp index ca14f79c3..33f4614c8 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable.cpp +++ b/src/storage/tx_table/ob_tx_ctx_memtable.cpp @@ -221,10 +221,10 @@ transaction::ObLSTxCtxMgr *ObTxCtxMemtable::get_ls_tx_ctx_mgr() return ls_ctx_mgr_guard_.get_ls_tx_ctx_mgr(); } -palf::SCN ObTxCtxMemtable::get_rec_scn() +SCN ObTxCtxMemtable::get_rec_scn() { int ret = OB_SUCCESS; - palf::SCN rec_scn; + SCN rec_scn; if (OB_FAIL(get_ls_tx_ctx_mgr()->get_rec_scn(rec_scn))) { TRANS_LOG(WARN, "get rec scn failed", K(ret)); @@ -261,7 +261,7 @@ bool ObTxCtxMemtable::is_active_memtable() const return !ATOMIC_LOAD(&is_frozen_); } -int ObTxCtxMemtable::flush(palf::SCN recycle_scn, bool need_freeze) +int ObTxCtxMemtable::flush(SCN recycle_scn, bool need_freeze) { int ret = OB_SUCCESS; ObSpinLockGuard guard(flush_lock_); diff --git a/src/storage/tx_table/ob_tx_ctx_memtable.h b/src/storage/tx_table/ob_tx_ctx_memtable.h index 7f2d9eaf1..06935e969 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable.h +++ b/src/storage/tx_table/ob_tx_ctx_memtable.h @@ -66,8 +66,8 @@ public: virtual int64_t get_occupied_size() const override { return 0; } // ================ INHERITED FROM ObCommonCheckpoint =============== - virtual palf::SCN get_rec_scn(); - virtual int flush(palf::SCN recycle_scn, bool need_freeze = true); + virtual share::SCN get_rec_scn(); + virtual int flush(share::SCN recycle_scn, bool need_freeze = true); virtual ObTabletID get_tablet_id() const override; virtual bool is_flushing() const override; diff --git a/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp b/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp index 22196fb47..97bbd9354 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp +++ b/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp @@ -78,8 +78,8 @@ int ObTxCtxMemtableMgr::create_memtable(const SCN last_replay_scn, table_key.table_type_ = ObITable::TX_CTX_MEMTABLE; table_key.tablet_id_ = ObTabletID(ObTabletID::LS_TX_CTX_TABLET_ID); - table_key.scn_range_.start_scn_ = palf::SCN::base_scn(); - table_key.scn_range_.end_scn_ = palf::SCN::plus(table_key.scn_range_.start_scn_, 1); + table_key.scn_range_.start_scn_ = SCN::base_scn(); + table_key.scn_range_.end_scn_ = SCN::plus(table_key.scn_range_.start_scn_, 1); // TODO: Donot use pool to create the only memtable if (get_memtable_count_() > 0) { diff --git a/src/storage/tx_table/ob_tx_ctx_memtable_mgr.h b/src/storage/tx_table/ob_tx_ctx_memtable_mgr.h index 7a4166d22..ef032d91d 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable_mgr.h +++ b/src/storage/tx_table/ob_tx_ctx_memtable_mgr.h @@ -57,7 +57,7 @@ public: virtual void destroy() override; // create_memtable is used for creating the only memtable for CheckpointMgr - virtual int create_memtable(const palf::SCN last_replay_scn, + virtual int create_memtable(const share::SCN last_replay_scn, const int64_t schema_version, const bool for_replay=false) override; diff --git a/src/storage/tx_table/ob_tx_ctx_table.cpp b/src/storage/tx_table/ob_tx_ctx_table.cpp index a682b5361..e1ea2f394 100644 --- a/src/storage/tx_table/ob_tx_ctx_table.cpp +++ b/src/storage/tx_table/ob_tx_ctx_table.cpp @@ -343,7 +343,7 @@ int ObTxCtxTable::check_with_tx_data(const transaction::ObTransID tx_id, ObITxDa return ret; } -int ObTxCtxTable::get_min_start_scn(palf::SCN &min_start_scn) +int ObTxCtxTable::get_min_start_scn(SCN &min_start_scn) { int ret = get_ls_tx_ctx_mgr()->get_min_start_scn(min_start_scn); return ret; diff --git a/src/storage/tx_table/ob_tx_ctx_table.h b/src/storage/tx_table/ob_tx_ctx_table.h index 9bf987225..37b9369d4 100644 --- a/src/storage/tx_table/ob_tx_ctx_table.h +++ b/src/storage/tx_table/ob_tx_ctx_table.h @@ -113,7 +113,7 @@ public: int check_with_tx_data(const transaction::ObTransID tx_id, ObITxDataCheckFunctor &fn); - int get_min_start_scn(palf::SCN &min_start_scn); + int get_min_start_scn(share::SCN &min_start_scn); int dump_single_tx_data_2_text(const int64_t tx_id_int, FILE *fd); diff --git a/src/storage/tx_table/ob_tx_data_memtable.cpp b/src/storage/tx_table/ob_tx_data_memtable.cpp index 7ac503ba7..5dc547b61 100644 --- a/src/storage/tx_table/ob_tx_data_memtable.cpp +++ b/src/storage/tx_table/ob_tx_data_memtable.cpp @@ -25,7 +25,7 @@ namespace storage { using namespace oceanbase::transaction; -using namespace oceanbase::palf; +using namespace oceanbase::share; int ObTxDataMemtable::init(const ObITable::TableKey &table_key, SliceAllocator *slice_allocator, @@ -424,7 +424,7 @@ bool ObTxDataMemtable::ready_for_flush() set_snapshot_version(min_tx_scn_); bool_ret = true; } else { - const palf::SCN &freeze_scn = key_.scn_range_.end_scn_; + const SCN &freeze_scn = key_.scn_range_.end_scn_; STORAGE_LOG(INFO, "tx data metmable is not ready for flush", K(max_consequent_callbacked_scn), K(freeze_scn)); } diff --git a/src/storage/tx_table/ob_tx_data_memtable.h b/src/storage/tx_table/ob_tx_data_memtable.h index a8b1a44a5..538e00fbc 100644 --- a/src/storage/tx_table/ob_tx_data_memtable.h +++ b/src/storage/tx_table/ob_tx_data_memtable.h @@ -15,7 +15,7 @@ #include "storage/memtable/ob_memtable_interface.h" #include "storage/tx/ob_tx_data_define.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -239,7 +239,7 @@ public: /* derived from ObIMemtable */ const blocksstable::ObDatumRowkey &rowkey) override; public: // checkpoint - palf::SCN get_rec_scn(); + share::SCN get_rec_scn(); // int freeze(); @@ -253,7 +253,7 @@ public: // checkpoint bool ready_for_flush(); public: // getter && setter - palf::SCN get_min_start_scn() { return min_start_scn_.atomic_get(); } + share::SCN get_min_start_scn() { return min_start_scn_.atomic_get(); } int64_t get_tx_data_count() { return tx_data_map_->count(); } int64_t size() { return get_tx_data_count(); } int64_t get_inserted_count() { return inserted_cnt_; } @@ -266,17 +266,17 @@ public: // getter && setter const char* get_state_string(); ObTxDataMemtableMgr *get_tx_data_memtable_mgr() { return memtable_mgr_; } - palf::SCN get_min_tx_scn() { return min_tx_scn_; } - palf::SCN get_max_tx_scn() { return max_tx_scn_; } + share::SCN get_min_tx_scn() { return min_tx_scn_; } + share::SCN get_max_tx_scn() { return max_tx_scn_; } int set_freezer(ObFreezer *handler); - void set_start_scn(const palf::SCN start_scn) {key_.scn_range_.start_scn_ = start_scn; } + void set_start_scn(const share::SCN start_scn) {key_.scn_range_.start_scn_ = start_scn; } void set_end_scn() { key_.scn_range_.end_scn_ = max_tx_scn_; } void set_state(const ObTxDataMemtable::State &state) { state_ = state; } void set_has_constructed_list(bool val) { has_constructed_list_ = val; } void reset_is_iterating() { ATOMIC_STORE(&is_iterating_, false); } - palf::SCN get_end_scn() { return key_.scn_range_.end_scn_;} + share::SCN get_end_scn() { return key_.scn_range_.end_scn_;} private: // ObTxDataMemtable @@ -308,13 +308,13 @@ private: // ObTxDataMemtable bool has_constructed_list_; // the minimum scn of commit_version in this tx data memtable - palf::SCN min_tx_scn_; + share::SCN min_tx_scn_; // the maximum scn in this tx data memtable - palf::SCN max_tx_scn_; + share::SCN max_tx_scn_; // the minimum start scn in this tx data memtable - palf::SCN min_start_scn_; + share::SCN min_start_scn_; int64_t inserted_cnt_; @@ -407,7 +407,7 @@ private: FILE *fd_; }; -OB_INLINE palf::SCN ObTxDataMemtable::get_rec_scn() +OB_INLINE share::SCN ObTxDataMemtable::get_rec_scn() { // TODO : @gengli // rec_scn changes constantly. The rec_scn obtained by checkpoint mgr diff --git a/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp b/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp index 02801d082..7cd88e1fa 100644 --- a/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp +++ b/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp @@ -123,7 +123,7 @@ int ObTxDataMemtableMgr::release_head_memtable_(memtable::ObIMemtable *imemtable return ret; } -int ObTxDataMemtableMgr::create_memtable(const palf::SCN clog_checkpoint_scn, +int ObTxDataMemtableMgr::create_memtable(const SCN clog_checkpoint_scn, const int64_t schema_version, const bool for_replay) { @@ -150,7 +150,7 @@ int ObTxDataMemtableMgr::create_memtable(const palf::SCN clog_checkpoint_scn, return ret; } -int ObTxDataMemtableMgr::create_memtable_(const palf::SCN clog_checkpoint_scn, int64_t schema_version) +int ObTxDataMemtableMgr::create_memtable_(const SCN clog_checkpoint_scn, int64_t schema_version) { UNUSED(schema_version); int ret = OB_SUCCESS; @@ -364,10 +364,10 @@ int ObTxDataMemtableMgr::get_all_memtables_for_write(ObTxDataMemtableWriteGuard return ret; } -palf::SCN ObTxDataMemtableMgr::get_rec_scn() +SCN ObTxDataMemtableMgr::get_rec_scn() { int ret = OB_SUCCESS; - palf::SCN rec_scn; + SCN rec_scn; rec_scn.set_max(); ObTxDataMemtable *oldest_memtable = nullptr; @@ -410,7 +410,7 @@ int ObTxDataMemtableMgr::flush_all_frozen_memtables_(ObTableHdlArray &memtable_h return ret; } -int ObTxDataMemtableMgr::flush(palf::SCN recycle_scn, bool need_freeze) +int ObTxDataMemtableMgr::flush(SCN recycle_scn, bool need_freeze) { int ret = OB_SUCCESS; @@ -419,7 +419,7 @@ int ObTxDataMemtableMgr::flush(palf::SCN recycle_scn, bool need_freeze) // self freeze task if (need_freeze) { TxDataMemtableMgrFreezeGuard freeze_guard; - palf::SCN rec_scn = get_rec_scn(); + SCN rec_scn = get_rec_scn(); if (rec_scn >= recycle_scn) { STORAGE_LOG(INFO, "no need freeze", K(recycle_scn), K(rec_scn)); } else if (OB_FAIL(freeze_guard.init(this))) { diff --git a/src/storage/tx_table/ob_tx_data_memtable_mgr.h b/src/storage/tx_table/ob_tx_data_memtable_mgr.h index cdee14343..df088b90f 100644 --- a/src/storage/tx_table/ob_tx_data_memtable_mgr.h +++ b/src/storage/tx_table/ob_tx_data_memtable_mgr.h @@ -70,7 +70,7 @@ public: // ObTxDataMemtableMgr * freeze(). * @param[in] schema_version schema_version, not used */ - virtual int create_memtable(const palf::SCN clog_checkpoint_scn, + virtual int create_memtable(const share::SCN clog_checkpoint_scn, const int64_t schema_version, const bool for_replay=false) override; /** @@ -94,9 +94,9 @@ public: // ObTxDataMemtableMgr int get_memtable_range(int64_t &memtable_head, int64_t &memtable_tail); // ================ INHERITED FROM ObCommonCheckpoint =============== - virtual palf::SCN get_rec_scn() override; + virtual share::SCN get_rec_scn() override; - virtual int flush(palf::SCN recycle_scn, bool need_freeze = true) override; + virtual int flush(share::SCN recycle_scn, bool need_freeze = true) override; virtual ObTabletID get_tablet_id() const override; @@ -120,7 +120,7 @@ protected: const bool force); private: // ObTxDataMemtableMgr - int create_memtable_(const palf::SCN clog_checkpoint_scn, const int64_t schema_version); + int create_memtable_(const share::SCN clog_checkpoint_scn, const int64_t schema_version); int freeze_(); diff --git a/src/storage/tx_table/ob_tx_data_table.cpp b/src/storage/tx_table/ob_tx_data_table.cpp index c2ddd95cd..d7c29087a 100644 --- a/src/storage/tx_table/ob_tx_data_table.cpp +++ b/src/storage/tx_table/ob_tx_data_table.cpp @@ -28,7 +28,7 @@ namespace oceanbase { using namespace oceanbase::transaction; -using namespace oceanbase::palf; +using namespace oceanbase::share; namespace storage { @@ -807,7 +807,7 @@ int ObTxDataTable::self_freeze_task() STORAGE_LOG(INFO, "start tx data table self freeze task", K(get_ls_id())); - if (OB_FAIL(memtable_mgr_->flush(palf::SCN::max_scn(), true))) { + if (OB_FAIL(memtable_mgr_->flush(SCN::max_scn(), true))) { share::ObLSID ls_id = get_ls_id(); STORAGE_LOG(WARN, "self freeze of tx data memtable failed.", KR(ret), K(ls_id), KPC(memtable_mgr_)); } diff --git a/src/storage/tx_table/ob_tx_data_table.h b/src/storage/tx_table/ob_tx_data_table.h index 6833f1fb9..25fcd9bac 100644 --- a/src/storage/tx_table/ob_tx_data_table.h +++ b/src/storage/tx_table/ob_tx_data_table.h @@ -184,12 +184,12 @@ public: // ObTxDataTable /** * @brief See ObTxTable::get_recycle_scn */ - int get_recycle_scn(palf::SCN &recycle_scn); + int get_recycle_scn(share::SCN &recycle_scn); /** * @brief see ObTxTable::get_upper_trans_version_before_given_scn() */ - int get_upper_trans_version_before_given_scn(const palf::SCN sstable_end_scn, palf::SCN &upper_trans_version); + int get_upper_trans_version_before_given_scn(const share::SCN sstable_end_scn, share::SCN &upper_trans_version); /** * @brief see ObTxTable::supplement_undo_actions_if_exist @@ -206,7 +206,7 @@ public: // ObTxDataTable /** * @brief see ObTxTable::get_start_tx_scn */ - int get_start_tx_scn(palf::SCN &start_tx_scn); + int get_start_tx_scn(share::SCN &start_tx_scn); void reuse_memtable_handles_cache(); @@ -232,7 +232,7 @@ public: // getter and setter private: virtual ObTxDataMemtableMgr *get_memtable_mgr_() { return memtable_mgr_; } - int get_ls_min_end_scn_in_latest_tablets_(palf::SCN &min_end_ts); + int get_ls_min_end_scn_in_latest_tablets_(share::SCN &min_end_ts); int init_sstable_cache_(); @@ -251,7 +251,7 @@ private: int insert_into_memtable_(ObTxDataMemtable *tx_data_memtable, ObTxData *&tx_data); // free the whole undo status list allocated by slice allocator - int get_min_end_scn_from_single_tablet_(ObTabletHandle &tablet_handle, palf::SCN &end_scn); + int get_min_end_scn_from_single_tablet_(ObTabletHandle &tablet_handle, share::SCN &end_scn); int deep_copy_undo_status_list_(ObUndoStatusList &in_list, ObUndoStatusList &out_list); int init_tx_data_read_schema_(); @@ -260,7 +260,7 @@ private: int update_calc_upper_trans_version_cache_(ObITable *table); - int calc_upper_trans_scn_(const palf::SCN sstable_end_scn, palf::SCN &upper_trans_version); + int calc_upper_trans_scn_(const share::SCN sstable_end_scn, share::SCN &upper_trans_version); int update_freeze_trigger_threshold_(); @@ -276,7 +276,7 @@ private: int dump_tx_data_in_memtable_2_text_(const transaction::ObTransID tx_id, FILE *fd); int dump_tx_data_in_sstable_2_text_(const transaction::ObTransID tx_id, FILE *fd); - bool skip_this_sstable_end_scn_(palf::SCN sstable_end_scn); + bool skip_this_sstable_end_scn_(share::SCN sstable_end_scn); void print_alloc_size_for_test_(); @@ -308,7 +308,7 @@ private: static const int64_t LS_TX_DATA_SCHEMA_COLUMN_CNT = 5; bool is_inited_; bool is_started_; - palf::SCN min_start_scn_in_ctx_; + share::SCN min_start_scn_in_ctx_; int64_t last_update_ts_; ObTabletID tablet_id_; ObMemAttr mem_attr_; diff --git a/src/storage/tx_table/ob_tx_table.cpp b/src/storage/tx_table/ob_tx_table.cpp index 286aed215..2f7582a0d 100644 --- a/src/storage/tx_table/ob_tx_table.cpp +++ b/src/storage/tx_table/ob_tx_table.cpp @@ -217,7 +217,7 @@ int ObTxTable::prepare_for_safe_destroy() return ret; } -int ObTxTable::create_tablet(const lib::Worker::CompatMode compat_mode, const palf::SCN &create_scn) +int ObTxTable::create_tablet(const lib::Worker::CompatMode compat_mode, const SCN &create_scn) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -341,7 +341,7 @@ int ObTxTable::gen_create_tablet_arg_( false/*is_create_bind_hidden_tablets*/))) { LOG_WARN("create tablet info init failed", K(ret), K(tablet_ids), K(tablet_id)); // create ObBatchCreateTabletArg - } else if (OB_FAIL(arg.init_create_tablet(ls_id, palf::SCN::base_scn()))) { + } else if (OB_FAIL(arg.init_create_tablet(ls_id, SCN::base_scn()))) { LOG_WARN("ObBatchCreateTabletArg init create tablet failed", K(ret), K(tenant_id), K(ls_id)); } else if (OB_FAIL(arg.table_schemas_.push_back(table_schema))) { LOG_WARN("add table schema failed", K(ret), K(table_schema)); @@ -372,13 +372,13 @@ int ObTxTable::create_ctx_tablet_( const uint64_t tenant_id, const ObLSID ls_id, const lib::Worker::CompatMode compat_mode, - const palf::SCN &create_scn) + const SCN &create_scn) { int ret = OB_SUCCESS; obrpc::ObBatchCreateTabletArg arg; const bool no_need_write_clog = true; share::schema::ObTableSchema table_schema; - palf::SCN tmp_scn; + SCN tmp_scn; if (OB_FAIL(get_ctx_table_schema_(tenant_id, table_schema))) { @@ -503,7 +503,7 @@ int ObTxTable::create_data_tablet_( const uint64_t tenant_id, const ObLSID ls_id, const lib::Worker::CompatMode compat_mode, - const palf::SCN &create_scn) + const SCN &create_scn) { int ret = OB_SUCCESS; obrpc::ObBatchCreateTabletArg arg; @@ -999,7 +999,7 @@ int ObTxTable::get_recycle_scn(SCN &recycle_scn) return ret; } -int ObTxTable::get_upper_trans_version_before_given_scn(const palf::SCN sstable_end_scn, palf::SCN &upper_trans_version) +int ObTxTable::get_upper_trans_version_before_given_scn(const SCN sstable_end_scn, SCN &upper_trans_version) { return tx_data_table_.get_upper_trans_version_before_given_scn(sstable_end_scn, upper_trans_version); } diff --git a/src/storage/tx_table/ob_tx_table.h b/src/storage/tx_table/ob_tx_table.h index 616264b1f..2dcfb2611 100644 --- a/src/storage/tx_table/ob_tx_table.h +++ b/src/storage/tx_table/ob_tx_table.h @@ -74,7 +74,7 @@ public: int start(); void stop(); void destroy(); - int create_tablet(const lib::Worker::CompatMode compat_mode, const palf::SCN &create_scn); + int create_tablet(const lib::Worker::CompatMode compat_mode, const share::SCN &create_scn); int remove_tablet(); int load_tx_table(); int prepare_offline(); @@ -151,10 +151,10 @@ public: * @param[out] trans_version */ int get_tx_state_with_scn(const transaction::ObTransID &data_trans_id, - const palf::SCN scn, + const share::SCN scn, const int64_t read_epoch, int64_t &state, - palf::SCN &trans_version); + share::SCN &trans_version); /** * @brief Try to get a tx data from tx_data_table. This function used in special situation when the trans service do @@ -168,7 +168,7 @@ public: int try_get_tx_state(const transaction::ObTransID tx_id, const int64_t read_epoch, int64_t &state, - palf::SCN &trans_version); + share::SCN &trans_version); /** * @brief the txn READ_TRANS_ID use SNAPSHOT_VERSION to read the data, and check whether the data is locked, readable or unreadable by txn DATA_TRANS_ID. READ_LATEST is used to check whether read the data belong to the same txn @@ -183,7 +183,7 @@ public: int lock_for_read(const transaction::ObLockForReadArg &lock_for_read_arg, const int64_t read_epoch, bool &can_read, - palf::SCN &trans_version, + share::SCN &trans_version, bool &is_determined_state, const ObCleanoutOp &cleanout_op = ObCleanoutNothingOperation(), const ObReCheckOp &recheck_op = ObReCheckNothingOperation()); @@ -212,7 +212,7 @@ public: * @param[out] recycle_scn the tx data whose end_scn is smaller or equals to the recycle_scn can * be cleared. */ - int get_recycle_scn(palf::SCN &recycle_scn); + int get_recycle_scn(share::SCN &recycle_scn); /** * @brief Get the upper trans version for each given end_scn @@ -220,7 +220,7 @@ public: * @param[in] sstable_end_scn the end_scn of the data sstable which is waitting to get the upper_trans_version * @param[out] upper_trans_version the upper_trans_version */ - int get_upper_trans_version_before_given_scn(const palf::SCN sstable_end_scn, palf::SCN &upper_trans_version); + int get_upper_trans_version_before_given_scn(const share::SCN sstable_end_scn, share::SCN &upper_trans_version); /** * @brief When a transaction is replayed in the middle, it will read tx data from tx data sstable @@ -241,7 +241,7 @@ public: * * @param[out] start_tx_scn */ - int get_start_tx_scn(palf::SCN &start_tx_scn); + int get_start_tx_scn(share::SCN &start_tx_scn); int dump_single_tx_data_2_text(const int64_t tx_id_int, const char *fname); @@ -263,12 +263,12 @@ private: const uint64_t tenant_id, const share::ObLSID ls_id, const lib::Worker::CompatMode compat_mode, - const palf::SCN &create_scn); + const share::SCN &create_scn); int create_ctx_tablet_( const uint64_t tenant_id, const share::ObLSID ls_id, const lib::Worker::CompatMode compat_mode, - const palf::SCN &create_scn); + const share::SCN &create_scn); int remove_tablet_(const common::ObTabletID &tablet_id); int remove_data_tablet_(); int remove_ctx_tablet_(); @@ -295,7 +295,7 @@ private: int load_tx_data_table_(); int offline_tx_ctx_table_(); int offline_tx_data_table_(); - int get_max_tablet_clog_checkpoint_(palf::SCN &max_tablet_clog_checkpoint); + int get_max_tablet_clog_checkpoint_(share::SCN &max_tablet_clog_checkpoint); void check_state_and_epoch_(const transaction::ObTransID tx_id, const int64_t read_epoch, @@ -308,7 +308,7 @@ private: static const int64_t LS_TX_CTX_SCHEMA_COLUMN_CNT = 3; bool is_inited_; int64_t epoch_ CACHE_ALIGNED; - palf::SCN max_tablet_clog_checkpoint_; + share::SCN max_tablet_clog_checkpoint_; int64_t prepare_online_ts_; TxTableState state_ CACHE_ALIGNED; ObLS *ls_; diff --git a/src/storage/tx_table/ob_tx_table_define.h b/src/storage/tx_table/ob_tx_table_define.h index b2ed98891..3c2396b1f 100644 --- a/src/storage/tx_table/ob_tx_table_define.h +++ b/src/storage/tx_table/ob_tx_table_define.h @@ -197,12 +197,12 @@ private: public: struct Node { - palf::SCN start_scn_; - palf::SCN commit_version_; + share::SCN start_scn_; + share::SCN commit_version_; Node() : start_scn_(), commit_version_() {} - Node(const palf::SCN start_scn, const palf::SCN commit_version) + Node(const share::SCN start_scn, const share::SCN commit_version) : start_scn_(start_scn), commit_version_(commit_version) {} bool operator==(const Node &rhs) const @@ -279,7 +279,7 @@ public: bool is_inited_; // The end_scn of the sstable will be used as the cache_version - palf::SCN cache_version_; + share::SCN cache_version_; mutable common::TCRWLock lock_; diff --git a/src/storage/tx_table/ob_tx_table_iterator.cpp b/src/storage/tx_table/ob_tx_table_iterator.cpp index be5f3f348..bca35ad7d 100644 --- a/src/storage/tx_table/ob_tx_table_iterator.cpp +++ b/src/storage/tx_table/ob_tx_table_iterator.cpp @@ -20,7 +20,7 @@ #include "storage/tx_table/ob_tx_table.h" namespace oceanbase { -using namespace palf; +using namespace share; namespace storage { @@ -562,7 +562,7 @@ int ObTxDataMemtableScanIterator::get_past_commit_scns_( return ret; } -int ObTxDataMemtableScanIterator::merge_cur_and_past_commit_verisons_(const palf::SCN recycle_scn, +int ObTxDataMemtableScanIterator::merge_cur_and_past_commit_verisons_(const SCN recycle_scn, ObCommitSCNsArray &cur_commit_scns, ObCommitSCNsArray &past_commit_scns, ObCommitSCNsArray &merged_commit_scns) @@ -590,8 +590,8 @@ int ObTxDataMemtableScanIterator::merge_cur_and_past_commit_verisons_(const palf // here we merge the past commit versions and current commit versions. To keep merged array correct, the node in past // array whose start_scn is larger than the minimum start_scn in current array will be dropped. The reason is in this // issue: https://work.aone.alibaba-inc.com/issue/43389863 - palf::SCN cur_min_start_scn = cur_arr.count() > 0 ? cur_arr.at(0).start_scn_ : palf::SCN::max_scn(); - palf::SCN max_commit_version = palf::SCN::min_scn(); + SCN cur_min_start_scn = cur_arr.count() > 0 ? cur_arr.at(0).start_scn_ : SCN::max_scn(); + SCN max_commit_version = SCN::min_scn(); if (OB_FAIL( merge_pre_process_node_(step_len, cur_min_start_scn, recycle_scn, past_arr, max_commit_version, merged_arr))) { STORAGE_LOG(WARN, "merge past commit versions failed.", KR(ret), K(past_arr), KPC(tx_data_memtable_)); @@ -618,10 +618,10 @@ int ObTxDataMemtableScanIterator::merge_cur_and_past_commit_verisons_(const palf } int ObTxDataMemtableScanIterator::merge_pre_process_node_(const int64_t step_len, - const palf::SCN start_scn_limit, - const palf::SCN recycle_scn, + const SCN start_scn_limit, + const SCN recycle_scn, const ObIArray &data_arr, - palf::SCN &max_commit_version, + SCN &max_commit_version, ObIArray &merged_arr) { int ret = OB_SUCCESS; diff --git a/src/storage/tx_table/ob_tx_table_iterator.h b/src/storage/tx_table/ob_tx_table_iterator.h index 58e15f2a1..e81a269fb 100644 --- a/src/storage/tx_table/ob_tx_table_iterator.h +++ b/src/storage/tx_table/ob_tx_table_iterator.h @@ -138,7 +138,7 @@ private: int deserialize_commit_scns_array_from_row_(const blocksstable::ObDatumRow *row, ObCommitSCNsArray &past_commit_scns); - int merge_cur_and_past_commit_verisons_(const palf::SCN recycle_scn, + int merge_cur_and_past_commit_verisons_(const share::SCN recycle_scn, ObCommitSCNsArray &cur_commit_scns, ObCommitSCNsArray &past_commit_scns, ObCommitSCNsArray &merged_commit_scns); @@ -154,10 +154,10 @@ private: * @param[out] merged_arr the target array to be dumped */ int merge_pre_process_node_(const int64_t step_len, - const palf::SCN start_scn_limit, - const palf::SCN recycle_scn, + const share::SCN start_scn_limit, + const share::SCN recycle_scn, const ObIArray &data_arr, - palf::SCN &max_commit_version, + share::SCN &max_commit_version, ObIArray &merged_arr); int set_row_with_merged_commit_scns_(ObCommitSCNsArray &merged_commit_scns, @@ -168,8 +168,8 @@ private: int DEBUG_try_calc_upper_and_check_(ObCommitSCNsArray &merged_commit_versions); - int DEBUG_fake_calc_upper_trans_version(const palf::SCN sstable_end_scn, - palf::SCN &upper_trans_version, + int DEBUG_fake_calc_upper_trans_version(const share::SCN sstable_end_scn, + share::SCN &upper_trans_version, ObCommitSCNsArray &merged_commit_versions); void DEBUG_print_start_scn_list_(); @@ -179,8 +179,8 @@ private: bool is_inited_; const ObTableIterParam &iter_param_; bool dump_tx_data_done_; - palf::SCN cur_max_commit_version_; - palf::SCN pre_start_scn_; + share::SCN cur_max_commit_version_; + share::SCN pre_start_scn_; int64_t tx_data_row_cnt_; ObTxData *pre_tx_data_; ObArenaAllocator arena_allocator_; @@ -190,7 +190,7 @@ private: ObTxDataMemtable *tx_data_memtable_; blocksstable::ObStorageDatum key_datum_; int64_t DEBUG_iter_commit_ts_cnt_; - palf::SCN DEBUG_last_start_scn_; + share::SCN DEBUG_last_start_scn_; }; /** diff --git a/tools/deploy/mysql_test/test_suite/otimestamp/t/otimestamp_import_time_zone_mysql.inc b/tools/deploy/mysql_test/test_suite/otimestamp/t/otimestamp_import_time_zone_mysql.inc index c52fe847c..41bf0dc37 100644 --- a/tools/deploy/mysql_test/test_suite/otimestamp/t/otimestamp_import_time_zone_mysql.inc +++ b/tools/deploy/mysql_test/test_suite/otimestamp/t/otimestamp_import_time_zone_mysql.inc @@ -8,6 +8,7 @@ connect (conn_admin_tz, $OBMYSQL_MS0,admin,$OBMYSQL_PWD,test,$OBMYSQL_PORT); connection conn_admin_tz; +set session ob_query_timeout = 20000000; select count(*) c1 from oceanbase.__all_tenant_time_zone; select count(*) c1 from oceanbase.__all_tenant_time_zone_name; select count(*) c1 from oceanbase.__all_tenant_time_zone_transition; diff --git a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp index 6d7b22c42..00cf8c059 100644 --- a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp +++ b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp @@ -2062,8 +2062,8 @@ int ObAdminDumpBackupDataExecutor::dump_backup_tablet_meta_(const backup::ObBack PrintHelper::print_dump_line("tablet_meta:tablet_id", tablet_meta.tablet_meta_.tablet_id_.id()); PrintHelper::print_dump_line("tablet_meta:data_tablet_id", tablet_meta.tablet_meta_.data_tablet_id_.id()); PrintHelper::print_dump_line("tablet_meta:ref_tablet_id", tablet_meta.tablet_meta_.ref_tablet_id_.id()); - PrintHelper::print_dump_line("tablet_meta:clog_checkpoint_scn", tablet_meta.tablet_meta_.clog_checkpoint_scn_.get_val_for_lsn_allocator()); - PrintHelper::print_dump_line("tablet_meta:ddl_checkpoint_scn", tablet_meta.tablet_meta_.ddl_checkpoint_scn_.get_val_for_lsn_allocator()); + PrintHelper::print_dump_line("tablet_meta:clog_checkpoint_scn", tablet_meta.tablet_meta_.clog_checkpoint_scn_.get_val_for_logservice()); + PrintHelper::print_dump_line("tablet_meta:ddl_checkpoint_scn", tablet_meta.tablet_meta_.ddl_checkpoint_scn_.get_val_for_logservice()); PrintHelper::print_dump_line("tablet_meta:tablet_status", tablet_meta.tablet_meta_.tx_data_.tablet_status_); PrintHelper::print_end_line(); return ret; @@ -2239,8 +2239,8 @@ int ObAdminDumpBackupDataExecutor::dump_backup_set_info(const share::ObBackupSet PrintHelper::print_dump_line("passwd", backup_set_info.passwd_.ptr()); PrintHelper::print_dump_line("file_status", backup_set_info.file_status_); PrintHelper::print_dump_line("backup_path", backup_set_info.backup_path_.ptr()); - PrintHelper::print_dump_line("start_replay_scn", backup_set_info.start_replay_scn_.get_val_for_lsn_allocator()); - PrintHelper::print_dump_line("min_restore_scn", backup_set_info.min_restore_scn_.get_val_for_lsn_allocator()); + PrintHelper::print_dump_line("start_replay_scn", backup_set_info.start_replay_scn_.get_val_for_logservice()); + PrintHelper::print_dump_line("min_restore_scn", backup_set_info.min_restore_scn_.get_val_for_logservice()); PrintHelper::print_dump_line("tenant_compatible", backup_set_info.tenant_compatible_); PrintHelper::print_dump_line("backup_compatible", backup_set_info.backup_compatible_); PrintHelper::print_dump_line("meta_turn_id", backup_set_info.meta_turn_id_); @@ -2255,7 +2255,7 @@ int ObAdminDumpBackupDataExecutor::dump_archive_round_start_file_(const share::O PrintHelper::print_dump_title("archive round start info"); PrintHelper::print_dump_line("dest_id", round_start_file.dest_id_); PrintHelper::print_dump_line("round_id", round_start_file.round_id_); - PrintHelper::print_dump_line("start_scn", round_start_file.start_scn_.get_val_for_lsn_allocator()); + PrintHelper::print_dump_line("start_scn", round_start_file.start_scn_.get_val_for_logservice()); PrintHelper::print_dump_line("base_piece_id", round_start_file.base_piece_id_); PrintHelper::print_dump_line("piece_switch_interval", round_start_file.piece_switch_interval_); PrintHelper::print_end_line(); @@ -2268,8 +2268,8 @@ int ObAdminDumpBackupDataExecutor::dump_archive_round_end_file_(const share::ObR PrintHelper::print_dump_title("archive round end info"); PrintHelper::print_dump_line("dest_id", round_end_file.dest_id_); PrintHelper::print_dump_line("round_id", round_end_file.round_id_); - PrintHelper::print_dump_line("start_scn", round_end_file.start_scn_.get_val_for_lsn_allocator()); - PrintHelper::print_dump_line("checkpoint_scn", round_end_file.checkpoint_scn_.get_val_for_lsn_allocator()); + PrintHelper::print_dump_line("start_scn", round_end_file.start_scn_.get_val_for_logservice()); + PrintHelper::print_dump_line("checkpoint_scn", round_end_file.checkpoint_scn_.get_val_for_logservice()); PrintHelper::print_dump_line("base_piece_id", round_end_file.base_piece_id_); PrintHelper::print_dump_line("piece_switch_interval", round_end_file.piece_switch_interval_); PrintHelper::print_end_line(); @@ -2283,7 +2283,7 @@ int ObAdminDumpBackupDataExecutor::dump_archive_piece_start_file_(const share::O PrintHelper::print_dump_line("dest_id", piece_start_file.dest_id_); PrintHelper::print_dump_line("round_id", piece_start_file.round_id_); PrintHelper::print_dump_line("piece_id", piece_start_file.piece_id_); - PrintHelper::print_dump_line("start_scn", piece_start_file.start_scn_.get_val_for_lsn_allocator()); + PrintHelper::print_dump_line("start_scn", piece_start_file.start_scn_.get_val_for_logservice()); PrintHelper::print_end_line(); return ret; } @@ -2295,7 +2295,7 @@ int ObAdminDumpBackupDataExecutor::dump_archive_piece_end_file_(const share::ObP PrintHelper::print_dump_line("dest_id", piece_end_file.dest_id_); PrintHelper::print_dump_line("round_id", piece_end_file.round_id_); PrintHelper::print_dump_line("piece_id", piece_end_file.piece_id_); - PrintHelper::print_dump_line("end_scn", piece_end_file.end_scn_.get_val_for_lsn_allocator()); + PrintHelper::print_dump_line("end_scn", piece_end_file.end_scn_.get_val_for_logservice()); PrintHelper::print_end_line(); return ret; } @@ -2310,8 +2310,8 @@ int ObAdminDumpBackupDataExecutor::dump_tenant_archive_piece_infos_file_(const s PrintHelper::print_dump_line("incarnation", piece_infos_file.incarnation_); PrintHelper::print_dump_line("dest_no", piece_infos_file.dest_no_); PrintHelper::print_dump_line("compatible", static_cast(piece_infos_file.compatible_.version_)); - PrintHelper::print_dump_line("start_scn", piece_infos_file.start_scn_.get_val_for_lsn_allocator()); - PrintHelper::print_dump_line("end_scn", piece_infos_file.end_scn_.get_val_for_lsn_allocator()); + PrintHelper::print_dump_line("start_scn", piece_infos_file.start_scn_.get_val_for_logservice()); + PrintHelper::print_dump_line("end_scn", piece_infos_file.end_scn_.get_val_for_logservice()); PrintHelper::print_dump_line("path", piece_infos_file.path_.ptr()); ARRAY_FOREACH_X(piece_infos_file.his_frozen_pieces_, i , cnt, OB_SUCC(ret)) { const ObTenantArchivePieceAttr &piece = piece_infos_file.his_frozen_pieces_.at(i); @@ -2338,10 +2338,10 @@ int ObAdminDumpBackupDataExecutor::dump_one_piece_(const share::ObTenantArchiveP PrintHelper::print_dump_line("incarnation", piece.incarnation_); PrintHelper::print_dump_line("dest_no", piece.dest_no_); PrintHelper::print_dump_line("file_count", piece.file_count_); - PrintHelper::print_dump_line("start_scn", piece.start_scn_.get_val_for_lsn_allocator()); - PrintHelper::print_dump_line("checkpoint_scn", piece.checkpoint_scn_.get_val_for_lsn_allocator()); - PrintHelper::print_dump_line("max_scn", piece.max_scn_.get_val_for_lsn_allocator()); - PrintHelper::print_dump_line("end_scn", piece.end_scn_.get_val_for_lsn_allocator()); + PrintHelper::print_dump_line("start_scn", piece.start_scn_.get_val_for_logservice()); + PrintHelper::print_dump_line("checkpoint_scn", piece.checkpoint_scn_.get_val_for_logservice()); + PrintHelper::print_dump_line("max_scn", piece.max_scn_.get_val_for_logservice()); + PrintHelper::print_dump_line("end_scn", piece.end_scn_.get_val_for_logservice()); PrintHelper::print_dump_line("compatible", static_cast(piece.compatible_.version_)); PrintHelper::print_dump_line("input_bytes", piece.input_bytes_); PrintHelper::print_dump_line("output_bytes", piece.output_bytes_); @@ -2369,8 +2369,8 @@ int ObAdminDumpBackupDataExecutor::dump_archive_piece_inner_placeholder_file_(co PrintHelper::print_dump_line("dest_id", piece_inner_placeholder.dest_id_); PrintHelper::print_dump_line("round_id", piece_inner_placeholder.round_id_); PrintHelper::print_dump_line("piece_id", piece_inner_placeholder.piece_id_); - PrintHelper::print_dump_line("start_scn", piece_inner_placeholder.start_scn_.get_val_for_lsn_allocator()); - PrintHelper::print_dump_line("checkpoint_scn", piece_inner_placeholder.checkpoint_scn_.get_val_for_lsn_allocator()); + PrintHelper::print_dump_line("start_scn", piece_inner_placeholder.start_scn_.get_val_for_logservice()); + PrintHelper::print_dump_line("checkpoint_scn", piece_inner_placeholder.checkpoint_scn_.get_val_for_logservice()); PrintHelper::print_end_line(); return ret; } @@ -2383,7 +2383,7 @@ int ObAdminDumpBackupDataExecutor::dump_archive_single_ls_info_file_(const share PrintHelper::print_dump_line("round_id", single_ls_info_file.round_id_); PrintHelper::print_dump_line("piece_id", single_ls_info_file.piece_id_); PrintHelper::print_dump_line("ls_id", single_ls_info_file.ls_id_.id()); - PrintHelper::print_dump_line("checkpoint_scn", single_ls_info_file.checkpoint_scn_.get_val_for_lsn_allocator()); + PrintHelper::print_dump_line("checkpoint_scn", single_ls_info_file.checkpoint_scn_.get_val_for_logservice()); PrintHelper::print_dump_line("max_lsn", single_ls_info_file.max_lsn_); ARRAY_FOREACH_X(single_ls_info_file.filelist_, i , cnt, OB_SUCC(ret)) { const ObSingleLSInfoDesc::OneFile &one_file = single_ls_info_file.filelist_.at(i); diff --git a/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.cpp b/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.cpp index 7c6d577fb..4ae14cef8 100644 --- a/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.cpp +++ b/tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.cpp @@ -24,7 +24,7 @@ #include "storage/tx/ob_keep_alive_ls_handler.h" #include "logservice/ob_log_base_header.h" #include "logservice/ob_garbage_collector.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include @@ -44,7 +44,7 @@ ObAdminParserLogEntry::ObAdminParserLogEntry(const LogEntry &entry, const LSN lsn, const ObAdminMutatorStringArg &str_arg) : buf_(entry.get_data_buf()), buf_len_(entry.get_data_len()), pos_(0), - scn_val_(entry.get_scn().get_val_for_lsn_allocator()), block_id_(block_id), lsn_(lsn) + scn_val_(entry.get_scn().get_val_for_logservice()), block_id_(block_id), lsn_(lsn) { str_arg_ = str_arg; } @@ -584,11 +584,11 @@ int ObAdminParserLogEntry::parse_trans_redo_log_(ObTxLogBlock &tx_log_block, ObTxRedoLogTempRef temp_ref; ObTxRedoLog redolog(temp_ref); memtable::ObMemtableMutatorIterator mmi; - palf::SCN scn; + share::SCN scn; str_arg_.log_stat_->total_tx_redo_log_count_++; if (OB_FAIL(tx_log_block.deserialize_log_body(redolog))) { LOG_WARN("tx_log_block.deserialize_log_body failed", K(ret), K(redolog)); - } else if (OB_FAIL(scn.convert_for_lsn_allocator(scn_val_))) { + } else if (OB_FAIL(scn.convert_for_logservice(scn_val_))) { LOG_WARN("failed to convert", K(ret), K(scn_val_)); } else if (OB_FAIL(redolog.ob_admin_dump(&mmi, str_arg_, block_id_, lsn_, tx_id, scn, has_dumped_tx_id))) { LOG_WARN("get mutator json string failed", K(block_id_), K(lsn_), K(tx_id), K(ret)); diff --git a/tools/upgrade/special_upgrade_action_post.py b/tools/upgrade/special_upgrade_action_post.py index 55693563b..bbc66492d 100755 --- a/tools/upgrade/special_upgrade_action_post.py +++ b/tools/upgrade/special_upgrade_action_post.py @@ -194,7 +194,7 @@ def get_max_used_job_id(cur): def check_can_run_upgrade_job(cur, version): try: sql = """select job_status from oceanbase.__all_rootservice_job - where job_type = 'RUN_UPGRADE_POST_JOB' and extra_info = '{0}' + where job_type = 'UPGRADE_POST_ACTION' and extra_info = '{0}' order by job_id desc limit 1""".format(version) results = query(cur, sql) diff --git a/tools/upgrade/upgrade_post.py b/tools/upgrade/upgrade_post.py index 3cacfc691..9cd75c5ef 100755 --- a/tools/upgrade/upgrade_post.py +++ b/tools/upgrade/upgrade_post.py @@ -12717,7 +12717,7 @@ #def check_can_run_upgrade_job(cur, version): # try: # sql = """select job_status from oceanbase.__all_rootservice_job -# where job_type = 'RUN_UPGRADE_POST_JOB' and extra_info = '{0}' +# where job_type = 'UPGRADE_POST_ACTION' and extra_info = '{0}' # order by job_id desc limit 1""".format(version) # results = query(cur, sql) # diff --git a/tools/upgrade/upgrade_pre.py b/tools/upgrade/upgrade_pre.py index 4a4ee620a..db336e0f7 100755 --- a/tools/upgrade/upgrade_pre.py +++ b/tools/upgrade/upgrade_pre.py @@ -12717,7 +12717,7 @@ #def check_can_run_upgrade_job(cur, version): # try: # sql = """select job_status from oceanbase.__all_rootservice_job -# where job_type = 'RUN_UPGRADE_POST_JOB' and extra_info = '{0}' +# where job_type = 'UPGRADE_POST_ACTION' and extra_info = '{0}' # order by job_id desc limit 1""".format(version) # results = query(cur, sql) # diff --git a/unittest/libobcdc/log_generator.h b/unittest/libobcdc/log_generator.h index f7bb2b634..80239d9eb 100644 --- a/unittest/libobcdc/log_generator.h +++ b/unittest/libobcdc/log_generator.h @@ -100,9 +100,9 @@ public: const bool is_padding_log = (palf::LOG_PADDING == log_type); int64_t log_entry_header_size = entry_header.get_serialize_size(); int64_t ts = get_timestamp(); - SCN scn; + share::SCN scn; - if (OB_FAIL(scn.convert_for_lsn_allocator(ts))) { + if (OB_FAIL(scn.convert_for_logservice(ts))) { LOG_ERROR("fail to convert ts", KR(ret), K(ts)); } else if (OB_FAIL(entry_header.generate_header(buf, buf_len, scn))) { LOG_ERROR("generate_header failed", KR(ret), K(buf), K(buf_len), K(scn)); @@ -136,9 +136,9 @@ public: logservice::ObLogBaseHeader log_base_header(logservice::ObLogBaseType::GC_LS_LOG_BASE_TYPE, logservice::ObReplayBarrierType::NO_NEED_BARRIER, 1); const int64_t serizlize_size = log_base_header.get_serialize_size(); char *buf = static_cast(allocator_.alloc(serizlize_size)); - SCN scn; + share::SCN scn; - if (OB_FAIL(scn.convert_for_lsn_allocator(ts))) { + if (OB_FAIL(scn.convert_for_logservice(ts))) { LOG_ERROR("fail to convert ts", KR(ret), K(ts)); } else if (OB_FAIL(log_base_header.serialize(buf, serizlize_size, pos))) { LOG_ERROR("serialize log_base_header failed", KR(ret), K(buf), K(serizlize_size), K(pos)); @@ -287,8 +287,8 @@ void ObTxLogGenerator::gen_prepare_log() void ObTxLogGenerator::gen_commit_log() { int64_t commit_ts = get_timestamp(); - palf::SCN commit_version; - commit_version.convert_for_lsn_allocator(commit_ts); + share::SCN commit_version; + commit_version.convert_for_logservice(commit_ts); uint64_t checksum = 0; share::ObLSArray inc_ls_arr; ObTxBufferNodeArray mds_arr; diff --git a/unittest/logservice/mock_logservice_container/mock_log_engine.h b/unittest/logservice/mock_logservice_container/mock_log_engine.h index 9cfec36ad..9ee936312 100644 --- a/unittest/logservice/mock_logservice_container/mock_log_engine.h +++ b/unittest/logservice/mock_logservice_container/mock_log_engine.h @@ -538,7 +538,7 @@ public: { return base_lsn_for_block_gc_; } - int get_min_block_id_and_min_ts_ns(block_id_t &block_id, int64_t &ts_ns) + int get_min_block_info_for_gc(block_id_t &block_id, int64_t &ts_ns) { int ret = OB_SUCCESS; UNUSED(block_id); diff --git a/unittest/logservice/mock_logservice_container/mock_log_sliding_window.h b/unittest/logservice/mock_logservice_container/mock_log_sliding_window.h index 91855549a..c0f0acb77 100644 --- a/unittest/logservice/mock_logservice_container/mock_log_sliding_window.h +++ b/unittest/logservice/mock_logservice_container/mock_log_sliding_window.h @@ -15,7 +15,7 @@ #define private public #include "logservice/palf/log_sliding_window.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #undef private namespace oceanbase @@ -307,7 +307,7 @@ public: int ret = OB_SUCCESS; return ret; } - int inc_update_scn_base(const palf::SCN &scn) + int inc_update_scn_base(const share::SCN &scn) { int ret = OB_SUCCESS; return ret; diff --git a/unittest/logservice/palf_performance_unittest.cpp b/unittest/logservice/palf_performance_unittest.cpp index 282eb129c..da9458e7a 100644 --- a/unittest/logservice/palf_performance_unittest.cpp +++ b/unittest/logservice/palf_performance_unittest.cpp @@ -21,7 +21,7 @@ #include "logservice/palf/log_engine.h" #undef private #include "logservice/palf/log_group_entry.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "logservice/palf/log_group_entry_header.h" #include "logservice/palf/log_io_worker.h" #include "logservice/palf/log_iterator_storage.h" @@ -187,7 +187,7 @@ public: } int test_log_read_block_scn(const block_id_t block_id) { - palf::SCN min_scn; + share::SCN min_scn; return log_engine_.get_block_min_scn(block_id, min_scn); } int test_log_engine_append_log(const std::function &write_size, const int64_t total_size) diff --git a/unittest/logservice/test_log_entry_and_group_entry.cpp b/unittest/logservice/test_log_entry_and_group_entry.cpp index 2e08af683..ad18eb217 100644 --- a/unittest/logservice/test_log_entry_and_group_entry.cpp +++ b/unittest/logservice/test_log_entry_and_group_entry.cpp @@ -19,7 +19,7 @@ #define private public #include "logservice/palf/log_group_entry_header.h" #include "logservice/palf/log_entry.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #undef private #include @@ -50,7 +50,7 @@ TEST(TestLogGroupEntryHeader, test_log_group_entry_header) const char *data = buf + header_size; int64_t data_len = strlen(ptr); int64_t min_timestamp = 0; - SCN max_scn = SCN::min_scn(); + share::SCN max_scn = share::SCN::min_scn(); int64_t log_id = 1; LSN committed_lsn; committed_lsn.val_ = 1; @@ -59,8 +59,8 @@ TEST(TestLogGroupEntryHeader, test_log_group_entry_header) // test LogEntry and LogEntryHeader LogEntry log_entry; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_entry_header.generate_header(NULL, 0, SCN::base_scn())); - EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(data, data_len, SCN::base_scn())); + EXPECT_EQ(OB_INVALID_ARGUMENT, log_entry_header.generate_header(NULL, 0, share::SCN::base_scn())); + EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(data, data_len, share::SCN::base_scn())); log_entry.header_ = log_entry_header; log_entry.buf_ = data; int64_t tmp_pos = 0; diff --git a/unittest/logservice/test_log_meta.cpp b/unittest/logservice/test_log_meta.cpp index da5e37dea..1f7d90df4 100644 --- a/unittest/logservice/test_log_meta.cpp +++ b/unittest/logservice/test_log_meta.cpp @@ -111,8 +111,8 @@ TEST(TestLogMeta, test_log_meta_generate) LogMeta meta1, meta2; LSN prev_lsn(10000), lsn(20000); int64_t init_pid(2); - SCN init_scn; - init_scn.convert_for_lsn_allocator(10); + share::SCN init_scn; + init_scn.convert_for_logservice(10); int64_t init_cksum(10); PalfBaseInfo base_info; LogInfo log_info; diff --git a/unittest/logservice/test_log_meta_info.cpp b/unittest/logservice/test_log_meta_info.cpp index 2b79e627c..2227f959f 100644 --- a/unittest/logservice/test_log_meta_info.cpp +++ b/unittest/logservice/test_log_meta_info.cpp @@ -13,7 +13,7 @@ #define private public #include "logservice/palf/log_meta_info.h" // LogPrepareMeta... #include "logservice/palf/palf_options.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #undef private #include @@ -202,14 +202,14 @@ TEST(TestLogMetaInfos, test_log_mode_meta) LSN lsn; lsn.val_ = 1; ObAddr addr(ObAddr::IPV4, "127.0.0.1", 4096); - SCN invalid_scn; + share::SCN invalid_scn; // Test invalid argument EXPECT_FALSE(log_mode_meta1.is_valid()); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(1, 1, AccessMode::INVALID_ACCESS_MODE, SCN::min_scn())); + EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(1, 1, AccessMode::INVALID_ACCESS_MODE, share::SCN::min_scn())); EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(1, 1, AccessMode::APPEND, invalid_scn)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(1, INVALID_PROPOSAL_ID, AccessMode::APPEND, SCN::min_scn())); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(INVALID_PROPOSAL_ID, 1, AccessMode::APPEND, SCN::min_scn())); - EXPECT_EQ(OB_SUCCESS, log_mode_meta1.generate(1, 1, AccessMode::APPEND, SCN::min_scn())); + EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(1, INVALID_PROPOSAL_ID, AccessMode::APPEND, share::SCN::min_scn())); + EXPECT_EQ(OB_INVALID_ARGUMENT, log_mode_meta1.generate(INVALID_PROPOSAL_ID, 1, AccessMode::APPEND, share::SCN::min_scn())); + EXPECT_EQ(OB_SUCCESS, log_mode_meta1.generate(1, 1, AccessMode::APPEND, share::SCN::min_scn())); EXPECT_TRUE(log_mode_meta1.is_valid()); // Test serialize and deserialize diff --git a/unittest/logservice/test_log_mode_mgr.cpp b/unittest/logservice/test_log_mode_mgr.cpp index 9cf812aff..f49727e8b 100644 --- a/unittest/logservice/test_log_mode_mgr.cpp +++ b/unittest/logservice/test_log_mode_mgr.cpp @@ -106,7 +106,7 @@ TEST_F(TestLogModeMgr, test_init) LogModeMgr mode_mgr; LogModeMeta valid_meta, invalid_meta; ObAddr invalid_addr; - EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, 1, AccessMode::APPEND, SCN::base_scn())); + EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, 1, AccessMode::APPEND, share::SCN::base_scn())); EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.init(-1, addr1, valid_meta, mock_state_mgr_, mock_log_engine_, mock_config_mgr_, mock_sw_)); EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.init(1, invalid_addr, valid_meta, @@ -195,7 +195,7 @@ TEST_F(TestLogModeMgr, test_receive_mode_meta) PALF_LOG(INFO, "test_receive_mode_meta case"); int64_t pid = 1; LogModeMeta valid_meta, invalid_meta; - EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, 1, AccessMode::APPEND, SCN::base_scn())); + EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, 1, AccessMode::APPEND, share::SCN::base_scn())); bool has_accepted = false; { @@ -226,7 +226,7 @@ TEST_F(TestLogModeMgr, test_receive_mode_meta) // can accpet LogModeMgr mode_mgr; init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, FOLLOWER, ACTIVE); - EXPECT_EQ(OB_SUCCESS, valid_meta.generate(2, 2, AccessMode::APPEND, SCN::base_scn())); + EXPECT_EQ(OB_SUCCESS, valid_meta.generate(2, 2, AccessMode::APPEND, share::SCN::base_scn())); EXPECT_TRUE(mode_mgr.can_receive_mode_meta(pid, valid_meta, has_accepted)); EXPECT_FALSE(has_accepted); EXPECT_EQ(OB_SUCCESS, mode_mgr.receive_mode_meta(addr2, 2, valid_meta)); @@ -243,9 +243,9 @@ TEST_F(TestLogModeMgr, test_can_change_access_mode) PALF_LOG(INFO, "test_can_change_access_mode case"); const int64_t mode_version = 1; const AccessMode access_mode = AccessMode::APPEND; - const SCN ref_scn = SCN::base_scn(); + const share::SCN ref_scn = share::SCN::base_scn(); LogModeMeta valid_meta; - EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, mode_version, AccessMode::APPEND, SCN::base_scn())); + EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, mode_version, AccessMode::APPEND, share::SCN::base_scn())); { LogModeMgr mode_mgr; // role check @@ -267,11 +267,11 @@ TEST_F(TestLogModeMgr, test_can_change_access_mode) LogModeMgr mode_mgr; init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, LEADER, ACTIVE); // switch to prepare state - EXPECT_EQ(OB_EAGAIN, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, SCN::base_scn())); + EXPECT_EQ(OB_EAGAIN, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, share::SCN::base_scn())); EXPECT_EQ(MODE_PREPARE, mode_mgr.state_); // accept_log_propopsal_id is 2 EXPECT_EQ(OB_SUCCESS, mode_mgr.handle_prepare_response(addr2, 2, 2, LSN(0), valid_meta)); - EXPECT_EQ(OB_NOT_MASTER, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, SCN::base_scn())); + EXPECT_EQ(OB_NOT_MASTER, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, share::SCN::base_scn())); EXPECT_EQ(MODE_PREPARE, mode_mgr.state_); } PALF_LOG(INFO, "test_can_change_access_mode case"); @@ -332,13 +332,13 @@ TEST_F(TestLogModeMgr, test_change_access_mode) PALF_LOG(INFO, "test_change_access_mode case"); const int64_t mode_version = 1; const AccessMode access_mode = AccessMode::APPEND; - const SCN ref_scn = SCN::base_scn(); + const share::SCN ref_scn = share::SCN::base_scn(); LogModeMeta valid_meta; - EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, mode_version, AccessMode::APPEND, SCN::base_scn())); + EXPECT_EQ(OB_SUCCESS, valid_meta.generate(1, mode_version, AccessMode::APPEND, share::SCN::base_scn())); { // not init, invalid args LogModeMgr mode_mgr; - SCN invalid_scn; + share::SCN invalid_scn; EXPECT_EQ(OB_NOT_INIT, mode_mgr.change_access_mode(mode_version, access_mode, ref_scn)); init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, LEADER, ACTIVE); EXPECT_EQ(OB_INVALID_ARGUMENT, mode_mgr.change_access_mode(-1, access_mode, ref_scn)); @@ -349,21 +349,21 @@ TEST_F(TestLogModeMgr, test_change_access_mode) LogModeMgr mode_mgr; init_test_mode_mgr_env(addr1, valid_meta, mode_mgr, LEADER, ACTIVE); // switch to prepare - EXPECT_EQ(OB_EAGAIN, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, SCN::base_scn())); + EXPECT_EQ(OB_EAGAIN, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, share::SCN::base_scn())); EXPECT_EQ(MODE_PREPARE, mode_mgr.state_); // handle prepare resp EXPECT_EQ(OB_SUCCESS, mode_mgr.handle_prepare_response(addr2, 2, 1, LSN(0), valid_meta)); // switch to accept - EXPECT_EQ(OB_EAGAIN, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, SCN::base_scn())); + EXPECT_EQ(OB_EAGAIN, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, share::SCN::base_scn())); EXPECT_EQ(MODE_ACCEPT, mode_mgr.state_); // switch to accept EXPECT_EQ(OB_SUCCESS, mode_mgr.ack_mode_meta(addr1, 2)); EXPECT_EQ(OB_SUCCESS, mode_mgr.ack_mode_meta(addr2, 2)); // should not reach majority before leader's AccessMode is flushed - EXPECT_EQ(OB_EAGAIN, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, SCN::base_scn())); + EXPECT_EQ(OB_EAGAIN, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, share::SCN::base_scn())); // self is flushed EXPECT_EQ(OB_SUCCESS, mode_mgr.after_flush_mode_meta(mode_mgr.last_submit_mode_meta_)); - EXPECT_EQ(OB_SUCCESS, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, SCN::base_scn())); + EXPECT_EQ(OB_SUCCESS, mode_mgr.change_access_mode(mode_version, AccessMode::RAW_WRITE, share::SCN::base_scn())); EXPECT_EQ(MODE_INIT, mode_mgr.state_); EXPECT_EQ(2, mode_mgr.applied_mode_meta_.mode_version_); EXPECT_EQ(AccessMode::RAW_WRITE, mode_mgr.applied_mode_meta_.access_mode_); diff --git a/unittest/logservice/test_log_sliding_window.cpp b/unittest/logservice/test_log_sliding_window.cpp index 40ed38feb..c4806896e 100644 --- a/unittest/logservice/test_log_sliding_window.cpp +++ b/unittest/logservice/test_log_sliding_window.cpp @@ -180,7 +180,7 @@ TEST_F(TestLogSlidingWindow, test_private_func_batch_01) EXPECT_EQ(OB_SUCCESS, log_sw_.get_last_submit_log_info(lsn, log_id, log_pid)); EXPECT_EQ(OB_SUCCESS, log_sw_.get_max_flushed_log_info(lsn, end_lsn, log_pid)); EXPECT_EQ(OB_SUCCESS, log_sw_.get_last_slide_end_lsn(end_lsn)); - SCN scn = log_sw_.get_last_slide_scn(); + share::SCN scn = log_sw_.get_last_slide_scn(); } TEST_F(TestLogSlidingWindow, test_to_follower_pending) @@ -194,10 +194,10 @@ TEST_F(TestLogSlidingWindow, test_to_follower_pending) &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, base_info)); char *buf = data_buf_; int64_t buf_len = 1 * 1024 * 1024; - SCN ref_scn; - ref_scn.convert_for_lsn_allocator(99); + share::SCN ref_scn; + ref_scn.convert_for_logservice(99); LSN lsn; - SCN scn; + share::SCN scn; buf_len = 2 * 1024 * 1024; EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); EXPECT_EQ(OB_SUCCESS, log_sw_.to_follower_pending(last_lsn)); @@ -246,9 +246,9 @@ TEST_F(TestLogSlidingWindow, test_report_log_task_trace) char *buf = data_buf_; int64_t buf_len = 2 * 1024 * 1024; LSN lsn; - SCN scn; - SCN ref_scn; - ref_scn.convert_for_lsn_allocator(99); + share::SCN scn; + share::SCN ref_scn; + ref_scn.convert_for_logservice(99); buf_len = 2 * 1024 * 1024; EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); EXPECT_EQ(OB_SUCCESS, log_sw_.report_log_task_trace(1)); @@ -286,10 +286,10 @@ TEST_F(TestLogSlidingWindow, test_submit_log) gen_default_palf_base_info_(base_info); char *buf = data_buf_; int64_t buf_len = 1000; - SCN ref_scn; - ref_scn.convert_for_lsn_allocator(99); + share::SCN ref_scn; + ref_scn.convert_for_logservice(99); LSN lsn; - SCN scn; + share::SCN scn; EXPECT_EQ(OB_NOT_INIT, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); EXPECT_EQ(OB_SUCCESS, log_sw_.init(palf_id_, self_, &mock_state_mgr_, &mock_mm_, &mock_mode_mgr_, &mock_log_engine_, &palf_fs_cb_, alloc_mgr_, base_info)); @@ -301,7 +301,7 @@ TEST_F(TestLogSlidingWindow, test_submit_log) buf_len = 1000; ref_scn.reset(); EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); - ref_scn.convert_for_lsn_allocator(99); + ref_scn.convert_for_logservice(99); buf_len = 2 * 1024 * 1024; for (int i = 0; i < 8; ++i) { EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); @@ -332,8 +332,8 @@ TEST_F(TestLogSlidingWindow, test_submit_group_log) // generate log entry and group entry LogEntryHeader log_entry_header; LogGroupEntryHeader group_header; - SCN max_scn; - max_scn.convert_for_lsn_allocator(111111); + share::SCN max_scn; + max_scn.convert_for_logservice(111111); int64_t log_id = 1; LSN committed_end_lsn(0); int64_t log_proposal_id = 10; @@ -387,17 +387,17 @@ TEST_F(TestLogSlidingWindow, test_receive_log) char *buf = data_buf_; int64_t buf_len = 2 * 1024 * 1024; - SCN ref_scn; - ref_scn.convert_for_lsn_allocator(999); - SCN scn; + share::SCN ref_scn; + ref_scn.convert_for_logservice(999); + share::SCN scn; EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); // update lsn for next group entry lsn.val_ = lsn.val_ + LogEntryHeader::HEADER_SER_SIZE + buf_len; // generate new group entry LogEntryHeader log_entry_header; LogGroupEntryHeader group_header; - SCN max_scn; - max_scn.convert_for_lsn_allocator(111111); + share::SCN max_scn; + max_scn.convert_for_logservice(111111); int64_t log_id = 2; LSN committed_end_lsn(0); int64_t log_proposal_id = 20; @@ -413,7 +413,7 @@ TEST_F(TestLogSlidingWindow, test_receive_log) memcpy(data_buf_ + group_header_size + pos, log_data, log_data_len); log_entry_size = pos + log_data_len; // gen 2nd log entry - max_scn.convert_for_lsn_allocator(222222); + max_scn.convert_for_logservice(222222); EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); pos = 0; log_entry_header.serialize(data_buf_ + group_header_size + log_entry_size, DATA_BUF_LEN, pos); @@ -520,8 +520,8 @@ TEST_F(TestLogSlidingWindow, test_receive_log) log_sw_.max_flushed_end_lsn_.val_ += 100; // 增大log_id，构造prev log空洞 log_id += 100; - uint64_t new_val = max_scn.get_val_for_lsn_allocator() - 10; - max_scn.convert_for_lsn_allocator(new_val); + uint64_t new_val = max_scn.get_val_for_logservice() - 10; + max_scn.convert_for_logservice(new_val); LogWriteBuf write_buf1; EXPECT_EQ(OB_SUCCESS, write_buf1.push_back(data_buf_, log_entry_size+group_header_size)); EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf1, log_entry_size, max_scn, log_id, @@ -565,10 +565,10 @@ TEST_F(TestLogSlidingWindow, test_after_flush_log) char *buf = data_buf_; int64_t buf_len = 2 * 1024 * 1024; - SCN ref_scn; - ref_scn.convert_for_lsn_allocator(999); + share::SCN ref_scn; + ref_scn.convert_for_logservice(999); LSN lsn; - SCN scn; + share::SCN scn; EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); EXPECT_EQ(OB_INVALID_ARGUMENT, log_sw_.after_flush_log(flush_log_ctx)); @@ -615,18 +615,18 @@ TEST_F(TestLogSlidingWindow, test_truncate_log) char *buf = data_buf_; int64_t buf_len = 2 * 1024 * 1024; - SCN ref_scn; - ref_scn.convert_for_lsn_allocator(999); + share::SCN ref_scn; + ref_scn.convert_for_logservice(999); LSN lsn; - SCN scn; + share::SCN scn; // submit first log EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); EXPECT_EQ(OB_SUCCESS, log_sw_.try_freeze_last_log()); // generate new group entry LogEntryHeader log_entry_header; LogGroupEntryHeader group_header; - SCN max_scn; - max_scn.convert_for_lsn_allocator(111111); + share::SCN max_scn; + max_scn.convert_for_logservice(111111); int64_t log_id = 2; LSN committed_end_lsn(0); int64_t log_proposal_id = 20; @@ -642,7 +642,7 @@ TEST_F(TestLogSlidingWindow, test_truncate_log) memcpy(data_buf_ + group_header_size + pos, log_data, log_data_len); int64_t log_entry_size = pos + log_data_len; // gen 2nd log entry - max_scn.convert_for_lsn_allocator(222222); + max_scn.convert_for_logservice(222222); EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); pos = 0; log_entry_header.serialize(data_buf_ + group_header_size + log_entry_size, DATA_BUF_LEN, pos); @@ -736,10 +736,10 @@ TEST_F(TestLogSlidingWindow, test_ack_log) char *buf = data_buf_; int64_t buf_len = 2 * 1024 * 1024; - SCN ref_scn; - ref_scn.convert_for_lsn_allocator(999); + share::SCN ref_scn; + ref_scn.convert_for_logservice(999); LSN lsn; - SCN scn; + share::SCN scn; EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); LSN end_lsn = lsn + LogEntryHeader::HEADER_SER_SIZE + buf_len; ObAddr server; @@ -781,16 +781,16 @@ TEST_F(TestLogSlidingWindow, test_truncate_for_rebuild) char *buf = data_buf_; int64_t buf_len = 2 * 1024 * 1024; - SCN ref_scn; - ref_scn.convert_for_lsn_allocator(999); + share::SCN ref_scn; + ref_scn.convert_for_logservice(999); LSN lsn; - SCN scn; + share::SCN scn; EXPECT_EQ(OB_SUCCESS, log_sw_.submit_log(buf, buf_len, ref_scn, lsn, scn)); // generate new group entry LogEntryHeader log_entry_header; LogGroupEntryHeader group_header; - SCN max_scn; - max_scn.convert_for_lsn_allocator(111111); + share::SCN max_scn; + max_scn.convert_for_logservice(111111); int64_t log_id = 2; LSN committed_end_lsn(0); int64_t log_proposal_id = 20; @@ -806,7 +806,7 @@ TEST_F(TestLogSlidingWindow, test_truncate_for_rebuild) memcpy(data_buf_ + group_header_size + pos, log_data, log_data_len); int64_t log_entry_size = pos + log_data_len; // gen 2nd log entry - max_scn.convert_for_lsn_allocator(222222); + max_scn.convert_for_logservice(222222); EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); pos = 0; log_entry_header.serialize(data_buf_ + group_header_size + log_entry_size, DATA_BUF_LEN, pos); @@ -842,8 +842,8 @@ TEST_F(TestLogSlidingWindow, test_truncate_for_rebuild) EXPECT_EQ(OB_SUCCESS, log_sw_.receive_log(src_server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, data_buf_, group_entry_size, false, truncate_log_info)); // gen next group log log_id = 10; - uint64_t new_val = max_scn.get_val_for_lsn_allocator() + 100; - max_scn.convert_for_lsn_allocator(new_val); + uint64_t new_val = max_scn.get_val_for_logservice() + 100; + max_scn.convert_for_logservice(new_val); LogWriteBuf write_buf1; EXPECT_EQ(OB_SUCCESS, write_buf1.push_back(data_buf_, log_entry_size+group_header_size)); EXPECT_EQ(OB_SUCCESS, group_header.generate(false, false, write_buf1, log_entry_size, max_scn, log_id, @@ -868,8 +868,8 @@ TEST_F(TestLogSlidingWindow, test_truncate_for_rebuild) EXPECT_EQ(OB_SUCCESS, log_sw_.truncate_for_rebuild(new_base_info)); new_base_info.prev_log_info_.log_id_ = 6; - new_val = max_scn.get_val_for_lsn_allocator() - 50; - new_base_info.prev_log_info_.scn_.convert_for_lsn_allocator(new_val); + new_val = max_scn.get_val_for_logservice() - 50; + new_base_info.prev_log_info_.scn_.convert_for_logservice(new_val); new_base_info.prev_log_info_.log_proposal_id_ = curr_proposal_id; new_base_info.prev_log_info_.lsn_ = lsn - 200; new_base_info.curr_lsn_ = lsn - 100; @@ -893,8 +893,8 @@ TEST_F(TestLogSlidingWindow, test_append_disk_log) LogEntry log_entry; LogEntryHeader log_entry_header; LogGroupEntryHeader group_header; - SCN max_scn; - max_scn.convert_for_lsn_allocator(111111); + share::SCN max_scn; + max_scn.convert_for_logservice(111111); int64_t log_id = 1; LSN committed_end_lsn(0); int64_t log_proposal_id = 20; @@ -923,7 +923,7 @@ TEST_F(TestLogSlidingWindow, test_append_disk_log) int64_t log_entry_size = pos + log_data_len; // gen 2nd log entry - max_scn.convert_for_lsn_allocator(222222); + max_scn.convert_for_logservice(222222); EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); pos = 0; log_entry_header.serialize(data_buf_ + group_header_size + log_entry_size, DATA_BUF_LEN, pos); @@ -973,8 +973,8 @@ TEST_F(TestLogSlidingWindow, test_append_disk_log) EXPECT_EQ(OB_SUCCESS, log_sw_.append_disk_log(lsn, group_entry)); // gen new group entry log_id++; - uint64_t new_val = max_scn.get_val_for_lsn_allocator() + 100; - max_scn.convert_for_lsn_allocator(new_val); + uint64_t new_val = max_scn.get_val_for_logservice() + 100; + max_scn.convert_for_logservice(new_val); lsn.val_ += group_entry_size; // gen group log LogWriteBuf write_buf1; @@ -1000,8 +1000,8 @@ TEST_F(TestLogSlidingWindow, test_group_entry_truncate) LSN lsn(0); LogGroupEntry group_entry; LogGroupEntryHeader group_header; - SCN truncate_scn; - truncate_scn.convert_for_lsn_allocator(111113); + share::SCN truncate_scn; + truncate_scn.convert_for_logservice(111113); int64_t pre_accum_checksum = 123456; EXPECT_EQ(OB_INVALID_ARGUMENT, group_header.truncate(NULL, 1024, truncate_scn, pre_accum_checksum)); EXPECT_EQ(OB_INVALID_ARGUMENT, group_header.truncate(data_buf_, 0, truncate_scn, pre_accum_checksum)); @@ -1014,8 +1014,8 @@ TEST_F(TestLogSlidingWindow, test_group_entry_truncate) // generate new group entry LogEntry log_entry; LogEntryHeader log_entry_header; - SCN max_scn; - max_scn.convert_for_lsn_allocator(111111); + share::SCN max_scn; + max_scn.convert_for_logservice(111111); int64_t log_id = 1; LSN committed_end_lsn(0); int64_t log_proposal_id = 20; @@ -1032,7 +1032,7 @@ TEST_F(TestLogSlidingWindow, test_group_entry_truncate) int64_t dser_pos = 0; int64_t log_entry_size = pos + log_data_len; // gen 2nd log entry - max_scn.convert_for_lsn_allocator(222222); + max_scn.convert_for_logservice(222222); EXPECT_EQ(OB_SUCCESS, log_entry_header.generate_header(log_data, log_data_len, max_scn)); pos = 0; log_entry_header.serialize(data_buf_ + group_header_size + log_entry_size, DATA_BUF_LEN, pos); diff --git a/unittest/logservice/test_lsn_allocator.cpp b/unittest/logservice/test_lsn_allocator.cpp index 6f23263ae..b2e872e1a 100644 --- a/unittest/logservice/test_lsn_allocator.cpp +++ b/unittest/logservice/test_lsn_allocator.cpp @@ -15,7 +15,7 @@ #include #include -#include "logservice/palf/scn.h" +#include "share/scn.h" #define private public #include "logservice/palf/lsn_allocator.h" #undef private @@ -75,7 +75,7 @@ void init_size_array() void init_offset_allocator() { LSN start_lsn(0); - EXPECT_EQ(OB_SUCCESS, golbal_lsn_allocator.init(0, SCN::base_scn(), start_lsn)); + EXPECT_EQ(OB_SUCCESS, golbal_lsn_allocator.init(0, share::SCN::base_scn(), start_lsn)); } TEST_F(TestLSNAllocator, test_struct_field_value_upper_bound) @@ -153,7 +153,7 @@ TEST_F(TestLSNAllocator, test_lsn_allocator_init) { LSN start_lsn; int64_t initial_log_id = OB_INVALID_LOG_ID; - SCN initial_scn; + share::SCN initial_scn; EXPECT_EQ(OB_INVALID_ARGUMENT, lsn_allocator_.init(initial_log_id, initial_scn, start_lsn)); initial_log_id = 0; EXPECT_EQ(OB_INVALID_ARGUMENT, lsn_allocator_.init(initial_log_id, initial_scn, start_lsn)); @@ -168,16 +168,16 @@ TEST_F(TestLSNAllocator, test_lsn_allocator_alloc_lsn_scn) { LSN start_lsn; int64_t initial_log_id = OB_INVALID_LOG_ID; - SCN initial_scn = SCN::min_scn(); + share::SCN initial_scn = share::SCN::min_scn(); initial_log_id = 0; start_lsn.val_ = 0; - SCN b_scn; - b_scn.convert_for_lsn_allocator(1000); + share::SCN b_scn; + b_scn.convert_for_logservice(1000); int64_t size = 1000000; LSN lsn; int64_t log_id; - SCN scn; + share::SCN scn; bool is_new_log = false; bool need_gen_padding_entry = false; int64_t padding_len = 0; @@ -199,30 +199,30 @@ TEST_F(TestLSNAllocator, test_lsn_allocator_truncate) LSN start_lsn; int64_t initial_log_id = OB_INVALID_LOG_ID; initial_log_id = 0; - SCN initial_scn = SCN::min_scn(); + share::SCN initial_scn = share::SCN::min_scn(); start_lsn.val_ = 0; LSN tmp_lsn; int64_t tmp_log_id = 9999; - SCN tmp_scn; - tmp_scn.convert_for_lsn_allocator(55555); + share::SCN tmp_scn; + tmp_scn.convert_for_logservice(55555); LSN end_lsn; int64_t end_log_id = OB_INVALID_LOG_ID; - SCN b_scn; - b_scn.convert_for_lsn_allocator(1000); + share::SCN b_scn; + b_scn.convert_for_logservice(1000); int64_t size = 1000000; LSN lsn; int64_t log_id; - SCN scn; + share::SCN scn; bool is_new_log = false; bool need_gen_padding_entry = false; int64_t padding_len = 0; // test truncate() - SCN new_scn; - new_scn.convert_for_lsn_allocator(10); + share::SCN new_scn; + new_scn.convert_for_logservice(10); const int64_t truncate_log_id = 1024; EXPECT_EQ(OB_NOT_INIT, lsn_allocator_.truncate(tmp_lsn, truncate_log_id, new_scn)); EXPECT_EQ(OB_NOT_INIT, lsn_allocator_.inc_update_last_log_info(tmp_lsn, tmp_log_id, tmp_scn)); @@ -281,11 +281,11 @@ TEST_F(TestLSNAllocator, test_alloc_offset_single_thread) int64_t idx = rand() % LOG_LOG_CNT; const int64_t begin_ts = ObTimeUtility::current_time_ns(); for (int i = 0; i < 1000000; i++) { - SCN b_scn = SCN::base_scn(); + share::SCN b_scn = share::SCN::base_scn(); int64_t size = log_size_array[idx]; LSN ret_offset; int64_t ret_log_id; - SCN ret_scn; + share::SCN ret_scn; bool is_new_log = false; bool need_gen_padding_entry = false; int64_t padding_len = 0; @@ -327,11 +327,11 @@ public: static void* routine(void *arg) { TestThread *test_thread = static_cast(arg); - SCN scn = SCN::base_scn(); + share::SCN scn = share::SCN::base_scn(); int64_t size = test_thread->log_size_; LSN ret_offset; int64_t ret_log_id; - SCN ret_scn; + share::SCN ret_scn; bool is_new_log = false; bool need_gen_padding_entry = false; int64_t padding_len = 0; diff --git a/unittest/logservice/test_ob_election_with_priority.cpp b/unittest/logservice/test_ob_election_with_priority.cpp index b6692c1e9..54b97215e 100644 --- a/unittest/logservice/test_ob_election_with_priority.cpp +++ b/unittest/logservice/test_ob_election_with_priority.cpp @@ -271,7 +271,7 @@ TEST_F(TestElectionWithPriority, not_change_leader_because_follower_memership_ve ASSERT_EQ(stop_to_be_follower_count, 0); // 1的membership version再加1，同时让2的log_ts超过比较阈值 election_group[1]->proposer_.memberlist_with_states_.p_impl_->member_list_.membership_version_.config_seq_ += 1; - v_pri[2].priority_tuple_.element<1>().scn_.convert_for_lsn_allocator(100 * 1000 * 1000 * 1000L);// 此时并不会触发切主 + v_pri[2].priority_tuple_.element<1>().scn_.convert_for_logservice(100 * 1000 * 1000 * 1000L);// 此时并不会触发切主 this_thread::sleep_for(chrono::seconds(1));// 等待切主 ASSERT_EQ(election_group[1]->proposer_.role_, ObRole::LEADER); ASSERT_EQ(leader_takeover_times, 1); diff --git a/unittest/logservice/test_ob_role_change_service.cpp b/unittest/logservice/test_ob_role_change_service.cpp index 0afb8b0ff..cba6df80d 100644 --- a/unittest/logservice/test_ob_role_change_service.cpp +++ b/unittest/logservice/test_ob_role_change_service.cpp @@ -15,7 +15,7 @@ #include "logservice/rcservice/ob_role_change_handler.h" #include "logservice/rcservice/ob_role_change_service.h" #include "logservice/replayservice/ob_log_replay_service.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "share/ob_ls_id.h" #include "storage/tx_storage/ob_ls_service.h" @@ -95,11 +95,11 @@ public: virtual bool is_valid() const { return true; } virtual int append(const void *buffer, const int64_t nbytes, - const palf::SCN &ref_scn, + const share::SCN &ref_scn, const bool need_nonblock, logservice::AppendCb *cb, palf::LSN &lsn, - palf::SCN &scn) + share::SCN &scn) { UNUSED(need_nonblock); UNUSED(buffer); @@ -160,21 +160,21 @@ public: {UNUSED(start_lsn); UNUSED(iter); return OB_SUCCESS;}; int seek(const palf::LSN &start_lsn, const int64_t iterate_size, palf::PalfGroupBufferIterator &iter) {UNUSED(start_lsn); UNUSED(iterate_size); UNUSED(iter); return OB_SUCCESS;}; - int seek(const palf::SCN &start_scn, palf::PalfGroupBufferIterator &iter) + int seek(const share::SCN &start_scn, palf::PalfGroupBufferIterator &iter) {UNUSED(start_scn); UNUSED(iter); return OB_SUCCESS;}; int set_initial_member_list(const common::ObMemberList &member_list, const int64_t paxos_replica_num) {UNUSED(member_list); UNUSED(paxos_replica_num); return OB_SUCCESS;} - int locate_by_scn_coarsely(const palf::SCN &scn, palf::LSN &lsn) + int locate_by_scn_coarsely(const share::SCN &scn, palf::LSN &lsn) {UNUSED(scn); UNUSED(lsn); return OB_SUCCESS;} - int locate_by_lsn_coarsely(const palf::LSN &lsn, palf::SCN &scn) + int locate_by_lsn_coarsely(const palf::LSN &lsn, share::SCN &scn) {UNUSED(scn); UNUSED(lsn); return OB_SUCCESS;} int advance_base_lsn(const palf::LSN &lsn) {UNUSED(lsn); return OB_SUCCESS;} int get_base_lsn(palf::LSN &lsn) {UNUSED(lsn); return OB_SUCCESS;} - int get_base_scn(palf::SCN &base_lsn_scn) const + int get_base_scn(share::SCN &base_lsn_scn) const {UNUSED(base_lsn_scn); return OB_SUCCESS;} - int get_end_scn(palf::SCN &scn) const + int get_end_scn(share::SCN &scn) const {UNUSED(scn); return OB_SUCCESS;} int get_paxos_member_list(common::ObMemberList &member_list, int64_t &paxos_replica_num) const {UNUSED(member_list); UNUSED(paxos_replica_num); return OB_SUCCESS;} diff --git a/unittest/logservice/test_scn.cpp b/unittest/logservice/test_scn.cpp index 930cd4873..0f34ad352 100644 --- a/unittest/logservice/test_scn.cpp +++ b/unittest/logservice/test_scn.cpp @@ -10,14 +10,14 @@ * See the Mulan PubL v2 for more details. */ -#include "logservice/palf/scn.h" +#include "share/scn.h" #include #include namespace oceanbase { using namespace common; -using namespace palf; +using namespace share; namespace unittest { @@ -51,18 +51,18 @@ TEST(TestSCN, test_scn) scn1.reset(); scn2.reset(); - EXPECT_EQ(OB_SUCCESS, scn1.convert_for_lsn_allocator(0)); + EXPECT_EQ(OB_SUCCESS, scn1.convert_for_logservice(0)); EXPECT_TRUE(scn1.is_valid()); scn1.reset(); - EXPECT_EQ(OB_INVALID_ARGUMENT, scn1.convert_for_lsn_allocator(4611686018427387904)); + EXPECT_EQ(OB_INVALID_ARGUMENT, scn1.convert_for_logservice(4611686018427387904)); EXPECT_FALSE(scn1.is_valid()); scn1.reset(); - EXPECT_EQ(OB_INVALID_ARGUMENT, scn1.convert_for_lsn_allocator(-1)); + EXPECT_EQ(OB_INVALID_ARGUMENT, scn1.convert_for_logservice(-1)); EXPECT_FALSE(scn1.is_valid()); scn1.reset(); - EXPECT_EQ(OB_SUCCESS, scn1.convert_for_lsn_allocator(4611686018427387903)); + EXPECT_EQ(OB_SUCCESS, scn1.convert_for_logservice(4611686018427387903)); EXPECT_TRUE(scn1.is_valid()); - EXPECT_EQ(OB_SUCCESS, scn1.convert_for_lsn_allocator(100000)); + EXPECT_EQ(OB_SUCCESS, scn1.convert_for_logservice(100000)); EXPECT_TRUE(scn1.is_valid()); scn1.reset(); @@ -162,9 +162,9 @@ TEST(TestSCN, test_update) SCN scn_invalid; //test inc_update and dec_update - EXPECT_EQ(OB_SUCCESS, self.convert_for_lsn_allocator(100)); - EXPECT_EQ(OB_SUCCESS, bigger.convert_for_lsn_allocator(200)); - EXPECT_EQ(OB_SUCCESS, smaller.convert_for_lsn_allocator(1)); + EXPECT_EQ(OB_SUCCESS, self.convert_for_logservice(100)); + EXPECT_EQ(OB_SUCCESS, bigger.convert_for_logservice(200)); + EXPECT_EQ(OB_SUCCESS, smaller.convert_for_logservice(1)); result = self.inc_update(scn_invalid); EXPECT_EQ(result, self); @@ -209,7 +209,7 @@ TEST(TestSCN, test_serialize) int64_t pos3 = 0; int64_t pos4 = 0; - EXPECT_EQ(OB_SUCCESS, scn.convert_for_lsn_allocator(100)); + EXPECT_EQ(OB_SUCCESS, scn.convert_for_logservice(100)); EXPECT_EQ(scn.get_fixed_serialize_size(), max.get_fixed_serialize_size()); EXPECT_EQ(OB_SUCCESS, scn.fixed_serialize(buf1, BUF_LEN, pos1)); EXPECT_EQ(OB_SUCCESS, max.fixed_serialize(buf2, BUF_LEN, pos2)); diff --git a/unittest/rootserver/test_archive_checkpoint.cpp b/unittest/rootserver/test_archive_checkpoint.cpp index 7e5ebbd71..4fcaf1851 100644 --- a/unittest/rootserver/test_archive_checkpoint.cpp +++ b/unittest/rootserver/test_archive_checkpoint.cpp @@ -62,9 +62,9 @@ public: round.dest_id_ = DEST_ID; round.round_id_ = ROUND_ID; // round.state_ - round.start_scn_ = palf::SCN::min_scn(); - round.checkpoint_scn_ = palf::SCN::min_scn(); - round.max_scn_ = palf::SCN::min_scn(); + round.start_scn_ = share::SCN::min_scn(); + round.checkpoint_scn_ = share::SCN::min_scn(); + round.max_scn_ = share::SCN::min_scn(); // round.compatible_ round.base_piece_id_ = 1; round.used_piece_id_ = 1; @@ -117,8 +117,8 @@ public: fill_beginning_round(round, start_time); round.state_ = state; - round.checkpoint_scn_.convert_for_lsn_allocator(convert_timestr_2_scn(checkpoint_time)); - round.max_scn_.convert_for_lsn_allocator(convert_timestr_2_scn(max_time)); + round.checkpoint_scn_.convert_for_logservice(convert_timestr_2_scn(checkpoint_time)); + round.max_scn_.convert_for_logservice(convert_timestr_2_scn(max_time)); round.used_piece_id_ = used_piece_id; round.frozen_input_bytes_ = frozen_input_bytes; @@ -146,8 +146,8 @@ public: new_round.deep_copy_from(old_round); new_round.state_ = state; - new_round.checkpoint_scn_.convert_for_lsn_allocator(convert_timestr_2_scn(checkpoint_time)); - new_round.max_scn_.convert_for_lsn_allocator(convert_timestr_2_scn(max_time)); + new_round.checkpoint_scn_.convert_for_logservice(convert_timestr_2_scn(checkpoint_time)); + new_round.max_scn_.convert_for_logservice(convert_timestr_2_scn(max_time)); new_round.used_piece_id_ = used_piece_id; new_round.frozen_input_bytes_ = frozen_input_bytes; @@ -177,8 +177,8 @@ public: piece.dest_no_ = old_round.key_.dest_no_; ObTenantArchiveMgr::decide_piece_start_scn(old_round.start_scn_, old_round.base_piece_id_, old_round.piece_switch_interval_, piece_id, piece.start_scn_); - piece.checkpoint_scn_.convert_for_lsn_allocator(convert_timestr_2_scn(checkpoint_time)); - piece.max_scn_.convert_for_lsn_allocator(convert_timestr_2_scn(max_time)); + piece.checkpoint_scn_.convert_for_logservice(convert_timestr_2_scn(checkpoint_time)); + piece.max_scn_.convert_for_logservice(convert_timestr_2_scn(max_time)); ObTenantArchiveMgr::decide_piece_end_scn(old_round.start_scn_, old_round.base_piece_id_, old_round.piece_switch_interval_, piece_id, piece.end_scn_); piece.input_bytes_ = input_bytes; @@ -226,8 +226,8 @@ public: piece_summary.incarnation_ = OB_START_INCARNATION; piece_summary.state_ = state; - piece_summary.start_scn_.convert_for_lsn_allocator(convert_timestr_2_scn(start_time)); - piece_summary.checkpoint_scn_.convert_for_lsn_allocator(convert_timestr_2_scn(checkpoint_time)); + piece_summary.start_scn_.convert_for_logservice(convert_timestr_2_scn(start_time)); + piece_summary.checkpoint_scn_.convert_for_logservice(convert_timestr_2_scn(checkpoint_time)); piece_summary.min_lsn_ = min_lsn; piece_summary.max_lsn_ = max_lsn; piece_summary.input_bytes_ = input_bytes; @@ -245,8 +245,8 @@ public: { ObDestRoundCheckpointer::GeneratedLSPiece piece; piece.ls_id_ = ObLSID(ls_id); - piece.start_scn_.convert_for_lsn_allocator(convert_timestr_2_scn(start_time)); - piece.checkpoint_scn_.convert_for_lsn_allocator(convert_timestr_2_scn(checkpoint_time)); + piece.start_scn_.convert_for_logservice(convert_timestr_2_scn(start_time)); + piece.checkpoint_scn_.convert_for_logservice(convert_timestr_2_scn(checkpoint_time)); piece.min_lsn_ = min_lsn; piece.max_lsn_ = max_lsn; piece.input_bytes_ = input_bytes; @@ -443,8 +443,8 @@ TEST_F(ArchiveCheckpointerTest, in_prepare) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN scn; - (void)scn.convert_for_lsn_allocator(1000); + share::SCN scn; + (void)scn.convert_for_logservice(1000); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -521,8 +521,8 @@ TEST_F(ArchiveCheckpointerTest, in_beginning_01) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:00:50")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:00:50")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -702,8 +702,8 @@ TEST_F(ArchiveCheckpointerTest, in_beginning_02) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:00:50")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:00:50")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -883,8 +883,8 @@ TEST_F(ArchiveCheckpointerTest, in_beginning_03) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:00:45")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:00:45")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -1076,8 +1076,8 @@ TEST_F(ArchiveCheckpointerTest, in_doing_01) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:00:45")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:00:45")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -1331,8 +1331,8 @@ TEST_F(ArchiveCheckpointerTest, in_doing_02) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:00:50")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:00:50")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -1668,8 +1668,8 @@ TEST_F(ArchiveCheckpointerTest, in_doing_03) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:03:00")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:03:00")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -1758,8 +1758,8 @@ TEST_F(ArchiveCheckpointerTest, in_doing_04) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:00:45")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:00:45")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -1951,8 +1951,8 @@ TEST_F(ArchiveCheckpointerTest, in_doing_05) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:00:40")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:00:40")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -2144,8 +2144,8 @@ TEST_F(ArchiveCheckpointerTest, in_stopping_01) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:02:00")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:02:00")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -2337,8 +2337,8 @@ TEST_F(ArchiveCheckpointerTest, in_stopping_02) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:02:00")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:02:00")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -2508,8 +2508,8 @@ TEST_F(ArchiveCheckpointerTest, in_stopping_03) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:02:00")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:02:00")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -2702,8 +2702,8 @@ TEST_F(ArchiveCheckpointerTest, in_stopping_04) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:00:40")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:00:40")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); @@ -3013,8 +3013,8 @@ TEST_F(ArchiveCheckpointerTest, some_ls_interrupt_01) g_call_cnt = 0; MockRoundHandler mock_handler; ObDestRoundCheckpointer checkpointer; - palf::SCN limit_scn; - (void)limit_scn.convert_for_lsn_allocator(convert_timestr_2_scn("2022-01-01 00:03:00")); + share::SCN limit_scn; + (void)limit_scn.convert_for_logservice(convert_timestr_2_scn("2022-01-01 00:03:00")); ret = checkpointer.init(&mock_handler, gen_piece_cb, round_cb, limit_scn); ASSERT_EQ(OB_SUCCESS, ret); diff --git a/unittest/storage/backup/test_backup.h b/unittest/storage/backup/test_backup.h index fb8c2d4fe..dfcce7439 100644 --- a/unittest/storage/backup/test_backup.h +++ b/unittest/storage/backup/test_backup.h @@ -20,7 +20,7 @@ #include "storage/blocksstable/ob_macro_block_id.h" #include "storage/backup/ob_backup_index_cache.h" #include "storage/backup/ob_backup_data_struct.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "storage/blocksstable/ob_logic_macro_id.h" namespace oceanbase diff --git a/unittest/storage/init_basic_struct.h b/unittest/storage/init_basic_struct.h index 8a662c0df..60a0de5a9 100644 --- a/unittest/storage/init_basic_struct.h +++ b/unittest/storage/init_basic_struct.h @@ -16,7 +16,7 @@ #include "share/schema/ob_table_schema.h" #include "share/ob_tenant_info_proxy.h" #include "logservice/palf/palf_base_info.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { namespace storage @@ -61,7 +61,7 @@ int gen_create_ls_arg(const int64_t tenant_id, palf::PalfBaseInfo palf_base_info; if (OB_FAIL(tenant_info.init(tenant_id, share::PRIMARY_TENANT_ROLE))) { STORAGE_LOG(WARN, "failed to init tenant info", KR(ret), K(tenant_id)); - } else if (OB_FAIL(arg.init(tenant_id, ls_id, replica_type, property, tenant_info, palf::SCN::min_scn(), compat_mode, false, palf_base_info))) { + } else if (OB_FAIL(arg.init(tenant_id, ls_id, replica_type, property, tenant_info, share::SCN::min_scn(), compat_mode, false, palf_base_info))) { STORAGE_LOG(WARN, "failed to init arg", KR(ret), K(tenant_id), K(ls_id), K(tenant_info), K(create_ts_ns), K(compat_mode), K(palf_base_info)); } return ret; @@ -109,7 +109,7 @@ int gen_create_tablet_arg(const int64_t tenant_id, false))) { STORAGE_LOG(WARN, "failed to init tablet info", KR(ret), K(index_tablet_ids), K(tablet_id), K(index_tablet_schema_idxs)); - } else if (OB_FAIL(arg.init_create_tablet(ls_id, palf::SCN::min_scn()))) { + } else if (OB_FAIL(arg.init_create_tablet(ls_id, share::SCN::min_scn()))) { STORAGE_LOG(WARN, "failed to init create tablet", KR(ret), K(tenant_id), K(ls_id)); } else if (OB_FAIL(arg.table_schemas_.push_back(table_schema))) { STORAGE_LOG(WARN, "failed to push back table schema", KR(ret), K(table_schema)); diff --git a/unittest/storage/memtable/mvcc/test_mvcc_callback.cpp b/unittest/storage/memtable/mvcc/test_mvcc_callback.cpp index e54404e7a..c0851747d 100644 --- a/unittest/storage/memtable/mvcc/test_mvcc_callback.cpp +++ b/unittest/storage/memtable/mvcc/test_mvcc_callback.cpp @@ -34,7 +34,7 @@ public: ObMockTxCallback(ObMemtable *mt, bool need_submit_log = true, bool need_fill_redo = true, - palf::SCN scn = palf::SCN::max_scn(), + share::SCN scn = share::SCN::max_scn(), int64_t seq_no = INT64_MAX) : ObITransCallback(need_fill_redo, need_submit_log), mt_(mt), seq_no_(seq_no) { scn_ = scn; } @@ -43,7 +43,7 @@ public: virtual int64_t get_seq_no() const override { return seq_no_; } virtual int checkpoint_callback() override; virtual int rollback_callback() override; - virtual int calc_checksum(const palf::SCN checksum_scn, + virtual int calc_checksum(const share::SCN checksum_scn, ObBatchChecksum *checksumer) override; ObMemtable *mt_; @@ -138,7 +138,7 @@ public: ObMockTxCallback *create_callback(ObMemtable *mt, bool need_submit_log = true, bool need_fill_redo = true, - palf::SCN scn = palf::SCN::max_scn()) + share::SCN scn = share::SCN::max_scn()) { int64_t seq_no = ++seq_counter_; ObMockTxCallback *cb = new ObMockTxCallback(mt, @@ -152,7 +152,7 @@ public: void create_and_append_callback(ObMemtable *mt, bool need_submit_log = true, bool need_fill_redo = true, - palf::SCN scn = palf::SCN::max_scn()) + share::SCN scn = share::SCN::max_scn()) { ObMockTxCallback *cb = create_callback(mt, need_submit_log, @@ -215,7 +215,7 @@ int ObMockTxCallback::rollback_callback() return OB_SUCCESS; } -int ObMockTxCallback::calc_checksum(const palf::SCN checksum_scn, +int ObMockTxCallback::calc_checksum(const share::SCN checksum_scn, ObBatchChecksum *) { if (checksum_scn <= scn_) { @@ -230,8 +230,8 @@ int ObMockTxCallback::calc_checksum(const palf::SCN checksum_scn, TEST_F(TestTxCallbackList, remove_callback_by_tx_commit) { ObMemtable *memtable = create_memtable(); - palf::SCN scn_1; - scn_1.convert_for_lsn_allocator(1); + share::SCN scn_1; + scn_1.convert_for_logservice(1); create_and_append_callback(memtable, false, /*need_submit_log*/ @@ -257,8 +257,8 @@ TEST_F(TestTxCallbackList, remove_callback_by_tx_commit) TEST_F(TestTxCallbackList, remove_callback_by_tx_abort) { ObMemtable *memtable = create_memtable(); - palf::SCN scn_1; - scn_1.convert_for_lsn_allocator(1); + share::SCN scn_1; + scn_1.convert_for_logservice(1); create_and_append_callback(memtable, false, /*need_submit_log*/ false, /*need_fill_redo*/ @@ -284,12 +284,12 @@ TEST_F(TestTxCallbackList, remove_callback_by_release_memtable) ObMemtable *memtable1 = create_memtable(); ObMemtable *memtable2 = create_memtable(); ObMemtable *memtable3 = create_memtable(); - palf::SCN scn_1; - palf::SCN scn_2; - palf::SCN scn_100; - scn_1.convert_for_lsn_allocator(1); - scn_2.convert_for_lsn_allocator(2); - scn_100.convert_for_lsn_allocator(100); + share::SCN scn_1; + share::SCN scn_2; + share::SCN scn_100; + scn_1.convert_for_logservice(1); + scn_2.convert_for_logservice(2); + scn_100.convert_for_logservice(100); create_and_append_callback(memtable1, false, /*need_submit_log*/ false, /*need_fill_redo*/ @@ -352,14 +352,14 @@ TEST_F(TestTxCallbackList, remove_callback_by_fast_commit) ObMemtable *memtable1 = create_memtable(); ObMemtable *memtable2 = create_memtable(); ObMemtable *memtable3 = create_memtable(); - palf::SCN scn_1; - palf::SCN scn_2; - palf::SCN scn_3; - palf::SCN scn_100; - scn_1.convert_for_lsn_allocator(1); - scn_2.convert_for_lsn_allocator(2); - scn_3.convert_for_lsn_allocator(3); - scn_100.convert_for_lsn_allocator(100); + share::SCN scn_1; + share::SCN scn_2; + share::SCN scn_3; + share::SCN scn_100; + scn_1.convert_for_logservice(1); + scn_2.convert_for_logservice(2); + scn_3.convert_for_logservice(3); + scn_100.convert_for_logservice(100); create_and_append_callback(memtable1, false, /*need_submit_log*/ @@ -430,12 +430,12 @@ TEST_F(TestTxCallbackList, remove_callback_by_rollback_to) ObMemtable *memtable1 = create_memtable(); ObMemtable *memtable2 = create_memtable(); ObMemtable *memtable3 = create_memtable(); - palf::SCN scn_1; - palf::SCN scn_2; - palf::SCN scn_3; - scn_1.convert_for_lsn_allocator(1); - scn_2.convert_for_lsn_allocator(2); - scn_3.convert_for_lsn_allocator(3); + share::SCN scn_1; + share::SCN scn_2; + share::SCN scn_3; + scn_1.convert_for_logservice(1); + scn_2.convert_for_logservice(2); + scn_3.convert_for_logservice(3); int64_t savepoint0 = get_seq_no(); create_and_append_callback(memtable1, @@ -500,12 +500,12 @@ TEST_F(TestTxCallbackList, remove_callback_by_clean_unlog_callbacks) ObMemtable *memtable1 = create_memtable(); ObMemtable *memtable2 = create_memtable(); ObMemtable *memtable3 = create_memtable(); - palf::SCN scn_1; - palf::SCN scn_2; - palf::SCN scn_3; - scn_1.convert_for_lsn_allocator(1); - scn_2.convert_for_lsn_allocator(2); - scn_3.convert_for_lsn_allocator(3); + share::SCN scn_1; + share::SCN scn_2; + share::SCN scn_3; + scn_1.convert_for_logservice(1); + scn_2.convert_for_logservice(2); + scn_3.convert_for_logservice(3); create_and_append_callback(memtable1, false, /*need_submit_log*/ @@ -554,14 +554,14 @@ TEST_F(TestTxCallbackList, remove_callback_by_replay_fail) ObMemtable *memtable1 = create_memtable(); ObMemtable *memtable2 = create_memtable(); ObMemtable *memtable3 = create_memtable(); - palf::SCN scn_1; - palf::SCN scn_2; - palf::SCN scn_3; - palf::SCN scn_4; - scn_1.convert_for_lsn_allocator(1); - scn_2.convert_for_lsn_allocator(2); - scn_3.convert_for_lsn_allocator(3); - scn_4.convert_for_lsn_allocator(4); + share::SCN scn_1; + share::SCN scn_2; + share::SCN scn_3; + share::SCN scn_4; + scn_1.convert_for_logservice(1); + scn_2.convert_for_logservice(2); + scn_3.convert_for_logservice(3); + scn_4.convert_for_logservice(4); create_and_append_callback(memtable1, false, /*need_submit_log*/ @@ -628,18 +628,18 @@ TEST_F(TestTxCallbackList, checksum_leader_tx_end_basic) EXPECT_EQ(OB_SUCCESS, callback_list_.tx_calc_checksum_all()); EXPECT_EQ(true, is_checksum_equal(3, checksum_)); - EXPECT_EQ(palf::SCN::max_scn(), callback_list_.checksum_scn_); + EXPECT_EQ(share::SCN::max_scn(), callback_list_.checksum_scn_); } TEST_F(TestTxCallbackList, checksum_follower_tx_end) { ObMemtable *memtable = create_memtable(); - palf::SCN scn_1; - palf::SCN scn_2; - palf::SCN scn_3; - scn_1.convert_for_lsn_allocator(1); - scn_2.convert_for_lsn_allocator(2); - scn_3.convert_for_lsn_allocator(3); + share::SCN scn_1; + share::SCN scn_2; + share::SCN scn_3; + scn_1.convert_for_logservice(1); + scn_2.convert_for_logservice(2); + scn_3.convert_for_logservice(3); create_and_append_callback(memtable, false, /*need_submit_log*/ @@ -659,17 +659,17 @@ TEST_F(TestTxCallbackList, checksum_follower_tx_end) EXPECT_EQ(OB_SUCCESS, callback_list_.tx_calc_checksum_all()); EXPECT_EQ(true, is_checksum_equal(3, checksum_)); - EXPECT_EQ(palf::SCN::max_scn(), callback_list_.checksum_scn_); + EXPECT_EQ(share::SCN::max_scn(), callback_list_.checksum_scn_); } TEST_F(TestTxCallbackList, checksum_leader_tx_end_harder) { TRANS_LOG(INFO, "CASE: checksum_leader_tx_end_harder"); ObMemtable *memtable = create_memtable(); - palf::SCN scn_1; - palf::SCN scn_2; - scn_1.convert_for_lsn_allocator(1); - scn_2.convert_for_lsn_allocator(2); + share::SCN scn_1; + share::SCN scn_2; + scn_1.convert_for_logservice(1); + scn_2.convert_for_logservice(2); create_and_append_callback(memtable, false, /*need_submit_log*/ @@ -695,17 +695,17 @@ TEST_F(TestTxCallbackList, checksum_leader_tx_end_harder) EXPECT_EQ(OB_SUCCESS, callback_list_.tx_calc_checksum_all()); EXPECT_EQ(true, is_checksum_equal(5, checksum_)); - EXPECT_EQ(palf::SCN::max_scn(), callback_list_.checksum_scn_); + EXPECT_EQ(share::SCN::max_scn(), callback_list_.checksum_scn_); } TEST_F(TestTxCallbackList, checksum_leader_tx_end_harderer) { TRANS_LOG(INFO, "CASE: checksum_leader_tx_end_harderer"); ObMemtable *memtable = create_memtable(); - palf::SCN scn_1; - palf::SCN scn_2; - scn_1.convert_for_lsn_allocator(1); - scn_2.convert_for_lsn_allocator(2); + share::SCN scn_1; + share::SCN scn_2; + scn_1.convert_for_logservice(1); + scn_2.convert_for_logservice(2); create_and_append_callback(memtable, false, /*need_submit_log*/ @@ -735,7 +735,7 @@ TEST_F(TestTxCallbackList, checksum_leader_tx_end_harderer) EXPECT_EQ(OB_SUCCESS, callback_list_.tx_calc_checksum_all()); EXPECT_EQ(true, is_checksum_equal(6, checksum_)); - EXPECT_EQ(palf::SCN::max_scn(), callback_list_.checksum_scn_); + EXPECT_EQ(share::SCN::max_scn(), callback_list_.checksum_scn_); } TEST_F(TestTxCallbackList, checksum_remove_memtable_and_tx_end) @@ -744,12 +744,12 @@ TEST_F(TestTxCallbackList, checksum_remove_memtable_and_tx_end) ObMemtable *memtable1 = create_memtable(); ObMemtable *memtable2 = create_memtable(); ObMemtable *memtable3 = create_memtable(); - palf::SCN scn_1; - palf::SCN scn_2; - palf::SCN scn_3; - scn_1.convert_for_lsn_allocator(1); - scn_2.convert_for_lsn_allocator(2); - scn_3.convert_for_lsn_allocator(3); + share::SCN scn_1; + share::SCN scn_2; + share::SCN scn_3; + scn_1.convert_for_logservice(1); + scn_2.convert_for_logservice(2); + scn_3.convert_for_logservice(3); create_and_append_callback(memtable1, false, /*need_submit_log*/ @@ -802,7 +802,7 @@ TEST_F(TestTxCallbackList, checksum_remove_memtable_and_tx_end) EXPECT_EQ(OB_SUCCESS, callback_list_.tx_calc_checksum_all()); EXPECT_EQ(true, is_checksum_equal(9, checksum_)); - EXPECT_EQ(palf::SCN::max_scn(), callback_list_.checksum_scn_); + EXPECT_EQ(share::SCN::max_scn(), callback_list_.checksum_scn_); } @@ -812,14 +812,14 @@ TEST_F(TestTxCallbackList, checksum_fast_commit_and_tx_end) ObMemtable *memtable1 = create_memtable(); ObMemtable *memtable2 = create_memtable(); ObMemtable *memtable3 = create_memtable(); - palf::SCN scn_1; - palf::SCN scn_2; - palf::SCN scn_3; - palf::SCN scn_4; - scn_1.convert_for_lsn_allocator(1); - scn_2.convert_for_lsn_allocator(2); - scn_3.convert_for_lsn_allocator(3); - scn_4.convert_for_lsn_allocator(4); + share::SCN scn_1; + share::SCN scn_2; + share::SCN scn_3; + share::SCN scn_4; + scn_1.convert_for_logservice(1); + scn_2.convert_for_logservice(2); + scn_3.convert_for_logservice(3); + scn_4.convert_for_logservice(4); create_and_append_callback(memtable1, false, /*need_submit_log*/ @@ -879,7 +879,7 @@ TEST_F(TestTxCallbackList, checksum_fast_commit_and_tx_end) EXPECT_EQ(OB_SUCCESS, callback_list_.tx_calc_checksum_all()); EXPECT_EQ(true, is_checksum_equal(9, checksum_)); - EXPECT_EQ(palf::SCN::max_scn(), callback_list_.checksum_scn_); + EXPECT_EQ(share::SCN::max_scn(), callback_list_.checksum_scn_); } TEST_F(TestTxCallbackList, checksum_rollback_to_and_tx_end) @@ -888,14 +888,14 @@ TEST_F(TestTxCallbackList, checksum_rollback_to_and_tx_end) ObMemtable *memtable1 = create_memtable(); ObMemtable *memtable2 = create_memtable(); ObMemtable *memtable3 = create_memtable(); - palf::SCN scn_1; - palf::SCN scn_2; - palf::SCN scn_3; - palf::SCN scn_4; - scn_1.convert_for_lsn_allocator(1); - scn_2.convert_for_lsn_allocator(2); - scn_3.convert_for_lsn_allocator(3); - scn_4.convert_for_lsn_allocator(4); + share::SCN scn_1; + share::SCN scn_2; + share::SCN scn_3; + share::SCN scn_4; + scn_1.convert_for_logservice(1); + scn_2.convert_for_logservice(2); + scn_3.convert_for_logservice(3); + scn_4.convert_for_logservice(4); int64_t savepoint0 = get_seq_no(); create_and_append_callback(memtable1, @@ -954,7 +954,7 @@ TEST_F(TestTxCallbackList, checksum_rollback_to_and_tx_end) EXPECT_EQ(OB_SUCCESS, callback_list_.tx_calc_checksum_all()); EXPECT_EQ(true, is_checksum_equal(5, checksum_)); - EXPECT_EQ(palf::SCN::max_scn(), callback_list_.checksum_scn_); + EXPECT_EQ(share::SCN::max_scn(), callback_list_.checksum_scn_); } TEST_F(TestTxCallbackList, checksum_all_and_tx_end_test) { @@ -994,7 +994,7 @@ TEST_F(TestTxCallbackList, checksum_all_and_tx_end_test) { i++; it->need_submit_log_ = false; it->need_fill_redo_ = false; - it->scn_.convert_for_lsn_allocator(cur_log); + it->scn_.convert_for_logservice(cur_log); enable = true; need_submit_head = it; my_calculate.add_bit(it->get_seq_no()); @@ -1130,7 +1130,7 @@ int ObTxCallbackList::remove_callbacks_for_fast_commit(bool &has_remove) } else { callback_mgr_.add_fast_commit_callback_remove_cnt(functor.get_remove_cnt()); ensure_checksum_(functor.get_checksum_last_scn()); - has_remove = palf::SCN::min_scn() != functor.get_checksum_last_scn(); + has_remove = share::SCN::min_scn() != functor.get_checksum_last_scn(); if (has_remove) { TRANS_LOG(INFO, "remove callbacks for fast commit", K(functor), K(*this)); } diff --git a/unittest/storage/memtable/test_memtable_basic.cpp b/unittest/storage/memtable/test_memtable_basic.cpp index f1a29d859..62d3b87a4 100644 --- a/unittest/storage/memtable/test_memtable_basic.cpp +++ b/unittest/storage/memtable/test_memtable_basic.cpp @@ -28,7 +28,7 @@ #include "storage/tx/ob_multi_data_source.h" #include "storage/tx/ob_trans_define_v4.h" #include "storage/memtable/mvcc/ob_mvcc_row.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -87,7 +87,7 @@ int ObTxTableGuard::init(ObTxTable *tx_table) namespace memtable { -int ObMvccRow::check_double_insert_(const palf::SCN , +int ObMvccRow::check_double_insert_(const share::SCN , ObMvccTransNode &, ObMvccTransNode *) { @@ -116,8 +116,8 @@ public: ObITable::TableKey table_key; table_key.table_type_ = ObITable::DATA_MEMTABLE; table_key.tablet_id_ = ObTabletID(tablet_id_.id()); - table_key.scn_range_.start_scn_ = palf::SCN::base_scn(); - table_key.scn_range_.end_scn_ = palf::SCN::max_scn(); + table_key.scn_range_.start_scn_ = share::SCN::base_scn(); + table_key.scn_range_.end_scn_ = share::SCN::max_scn(); int64_t schema_version = 1; uint32_t freeze_clock = 0; @@ -254,7 +254,7 @@ public: if (trans_node->is_aborted()) { trans_node = trans_node->prev_; } else if (trans_node->is_committed()) { - if (trans_node->trans_version_.get_val_for_lsn_allocator() <= snapshot) { + if (trans_node->trans_version_.get_val_for_logservice() <= snapshot) { break; } else { trans_node = trans_node->prev_; @@ -264,7 +264,7 @@ public: //if (trans_node->seq_no__ <= snapshot) { break; } else { - if (snapshot < trans_node->trans_version_.get_val_for_lsn_allocator()) { + if (snapshot < trans_node->trans_version_.get_val_for_logservice()) { trans_node = trans_node->prev_; } else { ret = OB_ERR_SHARED_LOCK_CONFLICT; @@ -295,8 +295,8 @@ void print(ObMvccRow *mvcc_row) ObMvccTransNode *node = mvcc_row->get_list_head(); while (node != nullptr) { printf("%p tx_id:%ld trans_version:%ld log_ts:%ld prev:%p next:%p version:%ld\n", - node, node->tx_id_.get_id(), node->trans_version_.get_val_for_lsn_allocator(), - node->scn_.get_val_for_lsn_allocator(), node->prev_, node->next_, node->version_); + node, node->tx_id_.get_id(), node->trans_version_.get_val_for_logservice(), + node->scn_.get_val_for_logservice(), node->prev_, node->next_, node->version_); node = node->prev_; } printf("\n"); @@ -328,8 +328,8 @@ TEST_F(TestMemtable, mt_set) print(mvcc_row); EXPECT_EQ(2, rg.mem_ctx_.trans_mgr_.get_main_list_length()); - palf::SCN val_1000; - val_1000.convert_for_lsn_allocator(1000); + share::SCN val_1000; + val_1000.convert_for_logservice(1000); EXPECT_EQ(OB_SUCCESS, rg.mem_ctx_.do_trans_end(true, val_1000, val_1000, 0)); print(mvcc_row); } @@ -349,16 +349,16 @@ TEST_F(TestMemtable, conflict) EXPECT_EQ(OB_SUCCESS, rg2.init(2, this)); EXPECT_EQ(OB_ERR_EXCLUSIVE_LOCK_CONFLICT, rg2.write(1, 3, mt)); - palf::SCN val_1000; - val_1000.convert_for_lsn_allocator(1000); + share::SCN val_1000; + val_1000.convert_for_logservice(1000); EXPECT_EQ(OB_SUCCESS, rg.mem_ctx_.do_trans_end(true, val_1000, val_1000, 0)); EXPECT_EQ(OB_TRANSACTION_SET_VIOLATION, rg2.write(1, 3, mt, 900)); EXPECT_EQ(OB_SUCCESS, rg2.write(1, 3, mt, 1000)); EXPECT_EQ(OB_SUCCESS, rg2.write(1, 4, mt, 1001)); - palf::SCN val_1002; - val_1002.convert_for_lsn_allocator(1002); + share::SCN val_1002; + val_1002.convert_for_logservice(1002); EXPECT_EQ(OB_SUCCESS, rg2.mem_ctx_.do_trans_end(true, val_1002, val_1002, 0)); print(mvcc_row); @@ -373,8 +373,8 @@ TEST_F(TestMemtable, except) EXPECT_EQ(OB_SUCCESS, rg.init(1, this)); ObMvccRow *mvcc_row = nullptr; - palf::SCN val_900; - val_900.convert_for_lsn_allocator(900); + share::SCN val_900; + val_900.convert_for_logservice(900); EXPECT_EQ(OB_SUCCESS, rg.write(1, 2, mt, mvcc_row, 1000)); EXPECT_EQ(OB_SUCCESS, rg.mem_ctx_.do_trans_end(true, val_900, val_900, 0)); @@ -403,8 +403,8 @@ TEST_F(TestMemtable, multi_key) RunCtxGuard rg; EXPECT_EQ(OB_SUCCESS, rg.init(1, this)); - palf::SCN val_900; - val_900.convert_for_lsn_allocator(900); + share::SCN val_900; + val_900.convert_for_logservice(900); ObMvccRow *mvcc_row = nullptr; ObMvccRow *mvcc_row2 = nullptr; diff --git a/unittest/storage/mock_ob_log_handler.h b/unittest/storage/mock_ob_log_handler.h index 85bd67c29..aeffe9b9c 100644 --- a/unittest/storage/mock_ob_log_handler.h +++ b/unittest/storage/mock_ob_log_handler.h @@ -14,7 +14,7 @@ #include "logservice/palf/palf_handle.h" #include "logservice/ob_log_service.h" #include "logservice/palf/palf_env.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #include "logservice/ob_log_base_type.h" #ifndef MOCK_OB_LOG_HANDLER_H_ @@ -34,11 +34,11 @@ public: virtual bool is_valid() const { return true; } virtual int append(const void *buffer, const int64_t nbytes, - const palf::SCN &ref_scn, + const share::SCN &ref_scn, const bool need_nonblock, logservice::AppendCb *cb, palf::LSN &lsn, - palf::SCN &scn) + share::SCN &scn) { UNUSED(need_nonblock); UNUSED(buffer); @@ -84,7 +84,7 @@ public: virtual int change_access_mode(const int64_t mode_version, const AccessMode &access_mode, - const palf::SCN &ref_scn) + const share::SCN &ref_scn) { UNUSED(mode_version); UNUSED(access_mode); @@ -118,7 +118,7 @@ public: UNUSED(iter); return OB_SUCCESS; }; - int seek(const palf::SCN &start_scn, + int seek(const share::SCN &start_scn, palf::PalfGroupBufferIterator &iter) { UNUSED(start_scn); @@ -146,7 +146,7 @@ public: UNUSEDx(member_list, arb_replica, paxos_replica_num); return OB_SUCCESS; } - int get_end_scn(palf::SCN &scn) const + int get_end_scn(share::SCN &scn) const { UNUSED(scn); return OB_SUCCESS; @@ -172,7 +172,7 @@ public: UNUSED(lsn); return OB_SUCCESS; } - int get_max_scn(palf::SCN &scn) const + int get_max_scn(share::SCN &scn) const { UNUSED(scn); return OB_SUCCESS; @@ -182,7 +182,7 @@ public: UNUSED(ts_ns); return OB_SUCCESS; } - int locate_by_scn_coarsely(const palf::SCN &scn, LSN &result_lsn) + int locate_by_scn_coarsely(const share::SCN &scn, LSN &result_lsn) { LSN tmp(scn.get_val_for_inner_table_field()); result_lsn = tmp; @@ -201,7 +201,7 @@ public: return OB_SUCCESS; } - int locate_by_lsn_coarsely(const palf::LSN &lsn, palf::SCN &result_scn) + int locate_by_lsn_coarsely(const palf::LSN &lsn, share::SCN &result_scn) { result_scn = result_scn_; return OB_SUCCESS; @@ -375,9 +375,9 @@ public: LSN base_lsn_; int64_t result_ts_ns_; - palf::SCN result_scn_; + share::SCN result_scn_; int enable_replay(const palf::LSN &initial_lsn, - const palf::SCN &initial_scn) + const share::SCN &initial_scn) { UNUSED(initial_lsn); UNUSED(initial_scn); @@ -394,7 +394,7 @@ public: { return OB_SUCCESS; } - int get_max_decided_scn(palf::SCN &scn) + int get_max_decided_scn(share::SCN &scn) { scn.set_max(); return OB_SUCCESS; diff --git a/unittest/storage/mock_ob_meta_report.h b/unittest/storage/mock_ob_meta_report.h index 3127a214c..653017f63 100644 --- a/unittest/storage/mock_ob_meta_report.h +++ b/unittest/storage/mock_ob_meta_report.h @@ -26,9 +26,6 @@ public: MOCK_METHOD3(submit_tablet_update_task, int(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObTabletID &tablet_id)); - MOCK_METHOD3(submit_tablet_checksums_task, int(const uint64_t tenant_id, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id)); }; } diff --git a/unittest/storage/test_compaction_policy.cpp b/unittest/storage/test_compaction_policy.cpp index 73c544a25..a19d5eb2b 100644 --- a/unittest/storage/test_compaction_policy.cpp +++ b/unittest/storage/test_compaction_policy.cpp @@ -32,7 +32,7 @@ #include "storage/test_dml_common.h" #include "storage/mockcontainer/mock_ob_iterator.h" #include "storage/slog_ckpt/ob_server_checkpoint_slog_handler.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase @@ -42,6 +42,7 @@ using namespace storage; using namespace blocksstable; using namespace memtable; using namespace share::schema; +using namespace share; using namespace compaction; namespace memtable @@ -283,7 +284,7 @@ int TestCompactionPolicy::mock_memtable( ObITable::TableKey table_key; int64_t end_border = -1; if (0 == end_scn || INT64_MAX == end_scn) { - end_border = palf::OB_MAX_SCN_TS_NS; + end_border = OB_MAX_SCN_TS_NS; } else { end_border = end_scn; } @@ -304,8 +305,8 @@ int TestCompactionPolicy::mock_memtable( } else if (OB_FAIL(memtable->add_to_data_checkpoint(mt_mgr->freezer_->get_data_checkpoint()))) { LOG_WARN("add to data_checkpoint failed", K(ret), KPC(memtable)); mt_mgr->clean_tail_memtable_(); - } else if (palf::OB_MAX_SCN_TS_NS != end_border) { // frozen memtable - palf::SCN snapshot_scn; + } else if (OB_MAX_SCN_TS_NS != end_border) { // frozen memtable + SCN snapshot_scn; snapshot_scn.convert_for_tx(snapshot_version); memtable->snapshot_version_ = snapshot_scn; memtable->write_ref_cnt_ = 0; @@ -357,11 +358,11 @@ int TestCompactionPolicy::mock_tablet( LOG_WARN("failed to acquire tablet", K(ret), K(key)); } else if (FALSE_IT(tablet = tablet_handle.get_obj())) { } else if (OB_FAIL(tablet->init(ls_id, tablet_id, tablet_id, empty_tablet_id, empty_tablet_id, - palf::SCN::min_scn(), snapshot_version, table_schema, compat_mode, table_store_flag, table_handle, ls_handle.get_ls()->get_freezer()))) { + SCN::min_scn(), snapshot_version, table_schema, compat_mode, table_store_flag, table_handle, ls_handle.get_ls()->get_freezer()))) { LOG_WARN("failed to init tablet", K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(table_schema), K(compat_mode)); } else { - tablet->tablet_meta_.clog_checkpoint_scn_.convert_for_lsn_allocator(clog_checkpoint_ts); + tablet->tablet_meta_.clog_checkpoint_scn_.convert_for_logservice(clog_checkpoint_ts); tablet->tablet_meta_.snapshot_version_ = snapshot_version; } return ret; @@ -535,7 +536,7 @@ int TestCompactionPolicy::prepare_freeze_info( int ret = OB_SUCCESS; ObTenantFreezeInfoMgr *mgr = MTL(ObTenantFreezeInfoMgr *); bool changed = false; - palf::SCN min_major_snapshot = palf::SCN::max_scn(); + SCN min_major_snapshot = SCN::max_scn(); if (OB_ISNULL(mgr)) { ret = OB_ERR_UNEXPECTED; @@ -726,7 +727,7 @@ TEST_F(TestCompactionPolicy, check_mini_merge_basic) ASSERT_EQ(OB_SUCCESS, ret); ASSERT_EQ(3, result.handle_.get_count()); - tablet_handle_.get_obj()->tablet_meta_.clog_checkpoint_scn_.convert_for_lsn_allocator(300); + tablet_handle_.get_obj()->tablet_meta_.clog_checkpoint_scn_.convert_for_logservice(300); tablet_handle_.get_obj()->tablet_meta_.snapshot_version_ = 300; result.reset(); ret = ObPartitionMergePolicy::get_mini_merge_tables(param, 0, *tablet_handle_.get_obj(), result); @@ -751,7 +752,7 @@ TEST_F(TestCompactionPolicy, check_minor_merge_basic) common::ObArray freeze_info; common::ObArray snapshots; - palf::SCN scn; + SCN scn; scn.convert_for_tx(1); ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); @@ -786,7 +787,7 @@ TEST_F(TestCompactionPolicy, check_no_need_minor_merge) common::ObArray freeze_info; common::ObArray snapshots; - palf::SCN scn; + SCN scn; scn.convert_for_tx(1); ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); scn.convert_for_tx(320); @@ -825,7 +826,7 @@ TEST_F(TestCompactionPolicy, check_major_merge_basic) common::ObArray freeze_info; common::ObArray snapshots; - palf::SCN scn; + SCN scn; scn.convert_for_tx(1); ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); scn.convert_for_tx(340); @@ -863,7 +864,7 @@ TEST_F(TestCompactionPolicy, check_no_need_major_merge) common::ObArray freeze_info; common::ObArray snapshots; - palf::SCN scn; + SCN scn; scn.convert_for_tx(1); ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); scn.convert_for_tx(340); diff --git a/unittest/storage/test_dml_common.h b/unittest/storage/test_dml_common.h index 8e24eb253..8c7ba4fff 100644 --- a/unittest/storage/test_dml_common.h +++ b/unittest/storage/test_dml_common.h @@ -218,16 +218,16 @@ int TestDmlCommon::create_data_tablet( } else { transaction::ObMulSourceDataNotifyArg trans_flags; trans_flags.tx_id_ = 123; - trans_flags.scn_ = palf::SCN::invalid_scn(); + trans_flags.scn_ = share::SCN::invalid_scn(); trans_flags.for_replay_ = false; ObLS *ls = ls_handle.get_ls(); if (OB_FAIL(ls->get_tablet_svr()->on_prepare_create_tablets(arg, trans_flags))) { STORAGE_LOG(WARN, "failed to prepare create tablets", K(ret), K(arg)); - } else if (FALSE_IT(trans_flags.scn_ = palf::SCN::minus(palf::SCN::max_scn(), 100))) { + } else if (FALSE_IT(trans_flags.scn_ = share::SCN::minus(share::SCN::max_scn(), 100))) { } else if (OB_FAIL(ls->get_tablet_svr()->on_redo_create_tablets(arg, trans_flags))) { STORAGE_LOG(WARN, "failed to redo create tablets", K(ret), K(arg)); - } else if (FALSE_IT(trans_flags.scn_ = palf::SCN::plus(trans_flags.scn_, 1))) { + } else if (FALSE_IT(trans_flags.scn_ = share::SCN::plus(trans_flags.scn_, 1))) { } else if (OB_FAIL(ls->get_tablet_svr()->on_commit_create_tablets(arg, trans_flags))) { STORAGE_LOG(WARN, "failed to commit create tablets", K(ret), K(arg)); } @@ -246,15 +246,15 @@ int TestDmlCommon::create_data_and_index_tablets( ObLSHandle ls_handle; obrpc::ObBatchCreateTabletArg arg; - palf::SCN log_scn; + share::SCN log_scn; if (OB_FAIL(create_ls(tenant_id, ls_id, ls_handle))) { STORAGE_LOG(WARN, "failed to create ls", K(ret), K(tenant_id), K(ls_id)); } else if (OB_FAIL(build_mixed_tablets_arg(tenant_id, ls_id, data_tablet_id, index_tablet_id_array, arg))) { STORAGE_LOG(WARN, "failed to build pure data tablet arg", K(ret), K(tenant_id), K(ls_id), K(data_tablet_id), K(index_tablet_id_array)); - } else if (OB_FAIL(log_scn.convert_for_lsn_allocator(1))) { - STORAGE_LOG(WARN, "failed to convert_for_lsn_allocator", K(ret)); + } else if (OB_FAIL(log_scn.convert_for_logservice(1))) { + STORAGE_LOG(WARN, "failed to convert_for_logservice", K(ret)); } else if (OB_FAIL(ls_handle.get_ls()->batch_create_tablets(arg, log_scn, true/*is_replay*/))) { STORAGE_LOG(WARN, "failed to batch create tablets", K(ret), K(arg)); @@ -591,7 +591,7 @@ int TestDmlCommon::build_pure_data_tablet_arg( } else if (OB_FAIL(tablet_info.init(tablet_id_array, data_tablet_id, tablet_schema_index_array, lib::Worker::CompatMode::MYSQL, false))) { STORAGE_LOG(WARN, "failed to init tablet info", K(ret), K(tablet_id_array), K(data_tablet_id), K(tablet_schema_index_array)); - } else if (OB_FAIL(arg.init_create_tablet(ls_id, palf::SCN::min_scn()))) { + } else if (OB_FAIL(arg.init_create_tablet(ls_id, share::SCN::min_scn()))) { STORAGE_LOG(WARN, "failed to init create tablet", K(ret), K(tenant_id), K(ls_id)); } else if (OB_FAIL(arg.table_schemas_.push_back(table_schema))) { STORAGE_LOG(WARN, "failed to push back table schema", K(ret), K(table_schema)); @@ -643,7 +643,7 @@ int TestDmlCommon::build_mixed_tablets_arg( } else if (OB_FAIL(tablet_info.init(tablet_id_array, data_tablet_id, tablet_schema_index_array, lib::Worker::CompatMode::MYSQL, false))) { STORAGE_LOG(WARN, "failed to init tablet info", K(ret), K(tablet_id_array), K(data_tablet_id), K(tablet_schema_index_array)); - } else if (OB_FAIL(arg.init_create_tablet(ls_id, palf::SCN::min_scn()))) { + } else if (OB_FAIL(arg.init_create_tablet(ls_id, share::SCN::min_scn()))) { STORAGE_LOG(WARN, "failed to init create tablet", K(ret), K(tenant_id), K(ls_id)); } else if (OB_FAIL(arg.table_schemas_.push_back(data_table_schema))) { STORAGE_LOG(WARN, "failed to push back data table schema", K(ret), K(data_table_schema)); diff --git a/unittest/storage/test_io_manager.cpp b/unittest/storage/test_io_manager.cpp index 48883882b..9f88be1ac 100644 --- a/unittest/storage/test_io_manager.cpp +++ b/unittest/storage/test_io_manager.cpp @@ -560,7 +560,8 @@ TEST_F(TestIOStruct, IOCallbackManager) ASSERT_FAIL(callback_mgr.enqueue_callback(req)); char buf[32] = "test"; req.io_buf_ = buf; - req.copied_callback_ = new (req.callback_buf_) TestIOCallback(); + char callback_buf_[ObIOCallback::CALLBACK_BUF_SIZE] __attribute__ ((aligned (16))); + req.copied_callback_ = new (callback_buf_) TestIOCallback(); // ObIOManager::get_instance().io_config_ = ObIOConfig::default_config(); ASSERT_SUCC(callback_mgr.enqueue_callback(req)); diff --git a/unittest/storage/test_lob_common.h b/unittest/storage/test_lob_common.h index 27fddab4d..4c93066ca 100644 --- a/unittest/storage/test_lob_common.h +++ b/unittest/storage/test_lob_common.h @@ -77,16 +77,16 @@ int TestLobCommon::create_data_tablet( } else { transaction::ObMulSourceDataNotifyArg trans_flags; trans_flags.tx_id_ = 123; - trans_flags.scn_ = palf::SCN::invalid_scn(); + trans_flags.scn_ = share::SCN::invalid_scn(); trans_flags.for_replay_ = false; ObLS *ls = ls_handle.get_ls(); if (OB_FAIL(ls->get_tablet_svr()->on_prepare_create_tablets(arg, trans_flags))) { STORAGE_LOG(WARN, "failed to prepare create tablets", K(ret), K(arg)); - } else if (FALSE_IT(trans_flags.scn_ = palf::SCN::minus(palf::SCN::max_scn(), 100))) { + } else if (FALSE_IT(trans_flags.scn_ = share::SCN::minus(share::SCN::max_scn(), 100))) { } else if (OB_FAIL(ls->get_tablet_svr()->on_redo_create_tablets(arg, trans_flags))) { STORAGE_LOG(WARN, "failed to redo create tablets", K(ret), K(arg)); - } else if (FALSE_IT(trans_flags.scn_ = palf::SCN::plus(trans_flags.scn_, 1))) { + } else if (FALSE_IT(trans_flags.scn_ = share::SCN::plus(trans_flags.scn_, 1))) { } else if (OB_FAIL(ls->get_tablet_svr()->on_commit_create_tablets(arg, trans_flags))) { STORAGE_LOG(WARN, "failed to commit create tablets", K(ret), K(arg)); } @@ -332,7 +332,7 @@ int TestLobCommon::build_lob_tablet_arg( lib::get_compat_mode(), false/*is_create_bind_hidden_tablets*/))) { STORAGE_LOG(WARN, "failed to init tablet info", K(ret), K(tablet_id_array), K(data_tablet_id), K(tablet_schema_index_array)); - } else if (OB_FAIL(arg.init_create_tablet(ls_id, palf::SCN::min_scn()))) { + } else if (OB_FAIL(arg.init_create_tablet(ls_id, share::SCN::min_scn()))) { STORAGE_LOG(WARN, "failed to init create tablet", K(ret), K(tenant_id), K(ls_id)); } else if (OB_FAIL(arg.table_schemas_.push_back(table_schema))) { STORAGE_LOG(WARN, "failed to push back table schema", K(ret), K(table_schema)); diff --git a/unittest/storage/test_sstable_log_ts_range_cut.cpp b/unittest/storage/test_sstable_log_ts_range_cut.cpp index 66cb68f09..198612433 100644 --- a/unittest/storage/test_sstable_log_ts_range_cut.cpp +++ b/unittest/storage/test_sstable_log_ts_range_cut.cpp @@ -20,7 +20,7 @@ #include "storage/tablet/ob_tablet_table_store.h" #include "storage/ob_i_table.h" #include "storage/ob_storage_struct.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { diff --git a/unittest/storage/test_tablet_helper.h b/unittest/storage/test_tablet_helper.h index ab2bac2f8..6a81e0aef 100644 --- a/unittest/storage/test_tablet_helper.h +++ b/unittest/storage/test_tablet_helper.h @@ -18,7 +18,7 @@ #include "share/ob_rpc_struct.h" #include "storage/tx/ob_trans_define.h" #include "storage/ls/ob_ls_tablet_service.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" namespace oceanbase { @@ -36,15 +36,15 @@ int TestTabletHelper::create_tablet(ObLSTabletService &ls_tablet_svr, obrpc::ObB int ret = common::OB_SUCCESS; transaction::ObMulSourceDataNotifyArg trans_flags; trans_flags.tx_id_ = 123; - trans_flags.scn_ = palf::SCN::invalid_scn(); + trans_flags.scn_ = share::SCN::invalid_scn(); trans_flags.for_replay_ = false; if (OB_FAIL(ls_tablet_svr.on_prepare_create_tablets(arg, trans_flags))) { STORAGE_LOG(WARN, "failed to prepare create tablets", K(ret), K(arg)); - } else if (FALSE_IT(trans_flags.scn_ = palf::SCN::minus(palf::SCN::max_scn(), 100))) { + } else if (FALSE_IT(trans_flags.scn_ = share::SCN::minus(share::SCN::max_scn(), 100))) { } else if (OB_FAIL(ls_tablet_svr.on_redo_create_tablets(arg, trans_flags))) { STORAGE_LOG(WARN, "failed to redo create tablets", K(ret), K(arg)); - } else if (FALSE_IT(trans_flags.scn_ = palf::SCN::plus(trans_flags.scn_, 1))) { + } else if (FALSE_IT(trans_flags.scn_ = share::SCN::plus(trans_flags.scn_, 1))) { } else if (OB_FAIL(ls_tablet_svr.on_commit_create_tablets(arg, trans_flags))) { STORAGE_LOG(WARN, "failed to commit create tablets", K(ret), K(arg)); } diff --git a/unittest/storage/tx/it/tx_node.cpp b/unittest/storage/tx/it/tx_node.cpp index 15b57edfc..0c403864e 100644 --- a/unittest/storage/tx/it/tx_node.cpp +++ b/unittest/storage/tx/it/tx_node.cpp @@ -12,7 +12,7 @@ #define USING_LOG_PREFIX TRANS #include "tx_node.h" -#include "logservice/palf/scn.h" +#include "share/scn.h" #define FAST_FAIL() \ do { \ if (OB_FAIL(ret)) { \ diff --git a/unittest/storage/tx/mock_utils/basic_fake_define.h b/unittest/storage/tx/mock_utils/basic_fake_define.h index f64d0831d..60626836e 100644 --- a/unittest/storage/tx/mock_utils/basic_fake_define.h +++ b/unittest/storage/tx/mock_utils/basic_fake_define.h @@ -215,7 +215,7 @@ public: int get_gts(const uint64_t tenant_id, const MonotonicTs stc, ObTsCbTask *task, - palf::SCN &scn, + share::SCN &scn, MonotonicTs &receive_gts_ts) { int ret = OB_SUCCESS; @@ -235,13 +235,13 @@ public: return ret; } - int get_gts(const uint64_t tenant_id, ObTsCbTask *task, palf::SCN &scn) { + int get_gts(const uint64_t tenant_id, ObTsCbTask *task, share::SCN &scn) { if (get_gts_error_) { return get_gts_error_; } return OB_SUCCESS; } int get_ts_sync(const uint64_t tenant_id, const int64_t timeout_ts, - palf::SCN &scn, bool &is_external_consistent) { return OB_SUCCESS; } - int wait_gts_elapse(const uint64_t tenant_id, const palf::SCN &scn, ObTsCbTask *task, + share::SCN &scn, bool &is_external_consistent) { return OB_SUCCESS; } + int wait_gts_elapse(const uint64_t tenant_id, const share::SCN &scn, ObTsCbTask *task, bool &need_wait) { TRANS_LOG(INFO, "wait_gts_elapse begin", K(gts_), K(scn)); @@ -257,7 +257,7 @@ public: return ret; } - int wait_gts_elapse(const uint64_t tenant_id, const palf::SCN &scn) + int wait_gts_elapse(const uint64_t tenant_id, const share::SCN &scn) { int ret = OB_SUCCESS; if (scn.get_val_for_gts() > gts_) { @@ -296,7 +296,7 @@ public: ObLink *task = elapse_queue_.pop(); if (task) { const MonotonicTs srr(MonotonicTs::current_time()); - palf::SCN ts; + share::SCN ts; ts.convert_for_gts(gts_); const MonotonicTs receive_gts_ts(gts_); const ObGTSCacheTaskType task_type = WAIT_GTS_ELAPSING; @@ -313,7 +313,7 @@ public: ObLink *task = get_gts_waiting_queue_.pop(); if (task) { const MonotonicTs srr(MonotonicTs::current_time()); - palf::SCN ts; + share::SCN ts; ts.convert_for_gts(gts_); const MonotonicTs receive_gts_ts(gts_); const ObGTSCacheTaskType task_type = GET_GTS; @@ -420,7 +420,7 @@ public: int submit_log(const char *buf, const int64_t size, - const palf::SCN base_ts, + const share::SCN base_ts, ObTxBaseLogCb *cb, const bool need_nonblock) { @@ -432,7 +432,7 @@ public: } else { const int64_t replay_hint = base_header.get_replay_hint(); const int64_t ts = MAX(base_ts.get_val_for_gts(), ts_) + 1; - palf::SCN scn; + share::SCN scn; scn.convert_for_gts(ts); int64_t queue_idx = replay_hint % TASK_QUEUE_CNT; if (!ATOMIC_LOAD(&log_drop_)) { @@ -516,7 +516,7 @@ public: ObFakeTxReplayExecutor(storage::ObLS *ls, storage::ObLSTxService *ls_tx_srv, const palf::LSN &lsn, - const palf::SCN &log_timestamp) + const share::SCN &log_timestamp) : ObTxReplayExecutor(ls, ls_tx_srv, lsn, log_timestamp) {memtable_ = nullptr;} ~ObFakeTxReplayExecutor() {} @@ -534,7 +534,7 @@ public: memtable::ObMemtable* memtable) { int ret = OB_SUCCESS; - palf::SCN log_scn; + share::SCN log_scn; log_scn.convert_for_gts(log_timestamp); ObFakeTxReplayExecutor replay_executor(ls, ls_tx_srv, lsn, log_scn); if (OB_ISNULL(ls) || OB_ISNULL(ls_tx_srv) || OB_ISNULL(buf) || size <= 0 @@ -602,8 +602,8 @@ class ObFakeLSTxService : public ObLSTxService public: ObFakeLSTxService(ObLS *parent) : ObLSTxService(parent) { } ~ObFakeLSTxService() {} - virtual palf::SCN get_ls_weak_read_ts() override { - return palf::SCN::min_scn(); + virtual share::SCN get_ls_weak_read_ts() override { + return share::SCN::min_scn(); } }; diff --git a/unittest/storage/tx/ob_mock_2pc_ctx.h b/unittest/storage/tx/ob_mock_2pc_ctx.h index 07e18e0eb..9712ec76d 100644 --- a/unittest/storage/tx/ob_mock_2pc_ctx.h +++ b/unittest/storage/tx/ob_mock_2pc_ctx.h @@ -54,7 +54,6 @@ public: // transaction committer, ObTxCycleTwoPhaseCommitter and ObTxOnePhaseCommitter based on // participants number class MockOb2pcCtx : public ObTxCycleTwoPhaseCommitter, - public ObTxOnePhaseCommitter, public ObMailHandler { public: diff --git a/unittest/storage/tx/ob_mock_tx_ctx.cpp b/unittest/storage/tx/ob_mock_tx_ctx.cpp index e0f1575db..a7879dea1 100644 --- a/unittest/storage/tx/ob_mock_tx_ctx.cpp +++ b/unittest/storage/tx/ob_mock_tx_ctx.cpp @@ -400,7 +400,7 @@ int MockObTxCtx::handle_all() return mailbox_.handle_all(); } -int MockObTxCtx::get_gts_(palf::SCN >s) +int MockObTxCtx::get_gts_(share::SCN >s) { // TODO(handora.qc): get gts failed gts.convert_for_gts(ObTimeUtility::current_time_ns()); @@ -413,13 +413,13 @@ int MockObTxCtx::wait_gts_elapse_commit_version_(bool &need_wait) return OB_SUCCESS; } -int MockObTxCtx::get_local_max_read_version_(palf::SCN &local_max_read_version) +int MockObTxCtx::get_local_max_read_version_(share::SCN &local_max_read_version) { local_max_read_version.convert_for_gts(ObTimeUtility::current_time()); return OB_SUCCESS; } -int MockObTxCtx::update_local_max_commit_version_(const palf::SCN &) +int MockObTxCtx::update_local_max_commit_version_(const share::SCN &) { return OB_SUCCESS; } diff --git a/unittest/storage/tx/ob_mock_tx_ctx.h b/unittest/storage/tx/ob_mock_tx_ctx.h index 8898fe88d..fccc70e50 100644 --- a/unittest/storage/tx/ob_mock_tx_ctx.h +++ b/unittest/storage/tx/ob_mock_tx_ctx.h @@ -60,10 +60,10 @@ public: protected: virtual int post_msg_(const share::ObLSID &receiver, ObTxMsg &msg) override; virtual int post_msg_(const ObAddr &receiver, ObTxMsg &msg) override; - virtual int get_gts_(palf::SCN >s) override; + virtual int get_gts_(share::SCN >s) override; virtual int wait_gts_elapse_commit_version_(bool &need_wait) override; - virtual int get_local_max_read_version_(palf::SCN &local_max_read_version) override; - virtual int update_local_max_commit_version_(const palf::SCN &) override; + virtual int get_local_max_read_version_(share::SCN &local_max_read_version) override; + virtual int update_local_max_commit_version_(const share::SCN &) override; private: MockObLogQueue log_queue_; int64_t addr_; diff --git a/unittest/storage/tx/ob_mock_tx_log_adapter.cpp b/unittest/storage/tx/ob_mock_tx_log_adapter.cpp index 81a1f86ef..dbff2789c 100644 --- a/unittest/storage/tx/ob_mock_tx_log_adapter.cpp +++ b/unittest/storage/tx/ob_mock_tx_log_adapter.cpp @@ -103,14 +103,14 @@ void MockTxLogAdapter::handle(void *task) int MockTxLogAdapter::submit_log(const char *buf, const int64_t size, - const palf::SCN base_ts, + const share::SCN base_ts, ObTxBaseLogCb *cb, const bool need_nonblock) { int ret = OB_SUCCESS; int64_t ts = 0; palf::LSN lsn; - palf::SCN scn; + share::SCN scn; UNUSED(need_nonblock); if (ATOMIC_LOAD(&is_running_)) { diff --git a/unittest/storage/tx/ob_mock_tx_log_adapter.h b/unittest/storage/tx/ob_mock_tx_log_adapter.h index 9d1491d34..444fab6e7 100644 --- a/unittest/storage/tx/ob_mock_tx_log_adapter.h +++ b/unittest/storage/tx/ob_mock_tx_log_adapter.h @@ -60,7 +60,7 @@ public: public: int submit_log(const char *buf, const int64_t size, - const palf::SCN base_ts, + const share::SCN base_ts, ObTxBaseLogCb *cb, const bool need_block); int get_role(bool &is_leader, int64_t &epoch); diff --git a/unittest/storage/tx/test_ls_log_writer.cpp b/unittest/storage/tx/test_ls_log_writer.cpp index 2092e332c..a0051731e 100644 --- a/unittest/storage/tx/test_ls_log_writer.cpp +++ b/unittest/storage/tx/test_ls_log_writer.cpp @@ -54,7 +54,7 @@ TEST_F(TestLSLogWriter, submit_start_working_log) ObTxLogBlock replay_block; int64_t replay_hint = 0; - palf::SCN log_ts; + share::SCN log_ts; std::string log_string; ObTxLogHeader log_header; ObTxStartWorkingLogTempRef tmp_ref; diff --git a/unittest/storage/tx/test_ob_id_meta.cpp b/unittest/storage/tx/test_ob_id_meta.cpp index e22f13114..a8673fb00 100644 --- a/unittest/storage/tx/test_ob_id_meta.cpp +++ b/unittest/storage/tx/test_ob_id_meta.cpp @@ -40,7 +40,7 @@ public: void init() { for (int i=0; iparts_[0].id_, msg1.tx_ptr_->parts_[0].id_); + if (OB_NOT_NULL(msg.tx_ptr_)) { + msg.tx_ptr_ = NULL; + } } TEST_F(TestObTxMsg, trans_keepalive_msg) diff --git a/unittest/storage/tx/test_simple_tx_ctx.cpp b/unittest/storage/tx/test_simple_tx_ctx.cpp index 9cb5e44e9..bd950cc85 100644 --- a/unittest/storage/tx/test_simple_tx_ctx.cpp +++ b/unittest/storage/tx/test_simple_tx_ctx.cpp @@ -12,6 +12,8 @@ #include #include +#define private public +#define protected public #include "storage/tx/ob_mock_tx_ctx.h" namespace oceanbase @@ -105,30 +107,48 @@ TEST_F(TestMockObTxCtx, test_simple_tx_ctx1) EXPECT_EQ(OB_SUCCESS, build_scheduler_mailbox()); - ObTxCommitMsg tx_commit_msg; - MockObTxCtx::mock_tx_commit_msg(trans_id1, - ls_id1, - participants, - tx_commit_msg); - ObMail tx_commit_mail; - EXPECT_EQ(OB_SUCCESS, tx_commit_mail.init(scheduler_addr_ /*from*/, - ctx1.get_mailbox_addr() /*to*/, - sizeof(ObTxCommitMsg), - tx_commit_msg)); + ctx1.addr_memo_[ls_id2] = ctx2.addr_; + ctx1.ls_memo_[ctx2.addr_] = ls_id2; + ctx1.set_trans_type_(TransType::DIST_TRANS); + ctx1.upstream_state_ = ObTxState::INIT; + ctx1.set_downstream_state(ObTxState::REDO_COMPLETE); + ctx1.exec_info_.participants_.push_back(ls_id1); + ctx1.exec_info_.participants_.push_back(ls_id2); + + ctx2.addr_memo_[ls_id1] = ctx1.addr_; + ctx2.ls_memo_[ctx1.addr_] = ls_id1; + ctx2.set_trans_type_(TransType::DIST_TRANS); + ctx2.upstream_state_ = ObTxState::PREPARE; + ctx2.exec_info_.upstream_ = ls_id1; + ctx2.log_queue_.push_back(ObTwoPhaseCommitLogType::OB_LOG_TX_PREPARE); + bool unused; + EXPECT_EQ(OB_SUCCESS, ctx2.do_prepare(unused)); + EXPECT_EQ(OB_SUCCESS, ctx2.apply()); + EXPECT_EQ(OB_SUCCESS, ctx2.handle_timeout(100000)); + + EXPECT_EQ(OB_SUCCESS, ctx1.handle()); + EXPECT_EQ(OB_SUCCESS, ctx1.two_phase_commit()); + + EXPECT_EQ(OB_SUCCESS, ctx2.handle()); + + EXPECT_EQ(OB_SUCCESS, ctx1.handle()); + EXPECT_EQ(OB_SUCCESS, ctx1.apply()); + + EXPECT_NE(share::SCN::max_scn(), ctx1.mt_ctx_.trans_version_); // ========== Two Phase Commit prepare Phase ========== // ctx1 start to commit - mailbox_mgr_.send_to_head(tx_commit_mail, tx_commit_mail.to_); - EXPECT_EQ(OB_SUCCESS, ctx1.handle()); + // mailbox_mgr_.send_to_head(tx_commit_mail, tx_commit_mail.to_); + // EXPECT_EQ(OB_SUCCESS, ctx1.handle()); - // ctx2 handle prepare request - EXPECT_EQ(OB_SUCCESS, ctx2.handle()); - // ctx2 handle prepare request - EXPECT_EQ(OB_SUCCESS, ctx2.apply()); - // ctx1 handle prepare response - EXPECT_EQ(OB_SUCCESS, ctx1.handle()); - // ctx1 apply prepare log - EXPECT_EQ(OB_SUCCESS, ctx1.apply()); + // // ctx2 handle prepare request + // EXPECT_EQ(OB_SUCCESS, ctx2.handle()); + // // ctx2 handle prepare request + // EXPECT_EQ(OB_SUCCESS, ctx2.apply()); + // // ctx1 handle prepare response + // EXPECT_EQ(OB_SUCCESS, ctx1.handle()); + // // ctx1 apply prepare log + // EXPECT_EQ(OB_SUCCESS, ctx1.apply()); // TODO shanyan.g /*