From 49b1cfbe6b6242cdfc17167b4ae326fa38217d8e Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 16 Apr 2024 10:45:36 +0000 Subject: [PATCH] [FEAT MERGE] support log cache for read optimazation Co-authored-by: HaHaJeff --- .../src/lib/statistic_event/ob_stat_event.h | 10 +- mittest/logservice/CMakeLists.txt | 2 +- .../env/ob_simple_log_cluster_env.cpp | 14 +- .../env/ob_simple_log_cluster_testbase.cpp | 21 + .../env/ob_simple_log_cluster_testbase.h | 1 + mittest/logservice/env/ob_simple_log_server.h | 1 + .../logservice/test_ob_simple_log_cache.cpp | 253 ++++ .../logservice/test_ob_simple_log_engine.cpp | 2 +- .../test_ob_simple_log_flashback.cpp | 56 + ...test_ob_simple_log_single_replica_func.cpp | 2 +- .../palf_cluster/logservice/log_service.cpp | 1 + .../archiveservice/ob_archive_fetcher.cpp | 4 +- src/logservice/cdcservice/ob_cdc_fetcher.cpp | 8 +- src/logservice/ob_log_service.cpp | 1 + src/logservice/palf/log_cache.cpp | 1046 ++++++++++++++++- src/logservice/palf/log_cache.h | 279 +++++ src/logservice/palf/log_define.h | 9 + src/logservice/palf/log_engine.cpp | 93 +- src/logservice/palf/log_engine.h | 10 +- src/logservice/palf/log_io_context.h | 28 +- src/logservice/palf/log_iterator_impl.h | 34 +- src/logservice/palf/log_iterator_info.h | 77 ++ src/logservice/palf/log_iterator_storage.cpp | 15 +- src/logservice/palf/log_iterator_storage.h | 12 +- src/logservice/palf/log_reader.cpp | 25 +- src/logservice/palf/log_reader.h | 11 +- src/logservice/palf/log_reader_utils.cpp | 3 +- src/logservice/palf/log_shared_queue_thread.h | 1 + src/logservice/palf/log_shared_task.cpp | 61 + src/logservice/palf/log_shared_task.h | 22 +- src/logservice/palf/log_storage.cpp | 117 +- src/logservice/palf/log_storage.h | 34 +- src/logservice/palf/palf_env_impl.cpp | 4 + src/logservice/palf/palf_env_impl.h | 2 + src/logservice/palf/palf_handle_impl.cpp | 40 +- src/logservice/palf/palf_handle_impl.h | 2 +- src/logservice/palf/palf_iterator.h | 52 +- src/logservice/palf/palf_options.cpp | 1 + src/logservice/palf/palf_options.h | 7 +- .../replayservice/ob_replay_status.cpp | 5 + src/observer/ob_server.cpp | 16 + src/observer/ob_server.h | 1 + .../virtual_table/ob_all_virtual_sys_stat.cpp | 3 + .../ob_information_kvcache_table.cpp | 2 + src/rootserver/ob_recovery_ls_service.cpp | 2 +- .../allocator/ob_tenant_mutil_allocator.cpp | 22 + .../allocator/ob_tenant_mutil_allocator.h | 8 + src/share/parameter/ob_parameter_seed.ipp | 5 + .../all_virtual_sys_parameter_stat.result | 1 + unittest/logservice/CMakeLists.txt | 1 + unittest/logservice/test_log_cache.cpp | 217 ++++ 51 files changed, 2472 insertions(+), 172 deletions(-) create mode 100644 mittest/logservice/test_ob_simple_log_cache.cpp create mode 100644 src/logservice/palf/log_iterator_info.h create mode 100644 unittest/logservice/test_log_cache.cpp diff --git a/deps/oblib/src/lib/statistic_event/ob_stat_event.h b/deps/oblib/src/lib/statistic_event/ob_stat_event.h index fddcee3cac..91de67b931 100644 --- a/deps/oblib/src/lib/statistic_event/ob_stat_event.h +++ b/deps/oblib/src/lib/statistic_event/ob_stat_event.h @@ -265,6 +265,9 @@ STAT_EVENT_ADD_DEF(SCHEMA_HISTORY_CACHE_MISS, "schema history cache miss", ObSta STAT_EVENT_ADD_DEF(OPT_SYSTEM_STAT_CACHE_HIT, "opt system stat cache hit", ObStatClassIds::CACHE, 50063, false, true, true) STAT_EVENT_ADD_DEF(OPT_SYSTEM_STAT_CACHE_MISS, "opt system stat cache miss", ObStatClassIds::CACHE, 50064, false, true, true) +STAT_EVENT_ADD_DEF(LOG_KV_CACHE_HIT, "log kv cache hit", ObStatClassIds::CACHE, 50065, false, true, true) +STAT_EVENT_ADD_DEF(LOG_KV_CACHE_MISS, "log kv cache miss", ObStatClassIds::CACHE, 50066, false, true, true) + // STORAGE STAT_EVENT_ADD_DEF(MEMSTORE_LOGICAL_READS, "MEMSTORE_LOGICAL_READS", STORAGE, "MEMSTORE_LOGICAL_READS", true, true, false) STAT_EVENT_ADD_DEF(MEMSTORE_LOGICAL_BYTES, "MEMSTORE_LOGICAL_BYTES", STORAGE, "MEMSTORE_LOGICAL_BYTES", true, true, false) @@ -385,9 +388,9 @@ STAT_EVENT_ADD_DEF(LS_ALL_TABLE_OPERATOR_GET_TIME, "log stream table operator ge STAT_EVENT_ADD_DEF(PALF_WRITE_IO_COUNT, "palf write io count to disk", ObStatClassIds::CLOG, 80001, true, true, true) STAT_EVENT_ADD_DEF(PALF_WRITE_SIZE, "palf write size to disk", ObStatClassIds::CLOG, 80002, true, true, true) STAT_EVENT_ADD_DEF(PALF_WRITE_TIME, "palf write total time to disk", ObStatClassIds::CLOG, 80003, true, true, true) -STAT_EVENT_ADD_DEF(PALF_READ_COUNT_FROM_CACHE, "palf read count from cache", ObStatClassIds::CLOG, 80004, true, true, true) -STAT_EVENT_ADD_DEF(PALF_READ_SIZE_FROM_CACHE, "palf read size from cache", ObStatClassIds::CLOG, 80005, true, true, true) -STAT_EVENT_ADD_DEF(PALF_READ_TIME_FROM_CACHE, "palf read total time from cache", ObStatClassIds::CLOG, 80006, true, true, true) +STAT_EVENT_ADD_DEF(PALF_READ_COUNT_FROM_HOT_CACHE, "palf read count from hot cache", ObStatClassIds::CLOG, 80004, true, true, true) +STAT_EVENT_ADD_DEF(PALF_READ_SIZE_FROM_HOT_CACHE, "palf read size from hot cache", ObStatClassIds::CLOG, 80005, true, true, true) +STAT_EVENT_ADD_DEF(PALF_READ_TIME_FROM_HOT_CACHE, "palf read total time from hot cache", ObStatClassIds::CLOG, 80006, true, true, true) STAT_EVENT_ADD_DEF(PALF_READ_IO_COUNT_FROM_DISK, "palf read io count from disk", ObStatClassIds::CLOG, 80007, true, true, true) STAT_EVENT_ADD_DEF(PALF_READ_SIZE_FROM_DISK, "palf read size from disk", ObStatClassIds::CLOG, 80008, true, true, true) STAT_EVENT_ADD_DEF(PALF_READ_TIME_FROM_DISK, "palf read total time from disk", ObStatClassIds::CLOG, 80009, true, true, true) @@ -584,6 +587,7 @@ STAT_EVENT_SET_DEF(SYS_BLOCK_CACHE_SIZE, "sys block cache size", ObStatClassIds: STAT_EVENT_SET_DEF(USER_BLOCK_CACHE_SIZE, "user block cache size", ObStatClassIds::CACHE, 120006, false, true, true) STAT_EVENT_SET_DEF(USER_ROW_CACHE_SIZE, "user row cache size", ObStatClassIds::CACHE, 120008, false, true, true) STAT_EVENT_SET_DEF(BLOOM_FILTER_CACHE_SIZE, "bloom filter cache size", ObStatClassIds::CACHE, 120009, false, true, true) +STAT_EVENT_SET_DEF(LOG_KV_CACHE_SIZE, "log kv cache size", ObStatClassIds::CACHE, 120010, false, true, true) // STORAGE STAT_EVENT_SET_DEF(ACTIVE_MEMSTORE_USED, "active memstore used", ObStatClassIds::STORAGE, 130000, false, true, true) diff --git a/mittest/logservice/CMakeLists.txt b/mittest/logservice/CMakeLists.txt index c28b80c359..90c65f734c 100644 --- a/mittest/logservice/CMakeLists.txt +++ b/mittest/logservice/CMakeLists.txt @@ -39,5 +39,5 @@ ob_unittest_clog(test_ob_simple_log_arb_mock_ele test_ob_simple_log_arb_mock_ele ob_unittest_clog(test_ob_simple_log_flashback_arb test_ob_simple_log_flashback_arb.cpp) ob_unittest_clog(test_ob_simple_log_restart test_ob_simple_log_restart.cpp) ob_unittest_clog(test_ob_simple_log_disk_mgr test_ob_simple_log_disk_mgr.cpp) - +ob_unittest_clog(test_ob_simple_log_cache test_ob_simple_log_cache.cpp) add_subdirectory(archiveservice) diff --git a/mittest/logservice/env/ob_simple_log_cluster_env.cpp b/mittest/logservice/env/ob_simple_log_cluster_env.cpp index e71dc02acc..a80b742a41 100644 --- a/mittest/logservice/env/ob_simple_log_cluster_env.cpp +++ b/mittest/logservice/env/ob_simple_log_cluster_env.cpp @@ -632,7 +632,7 @@ int ObSimpleLogClusterTestEnv::get_leader(const int64_t id, PalfHandleImplGuard } } while (OB_ENTRY_NOT_EXIST == ret); if (OB_SUCC(ret) && disable_hot_cache()) { - leader.get_palf_handle_impl()->log_engine_.log_storage_.hot_cache_ = NULL; + leader.get_palf_handle_impl()->log_engine_.log_storage_.log_cache_->hot_cache_.reset(); } PALF_LOG(INFO, "get_leader finished", K(ret), K(id), K(leader_idx)); return ret; @@ -1090,7 +1090,7 @@ int ObSimpleLogClusterTestEnv::read_log(PalfHandleImplGuard &leader) int ObSimpleLogClusterTestEnv::read_log(PalfHandleImplGuard &leader, const LSN &lsn) { int ret = OB_SUCCESS; - PalfBufferIterator iterator; + PalfBufferIterator iterator(leader.palf_id_); if (OB_FAIL(leader.palf_handle_impl_->alloc_palf_buffer_iterator(lsn, iterator))) { } else { while (OB_SUCCESS == ret) { @@ -1112,7 +1112,7 @@ int ObSimpleLogClusterTestEnv::read_log(PalfHandleImplGuard &leader, const LSN & int ObSimpleLogClusterTestEnv::read_group_log(PalfHandleImplGuard &leader, LSN lsn) { int ret = OB_SUCCESS; - PalfGroupBufferIterator iterator; + PalfGroupBufferIterator iterator(leader.palf_id_); if (OB_FAIL(leader.palf_handle_impl_->alloc_palf_group_buffer_iterator(lsn, iterator))) { } else { LogGroupEntry entry; @@ -1136,7 +1136,7 @@ int ObSimpleLogClusterTestEnv::read_and_submit_group_log(PalfHandleImplGuard &le const LSN &start_lsn) { int ret = OB_SUCCESS; - PalfGroupBufferIterator iterator; + PalfGroupBufferIterator iterator(leader.palf_id_); if (OB_FAIL(leader.palf_handle_impl_->alloc_palf_group_buffer_iterator(LSN(start_lsn), iterator))) { } else { LogGroupEntry entry; @@ -1153,7 +1153,7 @@ int ObSimpleLogClusterTestEnv::read_and_submit_group_log(PalfHandleImplGuard &le } if (OB_ITER_END == ret) { wait_until_has_committed(leader_raw_write, LSN(leader.palf_handle_impl_->get_end_lsn())); - PalfBufferIterator iterator_raw_write; + PalfBufferIterator iterator_raw_write(leader_raw_write.palf_id_); if (OB_FAIL(leader_raw_write.palf_handle_impl_->alloc_palf_buffer_iterator(LSN(0), iterator_raw_write))) { PALF_LOG(WARN, "leader seek failed", K(ret), K(iterator_raw_write)); } else { @@ -1202,7 +1202,7 @@ int ObSimpleLogClusterTestEnv::read_log_from_memory(PalfHandleImplGuard &leader) { int ret = OB_SUCCESS; LSN lsn(0); - MemPalfBufferIterator iterator; + MemPalfBufferIterator iterator(leader.palf_id_); MemoryStorage mem_storage; char *buf = nullptr; block_id_t min_block_id, max_block_id; @@ -1367,7 +1367,7 @@ void ObSimpleLogClusterTestEnv::wait_all_replcias_log_sync(const int64_t palf_id int ObSimpleLogClusterTestEnv::get_middle_scn(const int64_t log_num, PalfHandleImplGuard &leader, SCN &mid_scn, LogEntryHeader &log_entry_header) { int ret = OB_SUCCESS; - PalfBufferIterator iterator; + PalfBufferIterator iterator(leader.palf_id_); LSN init_lsn(PALF_INITIAL_LSN_VAL); if (OB_FAIL(leader.palf_handle_impl_->alloc_palf_buffer_iterator(init_lsn, iterator))) { PALF_LOG(ERROR, "seek failed", K(ret), K(iterator)); diff --git a/mittest/logservice/env/ob_simple_log_cluster_testbase.cpp b/mittest/logservice/env/ob_simple_log_cluster_testbase.cpp index 7b38261c46..681993a407 100644 --- a/mittest/logservice/env/ob_simple_log_cluster_testbase.cpp +++ b/mittest/logservice/env/ob_simple_log_cluster_testbase.cpp @@ -80,6 +80,7 @@ int ObSimpleLogClusterTestBase::start() } else if (OB_FAIL(member_region_map_.create(OB_MAX_MEMBER_NUMBER, ObMemAttr(MTL_ID(), ObModIds::OB_HASH_NODE, ObCtxIds::DEFAULT_CTX_ID)))) { } else if (OB_FAIL(generate_sorted_server_list_(node_cnt_))) { + } else if (OB_FAIL(init_log_kv_cache_())) { } else { // 如果需要新增arb server，将其作为memberlist最后一项 // TODO by runlin, 这个是暂时的解决方法，以后可以走加减成员的流程 @@ -130,6 +131,9 @@ int ObSimpleLogClusterTestBase::close() break; } } + + OB_LOG_KV_CACHE.destroy(); + ObKVGlobalCache::get_instance().destroy(); return ret; } @@ -166,5 +170,22 @@ int ObSimpleLogClusterTestBase::generate_sorted_server_list_(const int64_t node_ return ret; } +int ObSimpleLogClusterTestBase::init_log_kv_cache_() +{ + int ret = OB_SUCCESS; + const int64_t KV_CACHE_WASH_TIMER_INTERVAL_US = 60 * 1000L * 1000L; + const int64_t DEFAULT_BUCKET_NUM = 10000000L; + const int64_t DEFAULT_MAX_CACHE_SIZE = 1024L * 1024L * 1024L * 1024L; + if (OB_FAIL(ObKVGlobalCache::get_instance().init( + &ObTenantMemLimitGetter::get_instance(), DEFAULT_BUCKET_NUM, + DEFAULT_MAX_CACHE_SIZE, lib::ACHUNK_SIZE, + KV_CACHE_WASH_TIMER_INTERVAL_US))) { + PALF_LOG(WARN, "ObKVGlobalCache init failed"); + } else if (OB_FAIL(OB_LOG_KV_CACHE.init(OB_LOG_KV_CACHE_NAME, 1))) { + PALF_LOG(WARN, "OB_LOG_KV_CACHE init failed"); + } + return ret; +} + } // end unittest } // end oceanbase diff --git a/mittest/logservice/env/ob_simple_log_cluster_testbase.h b/mittest/logservice/env/ob_simple_log_cluster_testbase.h index fed04bc599..e6245e657e 100644 --- a/mittest/logservice/env/ob_simple_log_cluster_testbase.h +++ b/mittest/logservice/env/ob_simple_log_cluster_testbase.h @@ -77,6 +77,7 @@ public: private: static int generate_sorted_server_list_(const int64_t node_cnt); + static int init_log_kv_cache_(); protected: static void SetUpTestCase(); diff --git a/mittest/logservice/env/ob_simple_log_server.h b/mittest/logservice/env/ob_simple_log_server.h index 04df5dcd10..096b58b91b 100644 --- a/mittest/logservice/env/ob_simple_log_server.h +++ b/mittest/logservice/env/ob_simple_log_server.h @@ -51,6 +51,7 @@ #include "logservice/leader_coordinator/ob_failure_detector.h" #include #include +#include "share/ob_tenant_mem_limit_getter.h" namespace oceanbase { diff --git a/mittest/logservice/test_ob_simple_log_cache.cpp b/mittest/logservice/test_ob_simple_log_cache.cpp new file mode 100644 index 0000000000..4fc6988a97 --- /dev/null +++ b/mittest/logservice/test_ob_simple_log_cache.cpp @@ -0,0 +1,253 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "lib/ob_define.h" +#include "lib/ob_errno.h" +#include +#include +#include +#include +#include +#define private public +#define protected public +#include "env/ob_simple_log_cluster_env.h" +#include "logservice/palf/palf_iterator.h" +#undef private +#undef protected +#include "logservice/palf/log_define.h" +#include "logservice/palf/lsn.h" +#include "share/ob_tenant_mem_limit_getter.h" + + +const std::string TEST_NAME = "log_cache"; + +using namespace oceanbase::common; +using namespace oceanbase; + +namespace oceanbase +{ +using namespace logservice; + +namespace palf +{ +int LogColdCache::allow_filling_cache_(LogIteratorInfo *iterator_info, bool &enable_fill_cache) +{ + int ret = OB_SUCCESS; + enable_fill_cache = true; + return ret; +} +int FillCacheFsCb::update_end_lsn(int64_t id, + const palf::LSN &end_lsn, + const share::SCN &end_scn, + const int64_t proposal_id) +{ + int ret = OB_SUCCESS; + UNUSED(id); + UNUSED(end_scn); + UNUSED(proposal_id); + PalfOptions options; + // if (IS_NOT_INIT) { + // ret = OB_NOT_INIT; + // PALF_LOG(WARN, "FillCacheFsCb is not inited", K(ret)); + // } else if (!state_mgr_->is_leader_active()) { + // // // don't submit fill cache task + // } else { + // LSN begin_lsn = end_lsn - log_size; + // log_engine_->submit_fill_cache_task(begin_lsn, log_size); + // } + return ret; +} +} +namespace unittest +{ +class TestObSimpleLogCache : public ObSimpleLogClusterTestEnv +{ +public: + TestObSimpleLogCache() : ObSimpleLogClusterTestEnv() { + int ret = init(); + if (OB_SUCCESS != ret) { + throw std::runtime_error("TestObSimpleLogCache init failed"); + } + } + ~TestObSimpleLogCache() { destroy(); } + int init() { + return OB_SUCCESS; + } + void destroy() {} + using ObSimpleLogClusterTestEnv::read_log; + int read_log(PalfHandleImplGuard &leader, PalfBufferIterator &iterator) + { + return read_log(leader, LSN(PALF_INITIAL_LSN_VAL), iterator); + } + int read_log(PalfHandleImplGuard &leader, LSN read_lsn, PalfBufferIterator &iterator) + { + int ret = OB_SUCCESS; + if (OB_FAIL(leader.palf_handle_impl_->alloc_palf_buffer_iterator(read_lsn, iterator))) { + } else { + while (OB_SUCCESS == ret) { + const char *buf; + int64_t buf_len = 0; + share::SCN scn = share::SCN::min_scn(); + LSN log_offset; + bool is_raw_write = false; + if (OB_FAIL(iterator.next())) { + } else if (OB_FAIL(iterator.get_entry(buf, buf_len, scn, log_offset, is_raw_write))) { + } else { + PALF_LOG(TRACE, "print log entry", K(is_raw_write), K(iterator)); + } + } + } + return ret; + } + int64_t id_; +}; + +int64_t ObSimpleLogClusterTestBase::member_cnt_ = 1; +int64_t ObSimpleLogClusterTestBase::node_cnt_ = 1; +std::string ObSimpleLogClusterTestBase::test_name_ = TEST_NAME; +bool ObSimpleLogClusterTestBase::need_add_arb_server_ = false; + +TEST_F(TestObSimpleLogCache, read) +{ + update_server_log_disk(10*1024*1024*1024ul); + update_disk_options(10*1024*1024*1024ul/palf::PALF_PHY_BLOCK_SIZE); + disable_hot_cache_ = true; + SET_CASE_LOG_FILE(TEST_NAME, "read"); + OB_LOGGER.set_log_level("TRACE"); + int server_idx = 0; + int64_t leader_idx = 0; + int64_t id = ATOMIC_AAF(&palf_id_, 1); + int64_t total_used_size = 0, total_size = 0; + share::SCN create_scn = share::SCN::base_scn(); + PALF_LOG(INFO, "start to test log cache", K(id)); + + PalfHandleImplGuard leader; + EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader)); + std::vector lsn_array; + std::vector scn_array; + EXPECT_EQ(OB_SUCCESS, submit_log(leader, 5000, 30 * 1024, id, lsn_array, scn_array)); + const LSN max_lsn = leader.get_palf_handle_impl()->get_max_lsn(); + EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, max_lsn)); + PALF_LOG(INFO, "first to read log"); + EXPECT_EQ(OB_ITER_END, read_log(leader)); + + PALF_LOG(INFO, "start to hit cache"); + LSN read_lsn(lsn_array[2]); + EXPECT_EQ(OB_ITER_END, read_log(leader)); + + OB_LOG_KV_CACHE.destroy(); +} + +TEST_F(TestObSimpleLogCache, concurrent_read) +{ + disable_hot_cache_ = true; + SET_CASE_LOG_FILE(TEST_NAME, "concurrent_read"); + OB_LOGGER.set_log_level("TRACE"); + int server_idx = 0; + int64_t leader_idx = 0; + int64_t id = ATOMIC_AAF(&palf_id_, 1); + PALF_LOG(INFO, "start to test log cache", K(id)); + + EXPECT_EQ(OB_SUCCESS, OB_LOG_KV_CACHE.init(OB_LOG_KV_CACHE_NAME, 1)); + PalfHandleImplGuard leader; + EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader)); + ObTenantEnv::set_tenant(get_cluster()[leader_idx]->get_tenant_base()); + std::vector lsn_array; + std::vector scn_array; + EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, MAX_LOG_BODY_SIZE, id, lsn_array, scn_array)); + const LSN max_lsn = leader.get_palf_handle_impl()->get_max_lsn(); + EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, max_lsn)); + { + std::thread read_thread_1([this, &leader, &leader_idx] { + ObTenantEnv::set_tenant(get_cluster()[leader_idx]->get_tenant_base()); + EXPECT_EQ(OB_ITER_END, read_log(leader)); + }); + + std::thread read_thread_2([this, &lsn_array, &leader, &leader_idx] { + ObTenantEnv::set_tenant(get_cluster()[leader_idx]->get_tenant_base()); + PALF_LOG(INFO, "start to hit cache"); + LSN read_lsn(lsn_array[2]); + EXPECT_EQ(OB_ITER_END, read_log(leader)); + }); + read_thread_1.join(); + read_thread_2.join(); + } + + OB_LOG_KV_CACHE.destroy(); +} + +// enable in 4.4 +TEST_F(TestObSimpleLogCache, DISABLED_fill_cache_when_slide) +{ + disable_hot_cache_ = false; + SET_CASE_LOG_FILE(TEST_NAME, "fill_cache_when_slide"); + OB_LOGGER.set_log_level("TRACE"); + int server_idx = 0; + int64_t leader_idx = 0; + int64_t id = ATOMIC_AAF(&palf_id_, 1); + int64_t total_used_size = 0, total_size = 0; + share::SCN create_scn = share::SCN::base_scn(); + PALF_LOG(INFO, "start to test log cache", K(id)); + + EXPECT_EQ(OB_SUCCESS, OB_LOG_KV_CACHE.init(OB_LOG_KV_CACHE_NAME, 1)); + PalfHandleImplGuard leader; + EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader)); + std::vector lsn_array; + std::vector scn_array; + { + EXPECT_EQ(OB_SUCCESS, submit_log(leader, 5000, 30 * 1024, id, lsn_array, scn_array)); + const LSN max_lsn = leader.get_palf_handle_impl()->get_max_lsn(); + EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, max_lsn)); + + PalfBufferIterator iterator(leader.palf_id_); + PALF_LOG(INFO, "start to read log"); + EXPECT_EQ(OB_ITER_END, read_log(leader, iterator)); + // all hit cache, no read disk + EXPECT_EQ(0, iterator.io_ctx_.iterator_info_.miss_cnt_); + } + + { + PALF_LOG(INFO, "test exceptional situations", K(id)); + // miss hot cache when committed logs slide, unable to fill cold cache + EXPECT_EQ(OB_SUCCESS, submit_log(leader, 5000, 30 * 1024, id, lsn_array, scn_array)); + PALF_LOG(INFO, "reset hot cache", K(id)); + leader.get_palf_handle_impl()->log_engine_.log_storage_.log_cache_->hot_cache_.reset(); + + LSN failed_aligned_lsn = leader.get_palf_handle_impl()->log_engine_.log_storage_.log_cache_->fill_buf_.aligned_lsn_; + const LSN max_lsn = leader.get_palf_handle_impl()->get_max_lsn(); + SCN result_scn; + EXPECT_EQ(OB_SUCCESS, leader.get_palf_handle_impl()->locate_by_lsn_coarsely(failed_aligned_lsn, result_scn)); + LSN read_lsn; + EXPECT_EQ(OB_SUCCESS, leader.get_palf_handle_impl()->locate_by_scn_coarsely(result_scn, read_lsn)); + + EXPECT_EQ(OB_SUCCESS, leader.get_palf_handle_impl()->log_engine_.log_storage_.log_cache_->hot_cache_.init(id, leader.get_palf_handle_impl())); + EXPECT_EQ(OB_SUCCESS, submit_log(leader, 5000, 30 * 1024, id, lsn_array, scn_array)); + + PalfBufferIterator iterator(leader.palf_id_); + + EXPECT_EQ(OB_ITER_END, read_log(leader, read_lsn, iterator)); + // miss, have to read disk at lease once + EXPECT_LT(0, iterator.io_ctx_.iterator_info_.miss_cnt_); + } + + + OB_LOG_KV_CACHE.destroy(); +} + + +} // namespace unittest +} // namespace oceanbase + +int main(int argc, char **argv) +{ + RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME); +} \ No newline at end of file diff --git a/mittest/logservice/test_ob_simple_log_engine.cpp b/mittest/logservice/test_ob_simple_log_engine.cpp index 81dc18c0f7..d72c998229 100644 --- a/mittest/logservice/test_ob_simple_log_engine.cpp +++ b/mittest/logservice/test_ob_simple_log_engine.cpp @@ -80,7 +80,7 @@ public: leader_.palf_handle_impl_->log_dir_, alloc_mgr, log_block_pool, - &(leader_.palf_handle_impl_->hot_cache_), + &(leader_.palf_handle_impl_->log_cache_), log_rpc, log_io_worker, log_shared_queue_th, diff --git a/mittest/logservice/test_ob_simple_log_flashback.cpp b/mittest/logservice/test_ob_simple_log_flashback.cpp index 0b505a5c06..b825b07535 100644 --- a/mittest/logservice/test_ob_simple_log_flashback.cpp +++ b/mittest/logservice/test_ob_simple_log_flashback.cpp @@ -447,6 +447,62 @@ TEST_F(TestObSimpleLogClusterFlashback, flashback_after_restart) EXPECT_EQ(new_log_tail1, leader1.palf_handle_impl_->sw_.committed_end_lsn_); } +TEST_F(TestObSimpleLogClusterFlashback, flashback_log_cache) +{ + disable_hot_cache_ = true; + SET_CASE_LOG_FILE(TEST_NAME, "flashback_log_cache"); + OB_LOGGER.set_log_level("TRACE"); + int server_idx = 0; + int64_t leader_idx = 0; + int64_t id = ATOMIC_AAF(&palf_id_, 1); + + PalfHandleImplGuard leader; + EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader)); + + EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id, MAX_LOG_BODY_SIZE)); + const LSN max_lsn = leader.get_palf_handle_impl()->get_max_lsn(); + EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, max_lsn)); + + SCN flashback_scn; + flashback_scn.convert_from_ts(common::ObTimeUtility::current_time()); + EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id)); + EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.get_palf_handle_impl()->get_max_lsn())); + EXPECT_LT(flashback_scn, leader.get_palf_handle_impl()->get_max_scn()); + + PALF_LOG(INFO, "start to hit cache"); + EXPECT_EQ(OB_ITER_END, read_log(leader)); + + int64_t miss_cnt = leader.get_palf_handle_impl()->log_cache_.cold_cache_.log_cache_stat_.miss_cnt_; + EXPECT_NE(0, miss_cnt); + PALF_LOG(INFO, "miss cnt", K(miss_cnt)); + EXPECT_EQ(OB_ITER_END, read_log(leader)); + EXPECT_LE(miss_cnt + 1, leader.get_palf_handle_impl()->log_cache_.cold_cache_.log_cache_stat_.miss_cnt_); + + // switch mode from APPEND to RAW_WRITE + int64_t mode_version = INVALID_PROPOSAL_ID; + switch_append_to_raw_write(leader, mode_version); + + // execute flashback + PALF_LOG(INFO, "start to flashback", K(get_cluster().size())); + ObLogFlashbackService *flashback_srv = get_cluster()[0]->get_flashback_service(); + const int64_t timeout_us = 10 * 1000 * 1000; + int64_t flashabck_version = leader.palf_handle_impl_->log_engine_.log_storage_.flashback_version_; + EXPECT_EQ(OB_SUCCESS, flashback_srv->flashback(MTL_ID(), flashback_scn, timeout_us)); + + LSN new_log_tail1 = leader.palf_handle_impl_->log_engine_.log_storage_.log_tail_; + EXPECT_EQ(new_log_tail1, leader.palf_handle_impl_->sw_.committed_end_lsn_); + EXPECT_GE(flashback_scn, leader.palf_handle_impl_->sw_.last_slide_scn_); + EXPECT_EQ(flashabck_version + 1, leader.palf_handle_impl_->log_engine_.log_storage_.flashback_version_); + PALF_LOG(INFO, "miss cnt", K(leader.get_palf_handle_impl()->log_cache_.cold_cache_.log_cache_stat_.miss_cnt_)); + EXPECT_EQ(OB_ITER_END, read_log(leader)); + int64_t new_miss_cnt = leader.get_palf_handle_impl()->log_cache_.cold_cache_.log_cache_stat_.miss_cnt_; + if (new_miss_cnt < miss_cnt) { + EXPECT_NE(0, new_miss_cnt); + } else { + EXPECT_LT(miss_cnt + 1, new_miss_cnt); + } +} + } // end unittest } // end oceanbase diff --git a/mittest/logservice/test_ob_simple_log_single_replica_func.cpp b/mittest/logservice/test_ob_simple_log_single_replica_func.cpp index ab6a08414b..145b6533db 100644 --- a/mittest/logservice/test_ob_simple_log_single_replica_func.cpp +++ b/mittest/logservice/test_ob_simple_log_single_replica_func.cpp @@ -1902,7 +1902,7 @@ TEST_F(TestObSimpleLogClusterSingleReplica, test_raw_read) char *read_buf_ptr = reinterpret_cast(mtl_malloc_align( LOG_DIO_ALIGN_SIZE, PALF_BLOCK_SIZE + 2 * LOG_DIO_ALIGN_SIZE, "mittest")); EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader)); - leader.palf_handle_impl_->log_engine_.log_storage_.hot_cache_ = NULL; + leader.palf_handle_impl_->log_engine_.log_storage_.log_cache_->hot_cache_.reset(); // 提交100条日志, 每条日志大小为30K. { char *read_buf = read_buf_ptr; diff --git a/mittest/palf_cluster/logservice/log_service.cpp b/mittest/palf_cluster/logservice/log_service.cpp index 51d9b90fa6..f1151d23fd 100644 --- a/mittest/palf_cluster/logservice/log_service.cpp +++ b/mittest/palf_cluster/logservice/log_service.cpp @@ -499,6 +499,7 @@ int LogService::update_palf_options_except_disk_usage_limit_size() palf_opts.compress_options_.transport_compress_func_ = compressor_type; palf_opts.rebuild_replica_log_lag_threshold_ = tenant_config->_rebuild_replica_log_lag_threshold; palf_opts.disk_options_.log_writer_parallelism_ = tenant_config->_log_writer_parallelism; + palf_opts.enable_log_cache_ = tenant_config->_enable_log_cache; if (OB_FAIL(palf_env_->update_options(palf_opts))) { CLOG_LOG(WARN, "palf update_options failed", K(MTL_ID()), K(ret), K(palf_opts)); } else { diff --git a/src/logservice/archiveservice/ob_archive_fetcher.cpp b/src/logservice/archiveservice/ob_archive_fetcher.cpp index 6e842e7f3a..c09a8746f9 100644 --- a/src/logservice/archiveservice/ob_archive_fetcher.cpp +++ b/src/logservice/archiveservice/ob_archive_fetcher.cpp @@ -361,11 +361,11 @@ int ObArchiveFetcher::handle_log_fetch_task_(ObArchiveLogFetchTask &task) int ret = OB_SUCCESS; bool need_delay = false; bool submit_log = false; - PalfGroupBufferIterator iter; + const ObLSID id = task.get_ls_id(); + PalfGroupBufferIterator iter(id.id(), palf::LogIOUser::ARCHIVE); PalfHandleGuard palf_handle_guard; TmpMemoryHelper helper(unit_size_, allocator_); ObArchiveSendTask *send_task = NULL; - const ObLSID id = task.get_ls_id(); const ArchiveWorkStation &station = task.get_station(); ArchiveKey key = station.get_round(); SCN commit_scn; diff --git a/src/logservice/cdcservice/ob_cdc_fetcher.cpp b/src/logservice/cdcservice/ob_cdc_fetcher.cpp index 225681a296..1c14e0bf82 100644 --- a/src/logservice/cdcservice/ob_cdc_fetcher.cpp +++ b/src/logservice/cdcservice/ob_cdc_fetcher.cpp @@ -104,7 +104,7 @@ int ObCdcFetcher::fetch_log(const ObCdcLSFetchLogReq &req, const ObLSID &ls_id = req.get_ls_id(); const LSN &start_lsn = req.get_start_lsn(); PalfHandleGuard palf_handle_guard; - PalfGroupBufferIterator group_iter; + PalfGroupBufferIterator group_iter(ls_id.id(), palf::LogIOUser::CDC); const ObCdcRpcId &rpc_id = req.get_client_id(); ClientLSCtx *ls_ctx = NULL; @@ -508,7 +508,7 @@ int ObCdcFetcher::ls_fetch_log_(const ObLSID &ls_id, { int ret = OB_SUCCESS; const int64_t start_ls_fetch_log_time = ObTimeUtility::current_time(); - PalfGroupBufferIterator palf_iter; + PalfGroupBufferIterator palf_iter(ls_id.id(), palf::LogIOUser::CDC); PalfHandleGuard palf_guard; int64_t version = 0; // use cached remote_iter @@ -924,11 +924,11 @@ int ObCdcFetcher::do_fetch_missing_log_(const obrpc::ObCdcLSFetchMissLogReq &req } else { for (int64_t idx = 0; OB_SUCC(ret) && ! frt.is_stopped() && idx < miss_log_array.count(); idx++) { // need_init_iter should always be true, declared here to ensure need init iter be true in each loop - PalfBufferIterator palf_iter; + PalfBufferIterator palf_iter(ls_id.id(), palf::LogIOUser::CDC); ObRemoteLogpEntryIterator remote_iter(get_source_func); const obrpc::ObCdcLSFetchMissLogReq::MissLogParam &miss_log_info = miss_log_array[idx]; const LSN &missing_lsn = miss_log_info.miss_lsn_; - palf::PalfBufferIterator log_entry_iter; + palf::PalfBufferIterator log_entry_iter(ls_id.id(), palf::LogIOUser::CDC); LogEntry log_entry; LSN lsn; resp.set_next_miss_lsn(missing_lsn); diff --git a/src/logservice/ob_log_service.cpp b/src/logservice/ob_log_service.cpp index abffbe0ea6..77014dbd20 100644 --- a/src/logservice/ob_log_service.cpp +++ b/src/logservice/ob_log_service.cpp @@ -573,6 +573,7 @@ int ObLogService::update_palf_options_except_disk_usage_limit_size() palf_opts.compress_options_.transport_compress_func_ = compressor_type; palf_opts.rebuild_replica_log_lag_threshold_ = tenant_config->_rebuild_replica_log_lag_threshold; palf_opts.disk_options_.log_writer_parallelism_ = tenant_config->_log_writer_parallelism; + palf_opts.enable_log_cache_ = tenant_config->_enable_log_cache; if (OB_FAIL(palf_env_->update_options(palf_opts))) { CLOG_LOG(WARN, "palf update_options failed", K(MTL_ID()), K(ret), K(palf_opts)); } else { diff --git a/src/logservice/palf/log_cache.cpp b/src/logservice/palf/log_cache.cpp index e4be6c2b99..6971cce599 100644 --- a/src/logservice/palf/log_cache.cpp +++ b/src/logservice/palf/log_cache.cpp @@ -14,6 +14,7 @@ #include "lib/stat/ob_session_stat.h" #include "log_cache.h" #include "palf_handle_impl.h" +#include "palf_handle_impl_guard.h" namespace oceanbase { @@ -59,6 +60,7 @@ int LogHotCache::init(const int64_t palf_id, IPalfHandleImpl *palf_handle_impl) palf_id_ = palf_id; palf_handle_impl_ = palf_handle_impl; is_inited_ = true; + PALF_LOG(TRACE, "init hot cache successfully", K(palf_id_)); } return ret; } @@ -74,6 +76,7 @@ int LogHotCache::read(const LSN &read_begin_lsn, int64_t start_ts = ObTimeUtility::fast_current_time(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; + PALF_LOG(WARN, "hot cache is not inited", K(ret), K(palf_id_)); } else if (!read_begin_lsn.is_valid() || in_read_size <= 0 || OB_ISNULL(buf)) { ret = OB_INVALID_ARGUMENT; PALF_LOG(WARN, "invalid arguments", K(ret), K_(palf_id), K(read_begin_lsn), K(in_read_size), @@ -88,9 +91,9 @@ int LogHotCache::read(const LSN &read_begin_lsn, int64_t cost_ts = ObTimeUtility::fast_current_time() - start_ts; hit_cnt = ATOMIC_AAF(&hit_count_, 1); read_size = ATOMIC_AAF(&read_size_, out_read_size); - EVENT_TENANT_INC(ObStatEventIds::PALF_READ_COUNT_FROM_CACHE, MTL_ID()); - EVENT_ADD(ObStatEventIds::PALF_READ_SIZE_FROM_CACHE, out_read_size); - EVENT_ADD(ObStatEventIds::PALF_READ_TIME_FROM_CACHE, cost_ts); + EVENT_TENANT_INC(ObStatEventIds::PALF_READ_COUNT_FROM_HOT_CACHE, MTL_ID()); + EVENT_ADD(ObStatEventIds::PALF_READ_SIZE_FROM_HOT_CACHE, out_read_size); + EVENT_ADD(ObStatEventIds::PALF_READ_TIME_FROM_HOT_CACHE, cost_ts); PALF_LOG(TRACE, "read_data_from_buffer success", K(ret), K_(palf_id), K(read_begin_lsn), K(in_read_size), K(out_read_size)); } @@ -105,5 +108,1042 @@ int LogHotCache::read(const LSN &read_begin_lsn, return ret; } +// ==================== FillCacheFsCb ========================= +FillCacheFsCb::FillCacheFsCb() : palf_env_impl_(NULL), state_mgr_(NULL), log_engine_(NULL), is_inited_(false) +{} + +FillCacheFsCb::~FillCacheFsCb() +{ + destroy(); +} + +int FillCacheFsCb::init(IPalfEnvImpl *palf_env_impl, LogStateMgr *state_mgr, LogEngine *log_engine) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + PALF_LOG(WARN, "FillCacheFsCb has been inited!", K(ret)); + } else if (OB_ISNULL(palf_env_impl) || OB_ISNULL(log_engine)) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument!"); + } else { + palf_env_impl_ = palf_env_impl; + state_mgr_ = state_mgr; + log_engine_ = log_engine; + is_inited_ = true; + } + return ret; +} + +void FillCacheFsCb::destroy() +{ + palf_env_impl_ = NULL; + state_mgr_ = NULL; + log_engine_ = NULL; + is_inited_ = false; +} + +int FillCacheFsCb::update_end_lsn(int64_t id, + const palf::LSN &end_lsn, + const share::SCN &end_scn, + const int64_t proposal_id) +{ + int ret = OB_SUCCESS; + UNUSED(id); + UNUSED(end_scn); + UNUSED(proposal_id); + + PalfOptions options; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(WARN, "FillCacheFsCb is not inited", K(ret)); + } else if (!state_mgr_->is_leader_active()) { + // don't submit fill cache task when it isn't a leader + } else if (OB_FAIL(palf_env_impl_->get_options(options))) { + PALF_LOG(WARN, "get options failed", K(ret)); + } else if (!options.enable_log_cache_) { + // don't submit fill cache task when it isn't allowed + } else { + // it will be enable in 4.4 + //LSN begin_lsn = end_lsn - log_size; + //log_engine_->submit_fill_cache_task(begin_lsn, log_size); + } + return ret; +} + +//============================================= LogCacheUtils ========================== +LSN LogCacheUtils::lower_align_with_start(const LSN &input, int64_t align) +{ + offset_t start = lsn_2_block(input, PALF_BLOCK_SIZE) * PALF_BLOCK_SIZE; + offset_t ret = start + lower_align(input.val_ - start, align); + LSN aligned_lsn(ret); + return aligned_lsn; +} + +LSN LogCacheUtils::upper_align_with_start(const LSN &input, int64_t align) +{ + OB_ASSERT(!is_in_last_cache_line(input)); + return lower_align_with_start(input, align) + align; +} + +bool LogCacheUtils::is_lower_align_with_start(const LSN &input, int64_t align) +{ + return input == lower_align_with_start(input, align); +} + +bool LogCacheUtils::up_to_next_block(const LSN &input, const int64_t remained_size) +{ + return input + remained_size == next_block_start_lsn(input); +} + +LSN LogCacheUtils::next_block_start_lsn(const LSN &input) +{ + int64_t curr_block_id = lsn_2_block(input, PALF_BLOCK_SIZE); + return LSN((curr_block_id + 1) * PALF_BLOCK_SIZE); +} + +bool LogCacheUtils::is_in_last_cache_line(const LSN &input) +{ + return LogCacheUtils::next_block_start_lsn(input) - input <= LAST_CACHE_LINE_SIZE; +} + +//============================================= LogKVCacheKey ========================== +LogKVCacheKey::LogKVCacheKey() + : tenant_id_(OB_INVALID_TENANT_ID), palf_id_(INVALID_PALF_ID), aligned_lsn_(LOG_INVALID_LSN_VAL), + flashback_version_(OB_INVALID_TIMESTAMP) {} + +LogKVCacheKey::LogKVCacheKey(const uint64_t tenant_id, + const int64_t palf_id, + const LSN aligned_lsn, + const int64_t flashback_version) + : tenant_id_(tenant_id), palf_id_(palf_id), aligned_lsn_(aligned_lsn), + flashback_version_(flashback_version) {} + +LogKVCacheKey::~LogKVCacheKey() +{ + reset(); +} + +bool LogKVCacheKey::is_valid() const +{ + return is_valid_tenant_id(tenant_id_) && aligned_lsn_.is_valid() && + is_valid_palf_id(palf_id_) && is_valid_flashback_version(flashback_version_); +} + +void LogKVCacheKey::reset() +{ + tenant_id_ = OB_INVALID_TENANT_ID; + palf_id_ = INVALID_PALF_ID; + aligned_lsn_.reset(); + flashback_version_ = OB_INVALID_TIMESTAMP; +} + +bool LogKVCacheKey::operator ==(const ObIKVCacheKey &other) const +{ + const LogKVCacheKey &other_key = reinterpret_cast(other); + return tenant_id_ == other_key.tenant_id_ && palf_id_ == other_key.palf_id_ && + aligned_lsn_ == other_key.aligned_lsn_ && + flashback_version_ == other_key.flashback_version_; +} + +uint64_t LogKVCacheKey::hash() const +{ + uint64_t hash_code = 0; + hash_code = murmurhash(&tenant_id_, sizeof(tenant_id_), hash_code); + hash_code = murmurhash(&palf_id_, sizeof(palf_id_), hash_code); + hash_code = murmurhash(&aligned_lsn_, sizeof(aligned_lsn_), hash_code); + hash_code = murmurhash(&flashback_version_, sizeof(flashback_version_), hash_code); + return hash_code; +} + +uint64_t LogKVCacheKey::get_tenant_id() const +{ + return tenant_id_; +} + +int64_t LogKVCacheKey::size() const +{ + return sizeof(*this); +} + +int LogKVCacheKey::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len < size())) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", K(ret), K(buf_len), K(size())); + } else { + LogKVCacheKey *new_key = new (buf) LogKVCacheKey(tenant_id_, palf_id_, aligned_lsn_, flashback_version_); + if (OB_ISNULL(new_key)) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "new_key ptr is null", K(ret), K(*this)); + } else { + key = new_key; + } + } + + return ret; +} + +//============================================= LogKVCacheValue ========================== +LogKVCacheValue::LogKVCacheValue() : buf_(NULL), buf_size_(0) {} +LogKVCacheValue::LogKVCacheValue(char *buf, const int64_t buf_size) : buf_(buf), buf_size_(buf_size) {} +LogKVCacheValue::~LogKVCacheValue() +{ + reset(); +} + +void LogKVCacheValue::reset() +{ + if (OB_NOT_NULL(buf_)) { + mtl_free(buf_); + } + buf_ = NULL; + buf_size_ = 0; +} + +int LogKVCacheValue::init(char *buf, const int64_t size) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf_ = reinterpret_cast(mtl_malloc(size, "LOG_KV_CACHE")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + PALF_LOG(WARN, "allocate memory for log cold cache failed", K(ret), K(size)); + } else { + MEMCPY(buf_, buf, size); + buf_size_ = size; + } + return ret; +} + +bool LogKVCacheValue::is_valid() const +{ + return buf_size_ != 0; +} + +char *LogKVCacheValue::get_buf() const +{ + return buf_; +} + +int64_t LogKVCacheValue::get_buf_size() const +{ + return buf_size_; +} + +int64_t LogKVCacheValue::size() const +{ + return is_valid() ? (sizeof(*this) + buf_size_) : 0; +} + +int LogKVCacheValue::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf) || buf_len < size()) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", K(ret), K(buf_len), K(size())); + } else { + LogKVCacheValue *new_value = new (buf) LogKVCacheValue(); + if (OB_ISNULL(new_value)) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "new_value is null", K(ret), K(*this)); + } else { + new_value->buf_ = buf + sizeof(*this); + new_value->buf_size_ = buf_size_; + MEMCPY(new_value->buf_, buf_, buf_size_); + value = new_value; + } + } + + return ret; +} + +//============================================= LogKVCache ========================== +LogKVCache &LogKVCache::get_instance() +{ + static LogKVCache kv_cache; + return kv_cache; +} + +LogKVCache::LogKVCache() +{} + +LogKVCache::~LogKVCache() +{} + +int LogKVCache::get_log(const LogKVCacheKey &key, LogKVCacheValueHandle &val_handle) +{ + int ret = OB_SUCCESS; + if (!key.is_valid()) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", K(ret), K(key)); + } else if (OB_FAIL(get(key, val_handle.value_, val_handle.handle_))) { + CLOG_LOG(WARN, "get from cache failed", K(ret), K(key)); + } else if (OB_ISNULL(val_handle.value_)) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "get a null value from kv cache", K(ret), K(key)); + } + + return ret; +} + +int LogKVCache::put_log(const LogKVCacheKey &key, const LogKVCacheValue &value) +{ + int ret = OB_SUCCESS; + if (!key.is_valid() || !value.is_valid()) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", K(ret), K(key), K(value)); + } else if (OB_FAIL(put(key, value, false /* overwrite */))) { + CLOG_LOG(WARN, "put log failed", K(ret), K(key), K(value)); + } + return ret; +} + +//============================================= FillBuf ========================== +FillBuf::FillBuf() + : aligned_lsn_(LOG_INVALID_LSN_VAL), fill_pos_(0), + buf_len_(0), flashback_version_(OB_INVALID_TIMESTAMP), + buf_(NULL), kvpair_(NULL), handle_(), inst_handle_() {} + +FillBuf::~FillBuf() +{ + reset(); +} + +void FillBuf::reset() +{ + aligned_lsn_.reset(); + fill_pos_ = 0; + buf_len_ = 0; + flashback_version_ = OB_INVALID_TIMESTAMP; + buf_ = NULL; + kvpair_ = NULL; + handle_.reset(); + inst_handle_.reset(); +} + +char *FillBuf::get_writable_buf() +{ + return buf_ + fill_pos_; +} + +bool FillBuf::is_valid() +{ + return OB_NOT_NULL(buf_) && 0 < buf_len_ && aligned_lsn_.is_valid() && OB_NOT_NULL(kvpair_) && + handle_.is_valid() && inst_handle_.is_valid(); +} + +bool FillBuf::is_full() +{ + bool bool_ret = false; + if (fill_pos_ == buf_len_) { + bool_ret = true; + } else if (LogCacheUtils::is_in_last_cache_line(aligned_lsn_) && (LAST_CACHE_LINE_SIZE == fill_pos_)) { + bool_ret = true; + } + return bool_ret; +} + +void FillBuf::update_state(const LSN &aligned_lsn, const int64_t flashback_version) +{ + aligned_lsn_ = aligned_lsn; + fill_pos_ = 0; + buf_len_ = CACHE_LINE_SIZE; + flashback_version_ = flashback_version; +} + +//============================================= LogColdCache ========================== +LogColdCache::LogColdCache() + : palf_id_(INVALID_PALF_ID), palf_env_impl_(NULL), log_reader_(NULL), + kv_cache_(NULL), logical_block_size_(0), log_cache_stat_(), is_inited_(false) {} + +int LogColdCache::init(int64_t palf_id, + IPalfEnvImpl *palf_env_impl, + LogStorage *log_storage) { + int ret = OB_SUCCESS; + if (INVALID_PALF_ID == palf_id || OB_ISNULL(log_storage)) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "LogColdCache init failed", K(ret), K(palf_id)); + } else if (OB_FAIL(log_storage->get_logical_block_size(logical_block_size_))) { + PALF_LOG(WARN, "get_logical_block_size failed", K(ret), K(palf_id)); + } else { + palf_id_ = palf_id; + palf_env_impl_ = palf_env_impl; + log_reader_ = log_storage->get_log_reader(); + kv_cache_ = &OB_LOG_KV_CACHE.get_instance(); + is_inited_ = true; + PALF_LOG(INFO, "LogColdCache init successfully", K(is_inited_), K(palf_id), K(log_storage)); + } + + return ret; +} + +LogColdCache::~LogColdCache() +{ + destroy(); +} + +void LogColdCache::destroy() +{ + palf_id_ = INVALID_PALF_ID; + palf_env_impl_ = NULL; + log_reader_ = NULL; + kv_cache_ = NULL; + log_cache_stat_.reset(); + is_inited_ = false; +} + +int LogColdCache::alloc_kv_pair(const int64_t flashback_version, + const LSN &aligned_lsn, + FillBuf &fill_buf) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(WARN, "LogColdCache is not inited", K(ret)); + } else if (OB_FAIL(kv_cache_->alloc(MTL_ID(), sizeof(LogKVCacheKey), + sizeof(LogKVCacheValue) + CACHE_LINE_SIZE, + fill_buf.kvpair_, fill_buf.handle_, + fill_buf.inst_handle_))) { + PALF_LOG(WARN, "alloc kvpair failed", K(ret), K(palf_id_)); + } else { + fill_buf.kvpair_->key_ = new (fill_buf.kvpair_->key_) LogKVCacheKey(MTL_ID(), palf_id_, aligned_lsn, flashback_version); + fill_buf.buf_ = reinterpret_cast(fill_buf.kvpair_->value_) + sizeof(LogKVCacheValue); + fill_buf.kvpair_->value_ = new (fill_buf.kvpair_->value_) LogKVCacheValue(fill_buf.buf_, CACHE_LINE_SIZE); + + PALF_LOG(TRACE, "alloc kvpair successfully", K(palf_id_), K(fill_buf.kvpair_), K(fill_buf.kvpair_->value_), KP(fill_buf.kvpair_->value_)); + } + + return ret; +} + +int LogColdCache::read(const int64_t flashback_version, + const LSN &lsn, + const int64_t in_read_size, + ReadBuf &read_buf, + int64_t &out_read_size, + LogIteratorInfo *iterator_info) +{ + #define PRINT_INFO K(palf_id_), K(MTL_ID()) + + int ret = OB_SUCCESS; + bool enable_fill_cache = false; + int64_t cache_lines_read_size = 0; + int64_t cache_out_read_size = 0; + LSN read_lsn = lsn; + int64_t real_read_size = in_read_size; + int64_t disk_out_read_size = 0; + ObTenantStatEstGuard tenant_stat_guard(MTL_ID()); + // read process: + // 1. read from kv cache + // 2. deal with miss if miss happens + // 3. read from disk + // 4. fill kv cache when it's allowed + if (OB_SUCC(get_cache_lines_(lsn, flashback_version, + in_read_size, read_buf.buf_, cache_lines_read_size, iterator_info))) { + // read all logs from kv cache successfully + out_read_size += cache_lines_read_size; + } else if (OB_ENTRY_NOT_EXIST != ret) { + PALF_LOG(WARN, "fail to get cache lines", K(ret), K(lsn), K(flashback_version), + K(in_read_size), K(cache_lines_read_size), PRINT_INFO); + } else if (OB_FAIL(allow_filling_cache_(iterator_info, enable_fill_cache))) { + PALF_LOG(WARN, "allow_filling_cache failed", K(ret), K(enable_fill_cache), PRINT_INFO); + } else if (OB_FAIL(deal_with_miss_(enable_fill_cache, cache_lines_read_size, read_lsn, + real_read_size, cache_out_read_size, iterator_info))) { + PALF_LOG(WARN, "fail to deal with miss", K(ret), K(cache_lines_read_size), K(enable_fill_cache), + K(read_lsn), K(real_read_size), K(out_read_size), K(cache_out_read_size), PRINT_INFO); + } else if (FALSE_IT(read_buf.buf_ += cache_out_read_size)) { + } else if (OB_FAIL(read_from_disk_(read_lsn, real_read_size, read_buf, + disk_out_read_size, iterator_info))) { + read_buf.buf_ -= cache_out_read_size; + PALF_LOG(WARN, "read_from_disk_ failed", K(ret), K(read_lsn), K(real_read_size), K(out_read_size)); + } else { + read_buf.buf_ -= cache_out_read_size; + out_read_size = out_read_size + cache_out_read_size + disk_out_read_size; + + if (!enable_fill_cache) { + } else if (OB_FAIL(fill_cache_lines_(flashback_version, read_lsn, disk_out_read_size, read_buf.buf_ + cache_out_read_size))) { + PALF_LOG(WARN, "fail to fill cache", K(ret), K(read_lsn), K(flashback_version), K(out_read_size), PRINT_INFO); + } else if (0 == cache_out_read_size) { + // after cache miss, adjust read_lsn to new_read_lsn to read more log (up to 'diff') for filling first missing cache line + // so, adjust buf_ to ignore 'diff' part before return + offset_t diff = lsn - read_lsn; + out_read_size = out_read_size - diff; + MEMMOVE(read_buf.buf_, read_buf.buf_ + diff, out_read_size); + PALF_LOG(TRACE, "ignore redundant logs in read_buf", K(lsn), K(read_lsn), K(diff), K(out_read_size), K(enable_fill_cache)); + } + } + + #undef PRINT_INFO + + return ret; +} + +int LogColdCache::fill_cache_line(FillBuf &fill_buf) +{ + int ret = OB_SUCCESS; + if (!fill_buf.is_full()) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "can't fill incomplete cache line", K(fill_buf)); + } else if (OB_FAIL(kv_cache_->put_kvpair(fill_buf.inst_handle_, fill_buf.kvpair_, fill_buf.handle_, true /* overwrite*/))) { + PALF_LOG(WARN, "failed to put kvpair", K(ret), K(fill_buf)); + } else { + PALF_LOG(TRACE, "put kvpair into cold cache successfully", K(fill_buf)); + } + + return ret; +} + +int LogColdCache::allow_filling_cache_(LogIteratorInfo *iterator_info, bool &enable_fill_cache) +{ + int ret = OB_SUCCESS; + PalfOptions options; + enable_fill_cache = false; + if (OB_FAIL(palf_env_impl_->get_options(options))) { + PALF_LOG(WARN, "get options failed", K(ret)); + } else { + enable_fill_cache = options.enable_log_cache_ && iterator_info->get_allow_filling_cache(); + } + + return ret; +} + +int LogColdCache::deal_with_miss_(const bool enable_fill_cache, + const int64_t has_read_size, + LSN &lsn, + int64_t &in_read_size, + int64_t &out_read_size, + LogIteratorInfo *iterator_info) +{ + int ret = OB_SUCCESS; + if (!lsn.is_valid() || 0 >= in_read_size || 0 > has_read_size) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid arguement", K(ret), K(lsn), K(in_read_size), K(has_read_size)); + } else if (0 != has_read_size) { + // adjust to statify DIO requirements + int64_t real_read_size = lower_align(has_read_size, LOG_DIO_ALIGN_SIZE); + lsn.val_ += real_read_size; + in_read_size -= real_read_size; + out_read_size = real_read_size; + } else if (enable_fill_cache) { + // adjust lsn to lower_aligned_lsn to fill first miss cache line + LSN new_read_start_lsn = LogCacheUtils::lower_align_with_start(lsn, CACHE_LINE_SIZE); + in_read_size += (lsn - new_read_start_lsn); + lsn = new_read_start_lsn; + } + + iterator_info->inc_miss_cnt(); + log_cache_stat_.inc_miss_cnt(); + PALF_LOG(TRACE, "deal with miss", K(ret), K(enable_fill_cache), K(lsn), K(in_read_size), K(has_read_size), K(out_read_size)); + return ret; +} + +int LogColdCache::get_cache_lines_(const LSN &lsn, + const int64_t flashback_version, + const int64_t in_read_size, + char *buf, + int64_t &out_read_size, + LogIteratorInfo *iterator_info) +{ + #define PRINT_INFO K(palf_id_), K(MTL_ID()) + + int ret = OB_SUCCESS; + int64_t read_pos = 0; + LSN cache_read_lsn = lsn; + + while (OB_SUCC(ret) && out_read_size < in_read_size) { + int64_t curr_round_read_size = 0; + if (OB_FAIL(get_cache_line_(cache_read_lsn, flashback_version, + in_read_size, read_pos, buf, + curr_round_read_size))) { + PALF_LOG(WARN, "fail to get cache line", K(ret), K(cache_read_lsn), + K(flashback_version), K(in_read_size), K(read_pos), + K(out_read_size), PRINT_INFO); + } else { + read_pos += curr_round_read_size; + out_read_size += curr_round_read_size; + cache_read_lsn = cache_read_lsn + curr_round_read_size; + } + } + + if (out_read_size != 0) { + log_cache_stat_.inc_hit_cnt(); + log_cache_stat_.inc_cache_read_size(out_read_size); + + iterator_info->inc_hit_cnt(); + iterator_info->inc_cache_read_size(out_read_size); + } + + log_cache_stat_.print_stat_info(kv_cache_->store_size(MTL_ID()), palf_id_); + + #undef PRINT_INFO + return ret; +} + +int LogColdCache::get_cache_line_(const LSN &cache_read_lsn, + const int64_t flashback_version, + const int64_t in_read_size, + const int64_t read_pos, + char *buf, + int64_t &curr_round_read_size) +{ + #define PRINT_INFO K(palf_id_), K(MTL_ID()) + + int ret = OB_SUCCESS; + curr_round_read_size = 0; + LSN aligned_lsn = LogCacheUtils::lower_align_with_start(cache_read_lsn, CACHE_LINE_SIZE); + offset_t diff = cache_read_lsn - aligned_lsn; + LogKVCacheKey key(MTL_ID(), palf_id_, aligned_lsn, flashback_version); + LogKVCacheValueHandle val_handle; + int tmp_ret = OB_SUCCESS; + char *cache_log_buf = NULL; + if (OB_FAIL(kv_cache_->get_log(key, val_handle))) { + PALF_LOG(WARN, "fail to get log from kv cache", K(ret), K(key), PRINT_INFO); + } else if (OB_ISNULL((cache_log_buf = val_handle.value_->get_buf()))) { + ret = OB_ERR_UNEXPECTED; + PALF_LOG(WARN, "get null buf from log kv cache", K(ret), K(key)); + } else { + int64_t buf_size = val_handle.value_->get_buf_size(); + if (0 == diff) { + // start with aligned_lsn + int64_t remained_size = in_read_size - read_pos; + curr_round_read_size = (remained_size <= buf_size) ? remained_size : buf_size; + } else { + // start with lsn + int64_t remained_cache_line_size = buf_size - diff; + curr_round_read_size = (in_read_size <= remained_cache_line_size) ? in_read_size : remained_cache_line_size; + } + + MEMCPY(buf + read_pos, cache_log_buf + diff, curr_round_read_size); + + PALF_LOG(TRACE, "cache hit, read from log cold cache", K(key), K(read_pos), K(cache_read_lsn), + K(aligned_lsn), K(in_read_size), K(curr_round_read_size), PRINT_INFO); + } + + #undef PRINT_INFO + return ret; +} + +int LogColdCache::fill_cache_lines_(const int64_t flashback_version, + const LSN &lsn, + const int64_t fill_size, + char *buf) +{ + int ret = OB_SUCCESS; + #define PRINT_INFO K(palf_id_), K(MTL_ID()) + + if (!is_valid_flashback_version(flashback_version) || !lsn.is_valid() || 0 >= fill_size) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument", K(ret), K(lsn), K(fill_size), K(flashback_version)); + } else { + // lsn is always expected to be aligned to CACHE_LINE_SIZE + // if not, it means lsn is aligned to 4KB in deal_with_miss() for DIO. This part is already in kvcache, so need to ignore + LSN fill_lsn = LogCacheUtils::is_lower_align_with_start(lsn, CACHE_LINE_SIZE) ? + lsn : LogCacheUtils::upper_align_with_start(lsn, CACHE_LINE_SIZE); + int64_t diff = fill_lsn - lsn; + int64_t fill_pos = diff; + int64_t remained_size = fill_size - diff; + + // only fill in cache in two cases for simplicity: + // 1. remained_size is big enough to fill a complete cache line + // 2. fill an uncomplete cache line only if it's the last cache line in the block + while (OB_SUCC(ret) && 0 < remained_size) { + int64_t curr_round_fill_size = 0; + if (remained_size >= CACHE_LINE_SIZE) { + curr_round_fill_size = CACHE_LINE_SIZE; + } else if (LogCacheUtils::up_to_next_block(fill_lsn, remained_size)) { + // fill last cache line in the block, remained_size is equal to CACHE_LINE_SIZE - 4KB + OB_ASSERT(LAST_CACHE_LINE_SIZE == remained_size); + curr_round_fill_size = remained_size; + } else { + break; + } + + OB_ASSERT(LogCacheUtils::is_lower_align_with_start(fill_lsn, CACHE_LINE_SIZE)); + if (OB_FAIL(fill_cache_line_(flashback_version, fill_lsn, curr_round_fill_size, fill_pos, buf))) { + PALF_LOG(WARN, "fill cache line failed", K(ret), K(flashback_version), + K(fill_lsn), K(curr_round_fill_size), K(fill_pos), PRINT_INFO); + } else { + remained_size -= curr_round_fill_size; + fill_pos += curr_round_fill_size; + fill_lsn = fill_lsn + curr_round_fill_size; + } + } + + if (OB_SIZE_OVERFLOW == ret) { + ret = OB_SUCCESS; + PALF_LOG(TRACE, "LogCache size is up to limit, can't fill logs", K(flashback_version), K(fill_lsn), PRINT_INFO); + } + } + + #undef PRINT_INFO + + return ret; +} + +int LogColdCache::fill_cache_line_(const int64_t flashback_version, + const LSN &fill_lsn, + const int64_t fill_size, + const int64_t fill_pos, + char *buf) +{ + int ret = OB_SUCCESS; + LogKVCacheKey new_key(MTL_ID(), palf_id_, fill_lsn, flashback_version); + LogKVCacheValue new_value; + if (OB_FAIL(new_value.init(buf + fill_pos, fill_size))) { + PALF_LOG(WARN, "new value init failed", K(ret), K(new_key), K(new_value)); + } else if (OB_SUCC(kv_cache_->put_log(new_key, new_value))) { + // fill successfully + PALF_LOG(TRACE, "fill cache successfully", K(fill_lsn), K(fill_pos), K(fill_size), K(palf_id_), K(MTL_ID())); + } else if (OB_ENTRY_EXIST != ret){ + PALF_LOG(WARN, "put log into kv cache failed", K(ret), K(new_key), K(new_value)); + } else { + ret = OB_SUCCESS; + log_cache_stat_.inc_cache_fill_amplification(fill_size); + PALF_LOG(TRACE, "LogKVCacheKey has existed", K(new_key), K(fill_pos), K(fill_size), K(palf_id_), K(MTL_ID())); + } + return ret; +} + +int LogColdCache::read_from_disk_(const LSN &read_lsn, + const int64_t in_read_size, + ReadBuf &read_buf, + int64_t &out_read_size, + LogIteratorInfo *iterator_info) +{ + int ret = OB_SUCCESS; + const block_id_t read_block_id = lsn_2_block(read_lsn, logical_block_size_); + const offset_t real_read_offset = get_phy_offset_(read_lsn); + + if (OB_FAIL(log_reader_->pread(read_block_id, real_read_offset, + in_read_size, read_buf, out_read_size, iterator_info))) { + PALF_LOG(WARN, "read_from_disk failed", K(ret), K(read_lsn), K(in_read_size)); + } else { + PALF_LOG(TRACE, "read_from_disk succeed", K(read_lsn), K(in_read_size), + K(in_read_size), K(read_lsn), K(out_read_size)); + } + return ret; +} + +offset_t LogColdCache::get_phy_offset_(const LSN &lsn) const +{ + return lsn_2_offset(lsn, logical_block_size_) + MAX_INFO_BLOCK_SIZE; +} + +// =======================================LogCacheStat======================================= +LogColdCache::LogCacheStat::LogCacheStat() + : hit_cnt_(0), miss_cnt_(0), cache_read_size_(0), cache_fill_amplification_(0), last_print_time_(0), + last_record_hit_cnt_(0), last_record_miss_cnt_(0), + last_record_cache_read_size_(0) {} + +LogColdCache::LogCacheStat::~LogCacheStat() +{ + reset(); +} + +void LogColdCache::LogCacheStat::reset() +{ + hit_cnt_ = 0; + miss_cnt_ = 0; + cache_read_size_ = 0; + cache_fill_amplification_ = 0; + last_print_time_ = 0; + last_record_hit_cnt_ = 0; + last_record_miss_cnt_ = 0; + last_record_cache_read_size_ = 0; +} + +void LogColdCache::LogCacheStat::inc_hit_cnt() +{ + EVENT_INC(ObStatEventIds::LOG_KV_CACHE_HIT); + ATOMIC_INC(&hit_cnt_); +} + +void LogColdCache::LogCacheStat::inc_miss_cnt() +{ + EVENT_INC(ObStatEventIds::LOG_KV_CACHE_MISS); + ATOMIC_INC(&miss_cnt_); +} + +void LogColdCache::LogCacheStat::inc_cache_read_size(int64_t cache_read_size) +{ + ATOMIC_AAF(&cache_read_size_, cache_read_size); +} + +void LogColdCache::LogCacheStat::inc_cache_fill_amplification(int64_t cache_fill_amplification_) +{ + ATOMIC_AAF(&cache_fill_amplification_, cache_fill_amplification_); +} + +void LogColdCache::LogCacheStat::print_stat_info(int64_t cache_store_size, int64_t palf_id) +{ + if (palf_reach_time_interval(PALF_STAT_PRINT_INTERVAL_US, last_print_time_)) { + int64_t interval_hit_cnt = hit_cnt_ - last_record_hit_cnt_; + int64_t interval_miss_cnt = miss_cnt_ - last_record_miss_cnt_; + int64_t interval_cache_read_size = cache_read_size_ - last_record_cache_read_size_; + int64_t total_cnt = (interval_hit_cnt + interval_miss_cnt == 0) ? 1 : interval_hit_cnt + interval_miss_cnt; + PALF_LOG(INFO, "[PALF STAT LOG COLD CACHE HIT RATE]", "hit_cnt", interval_hit_cnt, + "miss_cnt", interval_miss_cnt, "hit_rate", + interval_hit_cnt * 1.0 / (interval_hit_cnt + interval_miss_cnt), + "cache_read_size", interval_cache_read_size, K(cache_store_size), + K(cache_fill_amplification_), K(palf_id), K(MTL_ID())); + last_record_hit_cnt_ = hit_cnt_; + last_record_miss_cnt_ = miss_cnt_; + last_record_cache_read_size_ = cache_read_size_; + } +} + +// =======================================LogCache======================================= +LogCache::LogCache() : hot_cache_(), cold_cache_(), fill_buf_(), is_inited_(false) {} + +LogCache::~LogCache() +{ + destroy(); +} + +void LogCache::destroy() +{ + palf_id_ = INVALID_PALF_ID; + hot_cache_.destroy(); + cold_cache_.destroy(); + fill_buf_.reset(); + is_inited_ = false; +} + +int LogCache::init(const int64_t palf_id, + IPalfHandleImpl *palf_handle_impl, + IPalfEnvImpl *palf_env_impl, + LogStorage *log_storage) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + PALF_LOG(WARN, "LogCache init failed", K(ret)); + } else if (OB_FAIL(hot_cache_.init(palf_id, palf_handle_impl))){ + PALF_LOG(WARN, "hot cache init failed", K(ret), K(palf_id)); + } else if (OB_FAIL(cold_cache_.init(palf_id, palf_env_impl, log_storage))) { + PALF_LOG(WARN, "cold cache init failed", K(ret), K(palf_id)); + } else { + palf_id_ = palf_id; + is_inited_ = true; + PALF_LOG(INFO, "LogCache init successfully", K(palf_id)); + } + + return ret; +} + +bool LogCache::is_inited() +{ + return is_inited_; +} + +int LogCache::read(const int64_t flashback_version, + const LSN &lsn, + const int64_t in_read_size, + ReadBuf &read_buf, + int64_t &out_read_size, + LogIteratorInfo *iterator_info) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(ERROR, "LogCache is not inited!", K(ret)); + } else if (!lsn.is_valid() || 0 >= in_read_size || !read_buf.is_valid()) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "Invalid argument!!!", K(ret), K(lsn), K(in_read_size), K(read_buf)); + } else if (OB_SUCC(read_hot_cache_(lsn, in_read_size, read_buf.buf_, out_read_size))) { + // read data from hot_cache successfully + } else if (OB_FAIL(read_cold_cache_(flashback_version, lsn, in_read_size, + read_buf, out_read_size, iterator_info))) { + PALF_LOG(WARN, "fail to read from cold cache", K(ret), K(lsn), K(in_read_size), K(read_buf), K(out_read_size)); + } else { + // read data from kv cache successfully + } + + return ret; +} + +int LogCache::fill_cache_when_slide(const LSN &lsn, + const int64_t fill_size, + const int64_t flashback_version) +{ + int ret = OB_SUCCESS; + LSN fill_lsn = lsn; + int64_t remained_size = fill_size; + char *buf = NULL; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(WARN, "LogCache has not been inited", K(ret), K(lsn), K(fill_size), K(flashback_version)); + } else if (!lsn.is_valid() || 0 >= fill_size || 0 > flashback_version) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument", K(ret), K(lsn), K(fill_size), K(flashback_version)); + // make sure that fill_lsn, fill_size, fill_buf_.fill_pos is match before enter 'while' + } else if (OB_FAIL(try_update_fill_buf_(flashback_version, fill_lsn, remained_size))) { + PALF_LOG(WARN, "try update fill_buf failed", K(ret), K(lsn), K(flashback_version), K(fill_buf_), K(remained_size)); + } else { + OB_ASSERT(fill_buf_.is_valid() && fill_lsn == fill_buf_.aligned_lsn_ + fill_buf_.fill_pos_); + + while (OB_SUCC(ret) && 0 < remained_size) { + int64_t out_read_size = 0; + int64_t in_read_size = cal_in_read_size_(remained_size); + + if (OB_ISNULL(buf = fill_buf_.get_writable_buf())) { + ret = OB_ERR_UNEXPECTED; + PALF_LOG(ERROR, "value buf is NULL in fill_buf_", K(ret), K(fill_lsn), K(remained_size)); + } else if (OB_FAIL(read_hot_cache_(fill_lsn, in_read_size, buf, out_read_size)) && OB_READ_NOTHING != ret) { + PALF_LOG(WARN, "read hot cache failed", K(ret), K(fill_lsn), K(in_read_size), K(out_read_size)); + } else if (out_read_size < in_read_size) { + // partitally miss hot cache, give up to fill current cache line + ret = OB_ENTRY_NOT_EXIST; + PALF_LOG(TRACE, "can not read enough logs from group buffer, skip this cache line", + K(ret), K(fill_lsn), K(remained_size), K(in_read_size), K(out_read_size), K(fill_buf_)); + } else { + fill_buf_.fill_pos_ += out_read_size; + remained_size = remained_size - out_read_size; + + if (!fill_buf_.is_full()) { + PALF_LOG(TRACE, "successfully memcpy committed logs from hot cache to fill_buf", + K(fill_lsn), K(remained_size), K(out_read_size), K(fill_buf_)); + } else if (OB_FAIL(cold_cache_.fill_cache_line(fill_buf_))) { + PALF_LOG(WARN, "fill_cache_line failed", K(ret), K(fill_lsn), K(remained_size), K(fill_buf_)); + } else if (FALSE_IT(fill_lsn = fill_lsn + out_read_size)) { + // current cache line is full, update fill_buf for next fill + } else if (OB_FAIL(update_fill_buf_(flashback_version, fill_lsn))) { + PALF_LOG(WARN, "failed to update fill_buf, stop filling cold cache", K(ret), K(flashback_version), K(fill_lsn)); + } else { + OB_ASSERT(fill_lsn == LogCacheUtils::lower_align_with_start(fill_lsn, CACHE_LINE_SIZE)); + PALF_LOG(TRACE, "insert kv into cold cache and update fill_buf successfully", K(fill_lsn), K(remained_size), K(fill_buf_)); + } + } + } + } + + if (OB_FAIL(ret) && fill_buf_.is_valid()) { + fill_buf_.reset(); + } + + return ret; +} + +int LogCache::read_hot_cache_(const LSN &read_begin_lsn, + const int64_t in_read_size, + char *buf, + int64_t &out_read_size) +{ + int ret = OB_SUCCESS; + if (OB_SUCC(hot_cache_.read(read_begin_lsn, in_read_size, buf, out_read_size)) + && out_read_size > 0) { + // read data from hot_cache successfully + PALF_LOG(TRACE, "read hot cache successfully", K(read_begin_lsn), K(in_read_size), K(out_read_size)); + } else if (OB_SUCCESS == ret && 0 == out_read_size){ + ret = OB_READ_NOTHING; + PALF_LOG(WARN, "read nothing from hot cache", K(ret), K(read_begin_lsn), K(in_read_size), K(out_read_size)); + } else { + PALF_LOG(WARN, "read hot cache failed", K(ret), K(read_begin_lsn), K(in_read_size), K(out_read_size)); + } + + return ret; +} + +int LogCache::read_cold_cache_(const int64_t flashback_version, + const LSN &lsn, + const int64_t in_read_size, + ReadBuf &read_buf, + int64_t &out_read_size, + LogIteratorInfo *iterator_info) +{ + int ret = OB_SUCCESS; + if (!lsn.is_valid() || 0 >= in_read_size || !read_buf.is_valid()) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument", K(ret), K(lsn), K(in_read_size), K(read_buf)); + } else if (OB_FAIL(cold_cache_.read(flashback_version, lsn, in_read_size, + read_buf, out_read_size, + iterator_info))) { + PALF_LOG(WARN, "read cold cache failed", K(ret), K(lsn), K(in_read_size), K(read_buf), K(out_read_size)); + } else { + PALF_LOG(TRACE, "read cold cache successfully", K(lsn), K(in_read_size), K(read_buf), K(out_read_size)); + } + + return ret; +} + +// in normal case, fill_lsn should be equal to fill_buf_.aligned_lsn_ + fill_buf_.fill_pos_ +// if not, we should update fill_buf +int LogCache::try_update_fill_buf_(const int64_t flashback_version, + LSN &fill_lsn, + int64_t &fill_size) +{ + int ret = OB_SUCCESS; + LSN aligned_lsn = LogCacheUtils::lower_align_with_start(fill_lsn, CACHE_LINE_SIZE); + + if (!fill_buf_.is_valid()) { + if (aligned_lsn == fill_lsn) { + if (OB_FAIL(update_fill_buf_(flashback_version, fill_lsn))) { + PALF_LOG(WARN, "update fill_buf failed", K(ret), K(fill_lsn), K(aligned_lsn), K(flashback_version), K(fill_buf_)); + } + } else { + ret = OB_DISCONTINUOUS_LOG; + PALF_LOG(WARN, "fill_buf_ is invalid, fill_lsn and aligned_lsn are not match", K(ret), K(fill_lsn), K(aligned_lsn), K(fill_buf_)); + } + // when fill_buf_ is valid, in the two following cases, we can't continue to fill cold cache + // 1. flashback_version changes + // 2. logs are not continuous, which means fill_lsn is not equal to fill_buf_.aligned_lsn_ + fill_buf_.fill_pos_ + } else if (flashback_version != fill_buf_.flashback_version_) { + ret = OB_STATE_NOT_MATCH; + PALF_LOG(WARN, "flashback happened during filling cache, skip it", K(ret), + K(flashback_version), K(fill_buf_)); + } else if (aligned_lsn != fill_buf_.aligned_lsn_ || + fill_buf_.fill_pos_ != fill_lsn - aligned_lsn) { + ret = OB_DISCONTINUOUS_LOG; + PALF_LOG(WARN, "fill_lsn and fill_buf_.aligned_lsn are not match", K(ret), + K(fill_lsn), K(aligned_lsn), K(fill_buf_)); + } + + if (OB_SUCC(ret)) { + } else if (LogCacheUtils::is_in_last_cache_line(fill_lsn)) { + PALF_LOG(TRACE, "give up to fill last cache line", K(fill_lsn), K(aligned_lsn)); + } else { + // Although we can't fill the cache line starting with aligned_lsn + // we can check whether we can start to fill the next cache line + LSN upper_aligned_lsn = LogCacheUtils::upper_align_with_start(fill_lsn, CACHE_LINE_SIZE); + int64_t diff = upper_aligned_lsn - fill_lsn; + if (fill_size <= diff) { + PALF_LOG(WARN, "current group entry is not enough to start next cache line", + K(ret), K(fill_size), K(diff), K(upper_aligned_lsn), K(fill_lsn)); + } else if (FALSE_IT(fill_lsn = fill_lsn + diff)) { + } else if (OB_FAIL(update_fill_buf_(flashback_version, fill_lsn))) { + PALF_LOG(WARN, "update fill_buf failed", K(ret), K(fill_lsn), K(aligned_lsn), K(flashback_version), K(fill_buf_)); + } else { + ret = OB_SUCCESS; + fill_size = fill_size - diff; + + PALF_LOG(TRACE, "skip current cache line, fill next cache line", + K(fill_lsn), K(fill_size), K(diff)); + } + } + + return ret; +} + +int LogCache::update_fill_buf_(const int64_t flashback_version, + LSN &aligned_lsn) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(cold_cache_.alloc_kv_pair(flashback_version, aligned_lsn, fill_buf_))) { + PALF_LOG(WARN, "failed to alloc kv pair", K(ret), K(flashback_version), K(aligned_lsn)); + } else { + fill_buf_.update_state(aligned_lsn, flashback_version); + PALF_LOG(TRACE, "update fill_buf successfully", K(fill_buf_)); + } + return ret; +} + +int64_t LogCache::cal_in_read_size_(const int64_t fill_size) +{ + return (fill_size > fill_buf_.buf_len_ - fill_buf_.fill_pos_) + ? fill_buf_.buf_len_ - fill_buf_.fill_pos_ + : fill_size; +} + } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_cache.h b/src/logservice/palf/log_cache.h index 72b3008eab..ea4ecbf687 100644 --- a/src/logservice/palf/log_cache.h +++ b/src/logservice/palf/log_cache.h @@ -14,13 +14,27 @@ #define OCEANBASE_PALF_LOG_CACHE_ #include // int64_t +#include "share/cache/ob_kv_storecache.h" // ObIKVCache +#include "log_reader_utils.h" // ReadBuf +#include "lsn.h" +#include "log_storage.h" +#include "log_storage_interface.h" // LogIteratorInfo +#define OB_LOG_KV_CACHE oceanbase::palf::LogKVCache::get_instance() namespace oceanbase { namespace palf { class LSN; class IPalfHandleImpl; +class IPalfEnvImpl; +class LogStorage; +class LogEngine; +class LogStateMgr; + +static const int64_t CACHE_LINE_SIZE = LOG_CACHE_ALIGN_SIZE; // 64KB +static const int64_t LAST_CACHE_LINE_SIZE = CACHE_LINE_SIZE - MAX_INFO_BLOCK_SIZE; // 60KB +static const char *const OB_LOG_KV_CACHE_NAME = "log_kv_cache"; class LogHotCache { @@ -44,6 +58,271 @@ private: bool is_inited_; }; +class FillCacheFsCb : public PalfFSCb +{ +public: + FillCacheFsCb(); + ~FillCacheFsCb(); + int init(IPalfEnvImpl *palf_env_impl, LogStateMgr *state_mgr, LogEngine *log_engine); + void destroy(); + int update_end_lsn(int64_t id, + const palf::LSN &end_lsn, + const share::SCN &end_scn, + const int64_t proposal_id) override final; +private: + IPalfEnvImpl *palf_env_impl_; + LogStateMgr *state_mgr_; + LogEngine *log_engine_; + bool is_inited_; +}; + +class LogCacheUtils +{ +public: + LogCacheUtils() {}; + ~LogCacheUtils() {}; +public: + static LSN lower_align_with_start(const LSN &input, const int64_t align); + // @brief: return the next aligned lsn in the same block. + // @param[in] const LSN &input: input should be in the range [block_start_lsn, next_block_start_lsn - CACHE_LINE_SIZE + MAX_INFO_BLOCK_SIZE) + // @param[in] const int64_t align: CACHE_LINE_SIZE + static LSN upper_align_with_start(const LSN &input, const int64_t align); + static bool is_lower_align_with_start(const LSN &input, const int64_t align); + static bool up_to_next_block(const LSN &lsn, const int64_t remained_size); + static LSN next_block_start_lsn(const LSN &input); + static bool is_in_last_cache_line(const LSN &input); +}; + +class LogKVCacheKey : public common::ObIKVCacheKey +{ +public: + LogKVCacheKey(); + LogKVCacheKey(const uint64_t tenant_id, const int64_t palf_id, const LSN aligned_lsn, const int64_t flashback_version); + ~LogKVCacheKey(); + bool is_valid() const; + void reset(); +public: // derived from ObIKVCacheKey + bool operator ==(const ObIKVCacheKey &other) const override; + uint64_t hash() const override; + uint64_t get_tenant_id() const override; + int64_t size() const override; + int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const override; + TO_STRING_KV(K_(tenant_id), K_(palf_id), K_(aligned_lsn), K_(flashback_version)); + +private: + uint64_t tenant_id_; + int64_t palf_id_; + LSN aligned_lsn_; + int64_t flashback_version_; +}; + +class LogKVCacheValue : public common::ObIKVCacheValue +{ +public: + LogKVCacheValue(); + LogKVCacheValue(char *buf, const int64_t buf_size); + ~LogKVCacheValue(); + int init(char *buf, const int64_t buf_size); + void reset(); + bool is_valid() const; + char *get_buf() const; + int64_t get_buf_size() const; +public: // derived from ObIKVCacheValue + int64_t size() const override; + int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const override; + TO_STRING_KV(KP_(buf), K_(buf), K_(buf_size)); +private: + char *buf_; + int64_t buf_size_; +}; + +struct LogKVCacheValueHandle +{ +public: + LogKVCacheValueHandle() : value_(NULL), handle_() {} + ~LogKVCacheValueHandle() {} + void reset() { + handle_.reset(); + value_ = NULL; + } + TO_STRING_KV(K_(value), K_(handle)); + const LogKVCacheValue *value_; + common::ObKVCacheHandle handle_; +}; + +class LogKVCache : public common::ObKVCache +{ +public: + static LogKVCache &get_instance(); + LogKVCache(); + ~LogKVCache(); + int get_log(const LogKVCacheKey &key, LogKVCacheValueHandle &val_handle); + int put_log(const LogKVCacheKey &key, const LogKVCacheValue &value); +}; + +struct FillBuf +{ +public: + FillBuf(); + ~FillBuf(); + void reset(); + bool is_valid(); + char *get_writable_buf(); + bool is_full(); + void update_state(const LSN &aligned_lsn, const int64_t flashback_version); + TO_STRING_KV(K(aligned_lsn_), K(fill_pos_), K(buf_len_)); + LSN aligned_lsn_; + int64_t fill_pos_; + int64_t buf_len_; + int64_t flashback_version_; + char *buf_; + common::ObKVCachePair *kvpair_; + common::ObKVCacheHandle handle_; + common::ObKVCacheInstHandle inst_handle_; +}; + +class LogColdCache +{ +public: + LogColdCache(); + ~LogColdCache(); + int init(int64_t palf_id, + IPalfEnvImpl *palf_env_impl, + LogStorage *log_storage); + void destroy(); + int read(const int64_t flashback_version, + const LSN &lsn, + const int64_t in_read_size, + ReadBuf &read_buf, + int64_t &out_read_size, + LogIteratorInfo *iterator_info); + int fill_cache_line(FillBuf &fill_buf); + int alloc_kv_pair(const int64_t flashback_version, const LSN &aligned_lsn, FillBuf &fill_buf); + TO_STRING_KV(K(is_inited_), K(palf_id_), K(log_cache_stat_)); +private: + int allow_filling_cache_(LogIteratorInfo *iterator_info, bool &enable_fill_cache); + /* + this func is used to adujst read position(lsn) and read size(in_read_size) before reading from the disk: + 1. if read from cache successfully, lsn must be aligned to CACHE_LINE_SIZE in the block. + lsn and in_read_size should be adjusted to statisfy the requirements of DIO. + 2. if cache miss, lsn and in_read_size should be adjusted to cover the first missing cache line. + When it is aligned to the cache line, it must also be aligned to 4K for DIO. + */ + int deal_with_miss_(const bool enable_fill_cache, + const int64_t has_read_size, + LSN &lsn, + int64_t &in_read_size, + int64_t &out_read_size, + LogIteratorInfo *iterator_info); + int get_cache_lines_(const LSN &lsn, + const int64_t flashback_version, + const int64_t in_read_size, + char *buf, + int64_t &out_read_size, + LogIteratorInfo *iterator_info); + int get_cache_line_(const LSN &cache_read_lsn, + const int64_t flashback_version, + const int64_t in_read_size, + const int64_t read_pos, + char *buf, + int64_t &out_read_size); + int fill_cache_lines_(const int64_t flashback_version, + const LSN &lsn, + const int64_t fill_size, + char *buf); + int fill_cache_line_(const int64_t flashback_version, + const LSN &fill_lsn, + const int64_t fill_size, + const int64_t fill_pos, + char *buf); + int read_from_disk_(const LSN &read_lsn, + const int64_t in_read_size, + ReadBuf &read_buf, + int64_t &out_read_size, + LogIteratorInfo *iterator_info); + offset_t get_phy_offset_(const LSN &lsn) const; +private: + class LogCacheStat + { + public: + LogCacheStat(); + ~LogCacheStat(); + void reset(); + void inc_hit_cnt(); + void inc_miss_cnt(); + void inc_cache_read_size(int64_t cache_read_size); + void inc_cache_fill_amplification(int64_t cache_fill_amplification_); + void print_stat_info(int64_t cache_store_size, int64_t palf_id); + TO_STRING_KV(K(hit_cnt_), K(miss_cnt_), K(cache_read_size_), K(cache_fill_amplification_)); + private: + // cache stat for accumulate + int64_t hit_cnt_; + int64_t miss_cnt_; + int64_t cache_read_size_; + int64_t cache_fill_amplification_; + // cache stat for real time + int64_t last_print_time_; + int64_t last_record_hit_cnt_; + int64_t last_record_miss_cnt_; + int64_t last_record_cache_read_size_; + }; +private: + int64_t palf_id_; + IPalfEnvImpl *palf_env_impl_; + LogReader *log_reader_; + LogKVCache *kv_cache_; + int64_t logical_block_size_; + LogCacheStat log_cache_stat_; + bool is_inited_; +}; + +class LogCache +{ +public: + LogCache(); + ~LogCache(); + void destroy(); + int init(const int64_t palf_id, + IPalfHandleImpl *palf_handle_impl, + IPalfEnvImpl *palf_env_impl, + LogStorage *log_storage); + bool is_inited(); + int read(const int64_t flashback_version, + const LSN &lsn, + const int64_t in_read_size, + ReadBuf &read_buf, + int64_t &out_read_size, + LogIteratorInfo *iterator_info); + int fill_cache_when_slide(const LSN &lsn, + const int64_t size, + const int64_t flashback_version); + TO_STRING_KV(K(is_inited_), K(palf_id_), K(cold_cache_)); +private: + int read_hot_cache_(const LSN &read_begin_lsn, + const int64_t in_read_size, + char *buf, + int64_t &out_read_size); + int read_cold_cache_(const int64_t flashback_version, + const LSN &lsn, + const int64_t in_read_size, + ReadBuf &read_buf, + int64_t &out_read_size, + LogIteratorInfo *iterator_info); + int try_update_fill_buf_(const int64_t flashback_version, + LSN &fill_lsn, + int64_t &fill_size); + int update_fill_buf_(const int64_t flashback_version, + LSN &fill_lsn); + int64_t cal_in_read_size_(const int64_t fill_size); +private: + int64_t palf_id_; + LogHotCache hot_cache_; + LogColdCache cold_cache_; + // used for fill cache actively + FillBuf fill_buf_; + bool is_inited_; +}; + } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_define.h b/src/logservice/palf/log_define.h index 65a2cd4bbe..eff5e18b98 100644 --- a/src/logservice/palf/log_define.h +++ b/src/logservice/palf/log_define.h @@ -157,6 +157,11 @@ constexpr int64_t LOG_BATCH_PUSH_LOG_REQ = 1; constexpr int64_t LOG_BATCH_PUSH_LOG_RESP = 2; // =========== BatchRPC end ================== +// ========== LogCache start ================= +constexpr offset_t LOG_CACHE_ALIGN_SIZE = 64 * 1024; +constexpr int64_t LOG_CACHE_MEMORY_LIMIT = 20; // memory limit ratio with tenant memory +// ========== LogCache end ================= + const int64_t OB_INVALID_CONFIG_CHANGE_LOCK_OWNER = -1; enum ObReplicaState { @@ -330,6 +335,10 @@ inline bool is_valid_file_desc(const FileDesc &fd) return 0 <= fd; } +inline bool is_valid_flashback_version(const int64_t flashback_version) +{ + return 0 <= flashback_version; +} int block_id_to_string(const block_id_t block_id, char *str, diff --git a/src/logservice/palf/log_engine.cpp b/src/logservice/palf/log_engine.cpp index d7b747cc2a..052e627c70 100644 --- a/src/logservice/palf/log_engine.cpp +++ b/src/logservice/palf/log_engine.cpp @@ -90,7 +90,7 @@ int LogEngine::init(const int64_t palf_id, const LogMeta &log_meta, ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, - LogHotCache *hot_cache, + LogCache *log_cache, LogRpc *log_rpc, LogIOWorker *log_io_worker, LogSharedQueueTh *log_shared_queue_th, @@ -120,7 +120,7 @@ int LogEngine::init(const int64_t palf_id, K(palf_id), K(base_dir), K(log_meta), - K(hot_cache), + K(log_cache), K(alloc_mgr), K(log_io_worker), K(log_shared_queue_th), @@ -136,7 +136,7 @@ int LogEngine::init(const int64_t palf_id, log_meta_storage_update_manifest_cb, log_block_pool, plugins, - NULL /*set hot_cache to NULL for meta storage*/))) { + NULL /*set log_cache to NULL for meta storage*/))) { PALF_LOG(ERROR, "LogMetaStorage init failed", K(ret), K(palf_id), K(base_dir)); } else if(0 != log_storage_block_size && OB_FAIL(log_storage_.init(base_dir, @@ -149,7 +149,7 @@ int LogEngine::init(const int64_t palf_id, log_storage_update_manifest_cb, log_block_pool, plugins, - hot_cache))) { + log_cache))) { PALF_LOG(ERROR, "LogStorage init failed!!!", K(ret), K(palf_id), K(base_dir), K(log_meta)); } else if (OB_FAIL(log_net_service_.init(palf_id, log_rpc))) { PALF_LOG(ERROR, "LogNetService init failed", K(ret), K(palf_id)); @@ -199,7 +199,7 @@ int LogEngine::load(const int64_t palf_id, const char *base_dir, common::ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, - LogHotCache *hot_cache, + LogCache *log_cache, LogRpc *log_rpc, LogIOWorker *log_io_worker, LogSharedQueueTh *log_shared_queue_th, @@ -240,7 +240,7 @@ int LogEngine::load(const int64_t palf_id, } else if (false == is_valid_palf_id(palf_id) || OB_ISNULL(base_dir) || OB_ISNULL(alloc_mgr) || OB_ISNULL(log_rpc) || OB_ISNULL(log_io_worker)) { ret = OB_INVALID_ARGUMENT; - PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(base_dir), K(hot_cache), K(alloc_mgr), + PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(base_dir), K(log_cache), K(alloc_mgr), K(log_io_worker)); } else if (OB_FAIL(log_meta_storage_.load(base_dir, "meta", @@ -252,7 +252,7 @@ int LogEngine::load(const int64_t palf_id, log_meta_storage_update_manifest_cb, log_block_pool, plugins, - NULL, /*set hot_cache to NULL for meta storage*/ + NULL, /*set log_cache to NULL for meta storage*/ unused_meta_entry_header, last_meta_entry_start_lsn))) { PALF_LOG(ERROR, "LogMetaStorage load failed", K(ret), K(palf_id)); @@ -265,7 +265,7 @@ int LogEngine::load(const int64_t palf_id, log_storage_block_size, LOG_DIO_ALIGN_SIZE, LOG_DIO_ALIGNED_BUF_SIZE_REDO, log_storage_update_manifest_cb, log_block_pool, plugins, - hot_cache, entry_header, last_group_entry_header_lsn)))) { + log_cache, entry_header, last_group_entry_header_lsn)))) { PALF_LOG(ERROR, "LogStorage load failed", K(ret), K(palf_id), K(base_dir)); } else if (FALSE_IT(guard.click("load log_storage")) || (0 != log_storage_block_size @@ -597,6 +597,38 @@ int LogEngine::submit_purge_throttling_task(const PurgeThrottlingType purge_type return ret; } +int LogEngine::submit_fill_cache_task(const LSN &lsn, const int64_t size) +{ + int ret = OB_SUCCESS; + LogFillCacheTask *fill_cache_task = NULL; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(ERROR, "LogEngine is not inited!!!", K(ret), KPC(this)); + } else if (OB_ISNULL(log_shared_queue_th_)) { + ret = OB_ERR_UNEXPECTED; + PALF_LOG(ERROR, "log_shared_queue_th_ is NULL", K(ret), KPC(this)); + } else if (OB_FAIL(generate_fill_cache_task_(lsn, size, fill_cache_task))) { + PALF_LOG(WARN, "generate fill cache task failed", K(ret), K(lsn), K(size)); + } else if (OB_FAIL(log_shared_queue_th_->push_task(fill_cache_task))) { + if (OB_IN_STOP_STATE == ret) { + if (REACH_TIME_INTERVAL(100 * 1000)) { + PALF_LOG(WARN, "push task failed", K(ret), KPC(this), KPC(fill_cache_task)); + } + } else { + PALF_LOG(ERROR, "push task failed", K(ret), KPC(this), KPC(fill_cache_task)); + } + } else { + PALF_LOG(TRACE, "log_shared_queue_th_->push_task success", K(ret), KPC(this)); + } + + if (OB_FAIL(ret) && OB_NOT_NULL(fill_cache_task)) { + alloc_mgr_->free_log_fill_cache_task(fill_cache_task); + fill_cache_task = NULL; + } + + return ret; +} + // ====================== LogStorage start ===================== int LogEngine::append_log(const LSN &lsn, const LogWriteBuf &write_buf, const SCN &scn) { @@ -646,7 +678,7 @@ int LogEngine::read_log(const LSN &lsn, } else if (OB_FAIL(log_storage_.pread(lsn, in_read_size, read_buf, out_read_size, io_ctx))) { PALF_LOG(ERROR, "LogEngine read_log failed", K(ret), K(lsn), K(in_read_size), K(read_buf)); } else { - PALF_LOG(TRACE, "LogEngine read_log success", K(ret), K(lsn), K(read_buf), K(out_read_size)); + PALF_LOG(TRACE, "LogEngine read_log success", K(ret), K(lsn), K(read_buf), K(out_read_size), K(io_ctx)); } return ret; } @@ -658,9 +690,10 @@ int LogEngine::read_group_entry_header(const LSN &lsn, LogGroupEntryHeader &log_ const int64_t in_read_size = MAX_LOG_HEADER_SIZE; ReadBufGuard read_buf_guard("LogEngine", in_read_size); ReadBuf &read_buf = read_buf_guard.read_buf_; + LogIOContext io_ctx(LogIOUser::DEFAULT); + io_ctx.set_allow_filling_cache(false); int64_t out_read_size = 0; int64_t pos = 0; - LogIOContext io_ctx(LogIOUser::DEFAULT); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -948,6 +981,19 @@ int LogEngine::get_total_used_disk_space(int64_t &total_used_size_byte, return ret; } +int LogEngine::fill_cache_when_slide(const LSN &begin_lsn, const int64_t size) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(WARN, "LogEngine is not inited", K(ret), KPC(this)); + } else if (OB_FAIL(log_storage_.fill_cache_when_slide(begin_lsn, size))) { + PALF_LOG(WARN, "fill_cache_when_slide failed", K(ret), K(begin_lsn), K(size)); + } + + return ret; +} + int LogEngine::raw_read(const LSN &lsn, const int64_t in_read_size, const bool need_read_block_header, @@ -965,7 +1011,7 @@ int LogEngine::raw_read(const LSN &lsn, ret = OB_INVALID_ARGUMENT; PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K_(palf_id), K_(is_inited), K(lsn), K(in_read_size), K(read_buf)); } else if (need_read_block_header) { - ret = log_storage_.pread_with_block_header(lsn, in_read_size, read_buf, out_read_size); + ret = log_storage_.pread_with_block_header(lsn, in_read_size, read_buf, out_read_size, io_ctx); } else if (!need_read_block_header) { ret = log_storage_.pread(lsn, in_read_size, read_buf, out_read_size, io_ctx); } else {} @@ -1588,6 +1634,31 @@ int LogEngine::generate_purge_throttling_task_(const PurgeThrottlingCbCtx &purge return ret; } +int LogEngine::generate_fill_cache_task_(const LSN &lsn, + const int64_t size, + LogFillCacheTask *&fill_cache_task) +{ + int ret = OB_SUCCESS; + fill_cache_task = NULL; + + if (!lsn.is_valid() || 0 >= size) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument", K(ret), K(lsn), K(size)); + } else if (NULL == (fill_cache_task = alloc_mgr_->alloc_log_fill_cache_task(palf_id_, palf_epoch_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + PALF_LOG(WARN, "alloc LogFillCacheTask failed", K(ret), K(palf_id_), K(palf_epoch_), K(lsn), K(size)); + } else if (OB_FAIL(fill_cache_task->init(lsn, size))) { + PALF_LOG(WARN, "LogFillCacheTask init failed", K(ret), K(lsn), K(size)); + } + + if (OB_FAIL(ret) && OB_NOT_NULL(fill_cache_task)) { + alloc_mgr_->free_log_fill_cache_task(fill_cache_task); + fill_cache_task = NULL; + } + return ret; + +} + int LogEngine::serialize_log_meta_(const LogMeta& log_meta, char *buf, int64_t buf_len) { int ret = OB_SUCCESS; diff --git a/src/logservice/palf/log_engine.h b/src/logservice/palf/log_engine.h index 5a9e1f3b2c..9b48abbe30 100644 --- a/src/logservice/palf/log_engine.h +++ b/src/logservice/palf/log_engine.h @@ -54,6 +54,7 @@ class LogIOFlashbackTask; class FlashbackCbCtx; class LogIOPurgeThrottlingTask; class PurgeThrottlingCbCtx; +class LogFillCacheTask; #define OVERLOAD_SUBMIT_CHANGE_CONFIG_META_REQ(type) \ virtual int submit_change_config_meta_req(const type &member_list, \ @@ -102,7 +103,7 @@ public: const LogMeta &log_meta, common::ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, - LogHotCache *hot_cache, + LogCache *log_cache, LogRpc *log_rpc, LogIOWorker *log_io_worker, LogSharedQueueTh *log_shared_queue_th, @@ -116,7 +117,7 @@ public: const char *base_dir, common::ObILogAllocator *alloc_mgr, ILogBlockPool *log_block_pool, - LogHotCache *hot_cache, + LogCache *log_cache, LogRpc *log_rpc, LogIOWorker *log_io_worker, LogSharedQueueTh *log_shared_queue_th, @@ -157,6 +158,7 @@ public: const TruncatePrefixBlocksCbCtx &truncate_prefix_blocks_ctx); int submit_flashback_task(const FlashbackCbCtx &flashback_ctx); int submit_purge_throttling_task(const PurgeThrottlingType purge_type); + int submit_fill_cache_task(const LSN &lsn, const int64_t size); // ==================== Submit aysnc task end ================== @@ -177,6 +179,7 @@ public: const LSN get_begin_lsn() const; int get_block_id_range(block_id_t &min_block_id, block_id_t &max_block_id) const; int get_block_min_scn(const block_id_t &block_id, share::SCN &scn) const; + int fill_cache_when_slide(const LSN &begin_lsn, const int64_t size); int raw_read(const LSN &lsn, const int64_t in_read_size, const bool need_read_block_header, @@ -468,6 +471,9 @@ private: LogIOFlashbackTask *&flashback_task); int generate_purge_throttling_task_(const PurgeThrottlingCbCtx &purge_cb_ctx, LogIOPurgeThrottlingTask *&purge_task); + int generate_fill_cache_task_(const LSN &lsn, + const int64_t size, + LogFillCacheTask *&fill_cache_task); int update_config_meta_guarded_by_lock_(const LogConfigMeta &meta, LogMeta &log_meta); int try_clear_up_holes_and_check_storage_integrity_( const LSN &last_entry_begin_lsn, diff --git a/src/logservice/palf/log_io_context.h b/src/logservice/palf/log_io_context.h index 9d3cafc1c0..c030907e2c 100644 --- a/src/logservice/palf/log_io_context.h +++ b/src/logservice/palf/log_io_context.h @@ -14,6 +14,7 @@ #define OCEANBASE_LOGSERVICE_LOG_IO_CONTEXT_ #include #include "lib/utility/ob_print_utils.h" +#include "log_iterator_info.h" namespace oceanbase { @@ -30,7 +31,9 @@ enum class LogIOUser { SHARED_UPLOAD = 7, META_INFO = 8, RESTART = 9, - OTHER = 10, + FLASHBACK = 10, + RECOVERY = 11, + OTHER = 12, }; inline const char *log_io_user_str(const LogIOUser user_type) @@ -48,6 +51,8 @@ inline const char *log_io_user_str(const LogIOUser user_type) USER_TYPE_STR(SHARED_UPLOAD); USER_TYPE_STR(META_INFO); USER_TYPE_STR(RESTART); + USER_TYPE_STR(FLASHBACK); + USER_TYPE_STR(RECOVERY); USER_TYPE_STR(OTHER); default: return "Invalid"; @@ -58,16 +63,33 @@ inline const char *log_io_user_str(const LogIOUser user_type) class LogIOContext { public: - LogIOContext(const LogIOUser &user) : user_(user) { } + LogIOContext(const LogIOUser &user) : palf_id_(INVALID_PALF_ID), user_(user), iterator_info_() {} + LogIOContext(const int64_t palf_id, const LogIOUser &user) : palf_id_(palf_id), user_(user), iterator_info_() {} ~LogIOContext() { destroy(); } void destroy() { user_ = LogIOUser::DEFAULT; + iterator_info_.reset(); } - TO_STRING_KV("user", log_io_user_str(user_)); + void set_user_type(const LogIOUser user) { + user_ = user; + } + void set_palf_id(const int64_t palf_id) { palf_id_ = palf_id; } + void set_allow_filling_cache(const bool allow_filling_cache) { + iterator_info_.set_allow_filling_cache(allow_filling_cache); + } + void set_start_lsn(const LSN &start_lsn) { + iterator_info_.set_start_lsn(start_lsn); + } + LogIteratorInfo *get_iterator_info() { + return &iterator_info_; + } + TO_STRING_KV("user", log_io_user_str(user_), K(palf_id_), K(iterator_info_)); private: + int64_t palf_id_; LogIOUser user_; + LogIteratorInfo iterator_info_; }; } } diff --git a/src/logservice/palf/log_iterator_impl.h b/src/logservice/palf/log_iterator_impl.h index 542bf414e6..129f66e60f 100644 --- a/src/logservice/palf/log_iterator_impl.h +++ b/src/logservice/palf/log_iterator_impl.h @@ -115,7 +115,7 @@ public: // same as the accumulate checksum of LogGroupEntry // OB_PARTIAL_LOG // - this replica has not finished flashback, and iterator start lsn is not the header of LogGroupEntry. - int next(const share::SCN &replayable_point_scn); + int next(const share::SCN &replayable_point_scn, LogIOContext &io_ctx); // param[in] replayable point scn, iterator will ensure that no log will return when the log scn is greater than // 'replayable_point_scn' and the log is raw write @@ -143,7 +143,8 @@ public: // - this replica has not finished flashback, and iterator start lsn is not the header of LogGroupEntry. int next(const share::SCN &replayable_point_scn, share::SCN &next_min_scn, - bool &iterate_end_by_replayable_point); + bool &iterate_end_by_replayable_point, + LogIOContext &io_ctx); // @retval // OB_SUCCESS // OB_INVALID_DATA @@ -154,7 +155,7 @@ public: int get_entry(ENTRY &entry, LSN &lsn, bool &is_raw_write); bool is_valid() const; - bool check_is_the_last_entry(); + bool check_is_the_last_entry(LogIOContext &io_ctx); LSN get_curr_read_lsn() const; @@ -174,7 +175,8 @@ private: // OB_PARTIAL_LOG: this replica has not finished flashback, and iterator start lsn // is not the header of LogGroupEntry. int get_next_entry_(const SCN &replayable_point_scn, - IterateEndInfo &info); + IterateEndInfo &info, + LogIOContext &io_ctx); // According to LogEntryType, deserialize different log entry // The log format @@ -278,7 +280,7 @@ private: // OB_SUCCESS. // OB_ITER_END. // OB_ERR_OUT_LOWER_BOUND - int read_data_from_storage_(); + int read_data_from_storage_(LogIOContext &io_ctx); void advance_read_lsn_(const offset_t step); void try_clean_up_cache_(); @@ -556,7 +558,8 @@ LSN LogIteratorImpl::get_curr_read_lsn() const // NB: for restarting, the committed offset of sliding window is invalid. template int LogIteratorImpl::get_next_entry_(const SCN &replayable_point_scn, - IterateEndInfo &info) + IterateEndInfo &info, + LogIOContext &io_ctx) { int ret = OB_SUCCESS; // NB: check need read next enty @@ -586,7 +589,7 @@ int LogIteratorImpl::get_next_entry_(const SCN &replayable_point_scn, if (OB_SUCC(parse_one_entry_(replayable_point_scn, info))) { curr_entry_size_ = curr_entry_.get_serialize_size(); } else if (OB_BUF_NOT_ENOUGH == ret) { - if (OB_FAIL(read_data_from_storage_()) && OB_ITER_END != ret + if (OB_FAIL(read_data_from_storage_(io_ctx)) && OB_ITER_END != ret && OB_ERR_OUT_OF_LOWER_BOUND != ret) { PALF_LOG(WARN, "read_data_from_storage_ failed", K(ret), KPC(this)); } else if (OB_ITER_END == ret) { @@ -622,17 +625,18 @@ int LogIteratorImpl::get_next_entry_(const SCN &replayable_point_scn, } template -int LogIteratorImpl::next(const share::SCN &replayable_point_scn) +int LogIteratorImpl::next(const share::SCN &replayable_point_scn, LogIOContext &io_ctx) { share::SCN next_min_scn; bool unused_bool = false; - return next(replayable_point_scn, next_min_scn, unused_bool); + return next(replayable_point_scn, next_min_scn, unused_bool, io_ctx); } template int LogIteratorImpl::next(const share::SCN &replayable_point_scn, share::SCN &next_min_scn, - bool &iterate_end_by_replayable_point) + bool &iterate_end_by_replayable_point, + LogIOContext &io_ctx) { int ret = OB_SUCCESS; next_min_scn.reset(); @@ -655,7 +659,7 @@ int LogIteratorImpl::next(const share::SCN &replayable_point_scn, if (!replayable_point_scn.is_valid()) { ret = OB_INVALID_ARGUMENT; PALF_LOG(WARN, "invalid argument", K(replayable_point_scn), KPC(this)); - } else if (OB_FAIL(get_next_entry_(replayable_point_scn, info))) { + } else if (OB_FAIL(get_next_entry_(replayable_point_scn, info, io_ctx))) { // NB: if the data which has been corrupted or accum_checksum_ is not match, clean cache. if (need_clean_cache_(ret)) { PALF_LOG(WARN, "read invalid data, need clean cache, maybe storage device cann't guarantee linear consistency reading", @@ -877,7 +881,7 @@ void LogIteratorImpl::advance_read_lsn_(const offset_t step) // @brief, start from `curr_read_lsn_`, check whether the entry is valid. template -bool LogIteratorImpl::check_is_the_last_entry() +bool LogIteratorImpl::check_is_the_last_entry(LogIOContext &io_ctx) { bool bool_ret = false; int ret = OB_SUCCESS; @@ -896,7 +900,7 @@ bool LogIteratorImpl::check_is_the_last_entry() using LogEntryHeaderType = typename ENTRY::LogEntryHeaderType; LogEntryHeaderType header; const int64_t header_size = header.get_serialize_size(); - if (OB_FAIL(read_data_from_storage_()) && OB_ITER_END != ret) { + if (OB_FAIL(read_data_from_storage_(io_ctx)) && OB_ITER_END != ret) { PALF_LOG(ERROR, "read_data_from_storage_ failed", K(ret), KPC(this)); } else if (OB_ITER_END == ret) { PALF_LOG(INFO, "has iterate end", K(ret), KPC(this)); @@ -987,12 +991,12 @@ int LogIteratorImpl::get_log_entry_type_(LogEntryType &log_entry_type) // // NB: when iterate to the end of block, need switch to next block. template -int LogIteratorImpl::read_data_from_storage_() +int LogIteratorImpl::read_data_from_storage_(LogIOContext &io_ctx) { int ret = OB_SUCCESS; int64_t out_read_size = 0; if (OB_FAIL(log_storage_->pread(curr_read_pos_, next_round_pread_size_, buf_, - out_read_size)) + out_read_size, io_ctx)) && OB_ERR_OUT_OF_UPPER_BOUND != ret) { PALF_LOG(WARN, "IteratorStorage pread failed", K(ret), KPC(this)); } else if (OB_ERR_OUT_OF_UPPER_BOUND == ret) { diff --git a/src/logservice/palf/log_iterator_info.h b/src/logservice/palf/log_iterator_info.h new file mode 100644 index 0000000000..8e28ee3bd6 --- /dev/null +++ b/src/logservice/palf/log_iterator_info.h @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#include // UINT64_MAX +#include "lib/ob_errno.h" // errno +#include "lib/utility/ob_print_utils.h" // TO_STRING_KV +#include "log_define.h" +#include "lsn.h" +namespace oceanbase +{ +namespace palf +{ +class LogIteratorInfo +{ +public: + LogIteratorInfo() + : allow_filling_cache_(true), hit_cnt_(0), miss_cnt_(0), cache_read_size_(0), + read_io_cnt_(0), read_io_size_(0), read_disk_cost_ts_(0) {} + LogIteratorInfo(bool allow_filling_cache) + : allow_filling_cache_(allow_filling_cache), hit_cnt_(0), miss_cnt_(0), + cache_read_size_(0), read_io_cnt_(0), read_io_size_(0), + read_disk_cost_ts_(0) {} + ~LogIteratorInfo() { + reset(); + } + void reset() { + allow_filling_cache_ = false; + hit_cnt_ = 0; + miss_cnt_ = 0; + cache_read_size_ = 0; + read_io_cnt_ = 0; + read_io_size_ = 0; + read_disk_cost_ts_ = 0; + } + bool get_allow_filling_cache() const { + return allow_filling_cache_; + } + void set_allow_filling_cache(const bool allow_filling_cache) { + allow_filling_cache_ = allow_filling_cache; + } + void inc_hit_cnt() { hit_cnt_++; } + void inc_miss_cnt() { miss_cnt_++; } + void inc_cache_read_size(int64_t cache_read_size) { cache_read_size_ += cache_read_size; } + void inc_read_io_cnt() { read_io_cnt_++; } + void inc_read_io_size(int64_t read_io_size) { read_io_size_ += read_io_size; } + void inc_read_disk_cost_ts(int64_t read_disk_cost_ts) { read_disk_cost_ts_ += read_disk_cost_ts; } + double get_hit_ratio() const + { + int64_t total_cnt = (hit_cnt_ + miss_cnt_ == 0) ? 1 : hit_cnt_ + miss_cnt_; + return hit_cnt_ * 1.0 /total_cnt; + } + void set_start_lsn(const LSN &start_lsn) { start_lsn_ = start_lsn; } + TO_STRING_KV(K(allow_filling_cache_), K(hit_cnt_), K(miss_cnt_), + K(cache_read_size_), K(read_io_cnt_), K(read_io_size_), + K(read_disk_cost_ts_), "hit rate", get_hit_ratio(), K(start_lsn_)); + +private: + bool allow_filling_cache_; + int64_t hit_cnt_; + int64_t miss_cnt_; + int64_t cache_read_size_; + int64_t read_io_cnt_; + int64_t read_io_size_; + int64_t read_disk_cost_ts_; + int64_t thread_id_; + LSN start_lsn_; +}; +} +} \ No newline at end of file diff --git a/src/logservice/palf/log_iterator_storage.cpp b/src/logservice/palf/log_iterator_storage.cpp index bafea52545..085b9a38d6 100644 --- a/src/logservice/palf/log_iterator_storage.cpp +++ b/src/logservice/palf/log_iterator_storage.cpp @@ -78,7 +78,8 @@ int IteratorStorage::pread( const int64_t pos, const int64_t in_read_size, char *&buf, - int64_t &out_read_size) + int64_t &out_read_size, + LogIOContext &io_ctx) { int ret = OB_SUCCESS; const int64_t real_in_read_size = MIN(in_read_size, get_file_end_lsn_() - (start_lsn_ + pos)); @@ -95,7 +96,7 @@ int IteratorStorage::pread( } else if (0 == real_in_read_size) { ret = OB_ITER_END; PALF_LOG(WARN, "IteratorStorage has iterate end", K(ret), KPC(this)); - } else if (OB_FAIL(read_data_from_storage_(real_pos, real_in_read_size, buf, out_read_size))) { + } else if (OB_FAIL(read_data_from_storage_(real_pos, real_in_read_size, buf, out_read_size, io_ctx))) { PALF_LOG(WARN, "read_data_from_storage_ failed", K(ret), K(pos), K(in_read_size), KP(buf), KPC(this)); } else { start_lsn_ = start_lsn_ + real_pos; @@ -199,11 +200,11 @@ int MemoryIteratorStorage::read_data_from_storage_( int64_t &pos, const int64_t in_read_size, char *&buf, - int64_t &out_read_size) + int64_t &out_read_size, + LogIOContext &io_ctx) { int ret = OB_SUCCESS; const LSN start_lsn = start_lsn_ + pos; - LogIOContext io_ctx(LogIOUser::DEFAULT); if (OB_FAIL(log_storage_->pread(start_lsn, in_read_size, read_buf_, out_read_size, io_ctx))) { PALF_LOG(WARN, "MemoryIteratorStorage pread failed", K(ret), KPC(this), K(start_lsn)); } else { @@ -228,7 +229,8 @@ int DiskIteratorStorage::read_data_from_storage_( int64_t &pos, const int64_t in_read_size, char *&buf, - int64_t &out_read_size) + int64_t &out_read_size, + LogIOContext &io_ctx) { int ret = OB_SUCCESS; int64_t remain_valid_data_size = 0; @@ -239,7 +241,6 @@ int DiskIteratorStorage::read_data_from_storage_( const LSN curr_round_read_lsn = start_lsn_ + pos + remain_valid_data_size; const int64_t real_in_read_size = in_read_size - remain_valid_data_size; read_buf_.buf_ += remain_valid_data_size; - LogIOContext io_ctx(LogIOUser::DEFAULT); if (0ul == real_in_read_size) { ret = OB_ERR_UNEXPECTED; PALF_LOG(ERROR, "real read size is zero, unexpected error!!!", K(ret), K(real_in_read_size)); @@ -266,7 +267,7 @@ int DiskIteratorStorage::ensure_memory_layout_correct_( int64_t &remain_valid_data_size) { int ret = OB_SUCCESS; - const int64_t max_valid_buf_len = read_buf_.buf_len_ - LOG_DIO_ALIGN_SIZE; + const int64_t max_valid_buf_len = read_buf_.buf_len_ - LOG_DIO_ALIGN_SIZE - LOG_CACHE_ALIGN_SIZE; ReadBuf tmp_read_buf = read_buf_; // buf not enough, need alloc or expand if (in_read_size > max_valid_buf_len) { diff --git a/src/logservice/palf/log_iterator_storage.h b/src/logservice/palf/log_iterator_storage.h index 91c9585920..fe8dfcf981 100644 --- a/src/logservice/palf/log_iterator_storage.h +++ b/src/logservice/palf/log_iterator_storage.h @@ -44,7 +44,8 @@ public: int pread(const int64_t pos, const int64_t in_read_size, char *&buf, - int64_t &out_read_size); + int64_t &out_read_size, + LogIOContext &io_ctx); VIRTUAL_TO_STRING_KV(K_(start_lsn), K_(end_lsn), K_(read_buf), K_(block_size), KP(log_storage_)); protected: inline int64_t get_valid_data_len_() @@ -52,7 +53,8 @@ protected: virtual int read_data_from_storage_(int64_t &pos, const int64_t in_read_size, char *&buf, - int64_t &out_read_size) = 0; + int64_t &out_read_size, + LogIOContext &io_ctx) = 0; protected: // update after read_data_from_storage // 'start_lsn_' is the base position @@ -92,7 +94,8 @@ private: int read_data_from_storage_(int64_t &pos, const int64_t in_read_size, char *&buf, - int64_t &out_read_size) final; + int64_t &out_read_size, + LogIOContext &io_ctx) final; public: INHERIT_TO_STRING_KV("IteratorStorage", IteratorStorage, "IteratorStorageType:", "MemoryIteratorStorage"); }; @@ -112,7 +115,8 @@ private: int64_t &pos, const int64_t in_read_size, char *&buf, - int64_t &out_read_size) final; + int64_t &out_read_size, + LogIOContext &io_ctx) final; int ensure_memory_layout_correct_(const int64_t pos, const int64_t in_read_size, int64_t &remain_valid_data_size); void do_memove_(ReadBuf &dst, const int64_t pos, int64_t &valid_tail_part_size); diff --git a/src/logservice/palf/log_reader.cpp b/src/logservice/palf/log_reader.cpp index cf5933b229..5caa714b75 100644 --- a/src/logservice/palf/log_reader.cpp +++ b/src/logservice/palf/log_reader.cpp @@ -25,9 +25,10 @@ using namespace common; namespace palf { -LogReader::LogReader() : is_inited_(false) -{ -} +LogReader::LogReader() + : last_accum_read_statistic_time_(OB_INVALID_TIMESTAMP), + accum_read_io_count_(0), accum_read_log_size_(0), accum_read_cost_ts_(0), + is_inited_(false) {} LogReader::~LogReader() { @@ -41,6 +42,7 @@ int LogReader::init(const char *log_dir, const offset_t block_size) } else { block_size_ = block_size; MEMCPY(log_dir_, log_dir, OB_MAX_FILE_NAME_LENGTH); + last_accum_read_statistic_time_ = ObTimeUtility::fast_current_time(); is_inited_ = true; } if (false == is_inited_) { @@ -55,6 +57,10 @@ void LogReader::destroy() is_inited_ = false; block_size_ = 0; MEMSET(log_dir_, '\0', OB_MAX_FILE_NAME_LENGTH); + last_accum_read_statistic_time_ = OB_INVALID_TIMESTAMP; + accum_read_io_count_ = 0; + accum_read_log_size_ = 0; + accum_read_cost_ts_ = 0; } } @@ -62,7 +68,8 @@ int LogReader::pread(const block_id_t block_id, const offset_t offset, int64_t in_read_size, ReadBuf &read_buf, - int64_t &out_read_size) const + int64_t &out_read_size, + LogIteratorInfo *iterator_info) const { int ret = OB_SUCCESS; int read_io_fd = -1; @@ -89,7 +96,7 @@ int LogReader::pread(const block_id_t block_id, const offset_t curr_read_offset = offset + in_read_size - remained_read_size; const int64_t curr_read_buf_len = remained_read_buf_len - out_read_size; int64_t curr_out_read_size = 0; - if (OB_FAIL(inner_pread_(read_io_fd, curr_read_offset, curr_in_read_size, curr_read_buf, curr_read_buf_len, curr_out_read_size))) { + if (OB_FAIL(inner_pread_(read_io_fd, curr_read_offset, curr_in_read_size, curr_read_buf, curr_read_buf_len, curr_out_read_size, iterator_info))) { PALF_LOG(WARN, "LogReader inner_pread_ failed", K(ret), K(read_io_fd), K(block_id), K(offset), K(in_read_size), K(read_buf), K(curr_in_read_size), K(curr_read_offset), K(curr_out_read_size), K(remained_read_size), K(block_path)); @@ -105,7 +112,7 @@ int LogReader::pread(const block_id_t block_id, if (-1 != read_io_fd && -1 == ::close(read_io_fd)) { ret = convert_sys_errno(); - PALF_LOG(ERROR, "close read_io_fd failed", K(ret), K(read_io_fd), K(errno)); + PALF_LOG(ERROR, "close read_io_fd failed", K(ret), K(read_io_fd)); } return ret; } @@ -115,7 +122,8 @@ int LogReader::inner_pread_(const int read_io_fd, int64_t in_read_size, char *read_buf, const int64_t read_buf_len, - int64_t &out_read_size) const + int64_t &out_read_size, + LogIteratorInfo *iterator_info) const { int ret = OB_SUCCESS; offset_t aligned_start_offset = lower_align(start_offset, LOG_DIO_ALIGN_SIZE); @@ -140,6 +148,9 @@ int LogReader::inner_pread_(const int read_io_fd, K(ret), K(read_io_fd), K(aligned_start_offset), K(backoff), K(aligned_in_read_size), K(out_read_size), K(limited_and_aligned_in_read_size), K(errno)); } else { + iterator_info->inc_read_io_cnt(); + iterator_info->inc_read_io_size(out_read_size); + out_read_size = MIN(out_read_size - static_cast(backoff), in_read_size); MEMMOVE(read_buf, read_buf + backoff, in_read_size); PALF_LOG(TRACE, "inner_read_ success", K(ret), K(read_io_fd), K(aligned_start_offset), diff --git a/src/logservice/palf/log_reader.h b/src/logservice/palf/log_reader.h index 7340ffa4a0..c0afa2a240 100644 --- a/src/logservice/palf/log_reader.h +++ b/src/logservice/palf/log_reader.h @@ -16,6 +16,7 @@ #include "lib/ob_define.h" // OB_MAX_FILE_NAME_LENGTH #include "lib/utility/ob_macro_utils.h" //DISALLOW_COPY_AND_ASSIGN #include "log_define.h" +#include "palf_iterator.h" namespace oceanbase { namespace common @@ -37,7 +38,8 @@ public: const offset_t offset, int64_t in_read_size, ReadBuf &read_buf, - int64_t &out_read_size) const; + int64_t &out_read_size, + LogIteratorInfo *iterator_info) const; private: int limit_and_align_in_read_size_by_block_size_( offset_t aligned_start_offset, @@ -51,12 +53,17 @@ private: int64_t in_read_size, char *read_buf, const int64_t read_buf_len, - int64_t &out_read_size) const; + int64_t &out_read_size, + LogIteratorInfo *iterator_info) const; private: offset_t block_size_; char log_dir_[OB_MAX_FILE_NAME_LENGTH]; // LogDir *log_dir_; + mutable int64_t last_accum_read_statistic_time_; + mutable int64_t accum_read_io_count_; + mutable int64_t accum_read_log_size_; + mutable int64_t accum_read_cost_ts_; bool is_inited_; DISALLOW_COPY_AND_ASSIGN(LogReader); // TODO by runlin diff --git a/src/logservice/palf/log_reader_utils.cpp b/src/logservice/palf/log_reader_utils.cpp index e7df5c7c44..bc33c439d6 100644 --- a/src/logservice/palf/log_reader_utils.cpp +++ b/src/logservice/palf/log_reader_utils.cpp @@ -82,7 +82,8 @@ int alloc_read_buf(const char *label, const int64_t buf_len, ReadBuf &read_buf) { int ret = OB_SUCCESS; const int64_t dio_align_size = LOG_DIO_ALIGN_SIZE; - const int64_t size = upper_align(buf_len, dio_align_size) + dio_align_size; + const int64_t cache_align_size = LOG_CACHE_ALIGN_SIZE; + const int64_t size = upper_align(buf_len, dio_align_size) + dio_align_size + cache_align_size; if (NULL == (read_buf.buf_ = static_cast( mtl_malloc_align(dio_align_size, size, label)))) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/logservice/palf/log_shared_queue_thread.h b/src/logservice/palf/log_shared_queue_thread.h index beee80d207..5b02ac3128 100644 --- a/src/logservice/palf/log_shared_queue_thread.h +++ b/src/logservice/palf/log_shared_queue_thread.h @@ -15,6 +15,7 @@ #include "lib/thread/thread_mgr_interface.h" #include "lib/utility/ob_print_utils.h" +#include "palf_callback.h" namespace oceanbase { diff --git a/src/logservice/palf/log_shared_task.cpp b/src/logservice/palf/log_shared_task.cpp index cf2c134205..3cfd8e5688 100644 --- a/src/logservice/palf/log_shared_task.cpp +++ b/src/logservice/palf/log_shared_task.cpp @@ -70,5 +70,66 @@ int LogHandleSubmitTask::do_task(IPalfEnvImpl *palf_env_impl) return ret; } +// ================================================= LogFillCacheTask ================================= +LogFillCacheTask::LogFillCacheTask(const int64_t palf_id, const int64_t palf_epoch) + : LogSharedTask(palf_id, palf_epoch), is_inited_(false), begin_lsn_(LOG_INVALID_LSN_VAL), size_(0) +{} + +LogFillCacheTask::~LogFillCacheTask() +{ + if (IS_INIT) { + is_inited_ = false; + begin_lsn_.reset(); + size_ = 0; + } +} + +int LogFillCacheTask::init(const LSN &begin_lsn, const int64_t size) +{ + int ret = OB_SUCCESS; + if (IS_INIT) { + ret = OB_INIT_TWICE; + PALF_LOG(ERROR, "LogFillCacheTask has been inited", K(ret), KP(this), K(begin_lsn), K(size), K(this)); + } else if (!begin_lsn.is_valid() || 0 >= size) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(ERROR, "invalid arguments", K(ret), K(begin_lsn), K(size)); + } else { + begin_lsn_ = begin_lsn; + size_ = size; + is_inited_ = true; + PALF_LOG(TRACE, "LogFillCacheTask init successfully", K(begin_lsn), K(size), KP(this), K(this)); + } + + return ret; +} + +int LogFillCacheTask::do_task(IPalfEnvImpl *palf_env_impl) +{ + int ret = OB_SUCCESS; + int64_t palf_epoch = -1; + IPalfHandleImplGuard guard; + char *buf = NULL; + common::ObTimeGuard time_guard("fill cache when log slide", 100 * 1000); + if (OB_FAIL(palf_env_impl->get_palf_handle_impl(palf_id_, guard))) { + PALF_LOG(WARN, "IPalfEnvImpl get_palf_handle_impl failed", K(ret), KPC(this)); + } else if (OB_FAIL(guard.get_palf_handle_impl()->get_palf_epoch(palf_epoch))) { + PALF_LOG(WARN, "PalfHandleImpl get_palf_epoch failed", K(ret), KPC(this)); + } else if (palf_epoch != palf_epoch_) { + ret = OB_STATE_NOT_MATCH; + PALF_LOG(WARN, "palf_epoch has changed, drop task", K(ret), K(palf_epoch), KPC(this)); + //} else if (OB_FAIL(guard.get_palf_handle_impl()->fill_cache_when_slide(begin_lsn_, size_))) { + PALF_LOG(WARN, "failed to fill committed logs to cold cache", K(ret), K(palf_id_), K(begin_lsn_), K(size_)); + } else { + PALF_LOG(TRACE, "fill committed logs to cold cache successfully", K(palf_id_), K(begin_lsn_), K(size_)); + } + + return ret; +} + +void LogFillCacheTask::free_this(IPalfEnvImpl *palf_env_impl) +{ + palf_env_impl->get_log_allocator()->free_log_fill_cache_task(this); +} + } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_shared_task.h b/src/logservice/palf/log_shared_task.h index 0e89bb8cb6..09147087b8 100644 --- a/src/logservice/palf/log_shared_task.h +++ b/src/logservice/palf/log_shared_task.h @@ -14,7 +14,7 @@ #define OCEANBASE_LOGSERVICE_LOG_SHARED_TASK_ #include "lib/utility/ob_print_utils.h" - +#include "lsn.h" namespace oceanbase { namespace palf @@ -24,6 +24,7 @@ class IPalfEnvImpl; enum class LogSharedTaskType { LogHandleSubmitType = 1, + LogFillCacheType = 2, }; inline const char *shared_type_2_str(const LogSharedTaskType type) @@ -32,7 +33,7 @@ inline const char *shared_type_2_str(const LogSharedTaskType type) switch(type) { EXTRACT_SHARED_TYPE(LogHandleSubmitType); - + EXTRACT_SHARED_TYPE(LogFillCacheType); default: return "Invalid Type"; } @@ -72,6 +73,23 @@ private: DISALLOW_COPY_AND_ASSIGN(LogHandleSubmitTask); }; +class LogFillCacheTask : public LogSharedTask +{ +public: + LogFillCacheTask(const int64_t palf_id, const int64_t palf_epoch); + ~LogFillCacheTask() override; + int init(const LSN &begin_lsn, const int64_t size); + int do_task(IPalfEnvImpl *palf_env_impl) override; + void free_this(IPalfEnvImpl *palf_env_impl) override; + virtual LogSharedTaskType get_shared_task_type() const override { return LogSharedTaskType::LogFillCacheType; } + INHERIT_TO_STRING_KV("LogSharedTask", LogSharedTask, "task type", shared_type_2_str(get_shared_task_type())); +private: + bool is_inited_; + LSN begin_lsn_; + int64_t size_; + DISALLOW_COPY_AND_ASSIGN(LogFillCacheTask); +}; + } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_storage.cpp b/src/logservice/palf/log_storage.cpp index 0275937328..76354ef43a 100644 --- a/src/logservice/palf/log_storage.cpp +++ b/src/logservice/palf/log_storage.cpp @@ -15,7 +15,7 @@ #include "lib/ob_errno.h" // OB_INVALID_ARGUMENT #include "lib/stat/ob_session_stat.h" // Session #include "log_reader_utils.h" // ReadBuf -#include "palf_handle_impl.h" // LogHotCache +#include "palf_handle_impl.h" // LogCache #include "share/scn.h" namespace oceanbase @@ -38,11 +38,7 @@ LogStorage::LogStorage() : delete_block_lock_(common::ObLatchIds::PALF_LOG_ENGINE_LOCK), update_manifest_cb_(), plugins_(NULL), - hot_cache_(NULL), - last_accum_read_statistic_time_(OB_INVALID_TIMESTAMP), - accum_read_io_count_(0), - accum_read_log_size_(0), - accum_read_cost_ts_(0), + log_cache_(NULL), flashback_version_(OB_INVALID_TIMESTAMP), is_inited_(false) {} @@ -57,7 +53,7 @@ int LogStorage::init(const char *base_dir, const char *sub_dir, const LSN &base_ const int64_t align_size, const int64_t align_buf_size, const UpdateManifestCallback &update_manifest_cb, ILogBlockPool *log_block_pool, LogPlugins *plugins, - LogHotCache *hot_cache) + LogCache *log_cache) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -72,7 +68,7 @@ int LogStorage::init(const char *base_dir, const char *sub_dir, const LSN &base_ update_manifest_cb, log_block_pool, plugins, - hot_cache))) { + log_cache))) { PALF_LOG(WARN, "LogStorage do_init_ failed", K(ret), K(base_dir), K(sub_dir), K(palf_id)); } else { PALF_LOG(INFO, "LogStorage init success", K(ret), K(base_dir), K(sub_dir), @@ -123,10 +119,6 @@ void LogStorage::destroy() log_tail_.reset(); log_reader_.destroy(); block_mgr_.destroy(); - last_accum_read_statistic_time_ = OB_INVALID_TIMESTAMP; - accum_read_io_count_ = 0; - accum_read_log_size_ = 0; - accum_read_cost_ts_ = 0; PALF_LOG(INFO, "LogStorage destroy success"); } @@ -279,7 +271,6 @@ int LogStorage::pread(const LSN &read_lsn, { int ret = OB_SUCCESS; bool need_read_with_block_header = false; - UNUSED(io_ctx); if (IS_NOT_INIT) { ret = OB_NOT_INIT; PALF_LOG(ERROR, "LogStorage not inited!!!", K(ret)); @@ -287,13 +278,12 @@ int LogStorage::pread(const LSN &read_lsn, || false == read_buf.is_valid()) { ret = OB_INVALID_ARGUMENT; PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(read_lsn), K(in_read_size), K(read_buf)); - } else if (OB_NOT_NULL(hot_cache_) - && OB_SUCCESS == (hot_cache_->read(read_lsn, in_read_size, read_buf.buf_, out_read_size)) - && out_read_size > 0) { - // read data from hot_cache successfully - } else if (OB_FAIL(inner_pread_(read_lsn, in_read_size, need_read_with_block_header, read_buf, out_read_size))) { - PALF_LOG(WARN, "inner_pread_ failed", K(ret), K(read_lsn), K(in_read_size), KPC(this)); + } else if (OB_FAIL(inner_pread_(read_lsn, in_read_size, + need_read_with_block_header, read_buf, + out_read_size, io_ctx))) { + PALF_LOG(WARN, "inner_pread_ failed", K(ret), K(read_lsn), K(in_read_size), K(read_buf), K(out_read_size), KPC(this)); } else { + PALF_LOG(TRACE, "inner_pread_ succeed", K(read_lsn), K(in_read_size), K(read_buf), K(out_read_size)); } return ret; } @@ -301,7 +291,8 @@ int LogStorage::pread(const LSN &read_lsn, int LogStorage::pread_with_block_header(const LSN &read_lsn, const int64_t in_read_size, ReadBuf &read_buf, - int64_t &out_read_size) + int64_t &out_read_size, + LogIOContext &io_ctx) { int ret = OB_SUCCESS; bool need_read_with_block_header = true; @@ -311,7 +302,7 @@ int LogStorage::pread_with_block_header(const LSN &read_lsn, } else if (false == read_lsn.is_valid() || 0 >= in_read_size || false == read_buf.is_valid()) { ret = OB_INVALID_ARGUMENT; PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(read_lsn), K(in_read_size), K(read_buf)); - } else if (OB_FAIL(inner_pread_(read_lsn, in_read_size, need_read_with_block_header, read_buf, out_read_size))) { + } else if (OB_FAIL(inner_pread_(read_lsn, in_read_size, need_read_with_block_header, read_buf, out_read_size, io_ctx))) { PALF_LOG(WARN, "inner_pread_ failed", K(ret), K(read_lsn), K(in_read_size), KPC(this)); } else { } @@ -624,7 +615,7 @@ int LogStorage::do_init_(const char *base_dir, const UpdateManifestCallback &update_manifest_cb, ILogBlockPool *log_block_pool, LogPlugins *plugins, - LogHotCache *hot_cache) + LogCache *log_cache) { int ret = OB_SUCCESS; int tmp_ret = 0; @@ -652,8 +643,7 @@ int LogStorage::do_init_(const char *base_dir, logical_block_size_ = logical_block_size; update_manifest_cb_ = update_manifest_cb; plugins_ = plugins; - hot_cache_ = hot_cache; - last_accum_read_statistic_time_ = ObTimeUtility::fast_current_time(); + log_cache_ = log_cache; flashback_version_ = 0; is_inited_ = true; } @@ -838,6 +828,12 @@ void LogStorage::get_readable_log_tail_guarded_by_lock_(LSN &readable_log_tail, flashback_version = flashback_version_; } +void LogStorage::get_flashback_version_guarded_by_lock_(int64_t &flashback_version) const +{ + ObSpinLockGuard guard(tail_info_lock_); + flashback_version = flashback_version_; +} + offset_t LogStorage::get_phy_offset_(const LSN &lsn) const { return lsn_2_offset(lsn, logical_block_size_) + MAX_INFO_BLOCK_SIZE; @@ -869,7 +865,8 @@ int LogStorage::read_block_header_(const block_id_t block_id, PALF_LOG(WARN, "block_id is large than max_block_id", K(ret), K(block_id), K(readable_log_tail), K(max_block_id), K(log_block_header)); } else { - if (OB_FAIL(log_reader_.pread(block_id, 0, in_read_size, read_buf, out_read_size))) { + LogIteratorInfo iterator_info(false); + if (OB_FAIL(log_reader_.pread(block_id, 0, in_read_size, read_buf, out_read_size, &iterator_info))) { PALF_LOG(WARN, "read info block failed", K(ret), K(read_buf)); } else if (OB_FAIL(log_block_header.deserialize(read_buf.buf_, out_read_size, pos))) { PALF_LOG(WARN, "deserialize info block failed", K(ret), K(read_buf), @@ -921,7 +918,8 @@ int LogStorage::inner_pread_(const LSN &read_lsn, const int64_t in_read_size, const bool need_read_log_block_header, ReadBuf &read_buf, - int64_t &out_read_size) + int64_t &out_read_size, + LogIOContext &io_ctx) { int ret = OB_SUCCESS; // NB: don't support read data from diffent file. @@ -935,19 +933,28 @@ int LogStorage::inner_pread_(const LSN &read_lsn, const offset_t read_offset = lsn_2_offset(read_lsn, logical_block_size_); const offset_t real_read_offset = read_offset == 0 && true == need_read_log_block_header ? 0 : get_phy_offset_(read_lsn); - const int64_t start_ts = ObTimeUtility::fast_current_time(); if (read_lsn >= readable_log_tail) { ret = OB_ERR_OUT_OF_UPPER_BOUND; PALF_LOG(WARN, "read something out of upper bound", K(ret), K(read_lsn), K(log_tail_)); } else { - if (OB_FAIL(log_reader_.pread(read_block_id, - real_read_offset, - real_in_read_size, - read_buf, - out_read_size))) { - PALF_LOG( - WARN, "LogReader pread failed", K(ret), K(read_lsn), K(log_tail_), K(real_in_read_size)); + if (is_log_cache_inited_()) { + if (OB_FAIL(log_cache_->read(flashback_version, read_lsn, real_in_read_size, + read_buf, out_read_size, io_ctx.get_iterator_info()))) { + PALF_LOG(WARN, "read log cache failed", K(flashback_version), K(read_lsn), + K(real_in_read_size), K(read_buf), K(out_read_size), KPC(this)); + } else { + PALF_LOG(TRACE, "read log cache successfully", K(read_lsn), K(in_read_size), + K(need_read_log_block_header), K(read_buf), K(out_read_size)); + } + } else if (OB_FAIL(log_reader_.pread(read_block_id, + real_read_offset, + real_in_read_size, + read_buf, + out_read_size, + io_ctx.get_iterator_info()))) { + PALF_LOG(WARN, "LogReader pread failed", K(ret), K(read_lsn), + K(log_tail_), K(real_in_read_size), KPC(this)); } else { PALF_LOG(TRACE, "inner_pread success", @@ -958,22 +965,8 @@ int LogStorage::inner_pread_(const LSN &read_lsn, K(read_lsn), K(out_read_size), K(readable_log_tail)); - const int64_t cost_ts = ObTimeUtility::fast_current_time() - start_ts; - EVENT_TENANT_INC(ObStatEventIds::PALF_READ_IO_COUNT_FROM_DISK, MTL_ID()); - EVENT_ADD(ObStatEventIds::PALF_READ_SIZE_FROM_DISK, out_read_size); - EVENT_ADD(ObStatEventIds::PALF_READ_TIME_FROM_DISK, cost_ts); - ATOMIC_INC(&accum_read_io_count_); - ATOMIC_AAF(&accum_read_log_size_, out_read_size); - ATOMIC_AAF(&accum_read_cost_ts_, cost_ts); - if (palf_reach_time_interval(PALF_IO_STAT_PRINT_INTERVAL_US, last_accum_read_statistic_time_)) { - const int64_t avg_pread_cost = accum_read_cost_ts_ / accum_read_io_count_; - PALF_LOG(INFO, "[PALF STAT READ LOG INFO FROM DISK]", KPC(this), K_(accum_read_io_count), - K_(accum_read_log_size), K(avg_pread_cost)); - ATOMIC_STORE(&accum_read_io_count_, 0); - ATOMIC_STORE(&accum_read_log_size_, 0); - ATOMIC_STORE(&accum_read_cost_ts_, 0); - } } + // to ensure the data integrity, we should check 'read_block_id' whether has integrity data. int tmp_ret = check_read_out_of_bound_(read_block_id, flashback_version, OB_NO_SUCH_FILE_OR_DIRECTORY == ret); // overwrite ret code: @@ -985,6 +978,7 @@ int LogStorage::inner_pread_(const LSN &read_lsn, ret = tmp_ret; } } + return ret; } @@ -1015,5 +1009,30 @@ int LogStorage::get_logical_block_size(int64_t &logical_block_size) const return ret; } +LogReader *LogStorage::get_log_reader() +{ + return &log_reader_; +} + +bool LogStorage::is_log_cache_inited_() +{ + return OB_NOT_NULL(log_cache_) && log_cache_->is_inited(); +} + +int LogStorage::fill_cache_when_slide(const LSN &begin_lsn, const int64_t size) +{ + int ret = OB_SUCCESS; + int64_t flashback_version = OB_INVALID_TIMESTAMP; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(WARN, "LogStorage has not been inited!", K(ret), K(palf_id_)); + } else if (FALSE_IT(get_flashback_version_guarded_by_lock_(flashback_version))) { + } else if (OB_FAIL(log_cache_->fill_cache_when_slide(begin_lsn, size, flashback_version))) { + PALF_LOG(WARN, "failed to fill committed log into cold cache", K(ret), K(begin_lsn), K(size), K(flashback_version)); + } + + return ret; +} + } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_storage.h b/src/logservice/palf/log_storage.h index 2a8caffca4..a083725d84 100644 --- a/src/logservice/palf/log_storage.h +++ b/src/logservice/palf/log_storage.h @@ -23,6 +23,7 @@ #include "lsn.h" // LSN #include "palf_iterator.h" // PalfIteraor #include "palf_callback_wrapper.h" +#include "log_cache.h" namespace oceanbase { @@ -37,7 +38,8 @@ class SCN; namespace palf { class ReadBuf; -class LogHotCache; +class LogCache; +class LogIOContext; class LogStorage : public ILogStorage { public: @@ -54,7 +56,7 @@ public: const UpdateManifestCallback &update_manifest_cb, ILogBlockPool *log_block_pool, LogPlugins *plugins, - LogHotCache *hot_cache); + LogCache *log_cache); template int load(const char *log_dir, @@ -67,7 +69,7 @@ public: const UpdateManifestCallback &update_manifest_cb, ILogBlockPool *log_block_pool, LogPlugins *plugins, - LogHotCache *hot_cache, + LogCache *log_cache, EntryHeaderType &entry_header, LSN &lsn); @@ -94,7 +96,8 @@ public: int pread_with_block_header(const LSN &read_lsn, const int64_t in_read_size, ReadBuf &read_buf, - int64_t &out_read_size); + int64_t &out_read_size, + LogIOContext &io_ctx); int truncate(const LSN &lsn); int truncate_prefix_blocks(const LSN &lsn); @@ -118,6 +121,9 @@ public: int get_logical_block_size(int64_t &logical_block_size) const; + LogReader *get_log_reader(); + int fill_cache_when_slide(const LSN &begin_lsn, const int64_t size); + TO_STRING_KV(K_(log_tail), K_(readable_log_tail), K_(log_block_header), @@ -138,7 +144,7 @@ private: const UpdateManifestCallback &update_manifest_cb, ILogBlockPool *log_block_pool, LogPlugins *plugins, - LogHotCache *hot_cache); + LogCache *log_cache); // @ret val: // OB_SUCCESS // OB_ERR_OUT_OF_LOWER_BOUND @@ -173,6 +179,7 @@ private: const LSN &get_log_tail_guarded_by_lock_() const; void get_readable_log_tail_guarded_by_lock_(LSN &readable_log_tail, int64_t &flashback_version) const; + void get_flashback_version_guarded_by_lock_(int64_t &flashback_version) const; offset_t get_phy_offset_(const LSN &lsn) const; int read_block_header_(const block_id_t block_id, LogBlockHeader &block_header) const; bool check_last_block_is_full_(const block_id_t max_block_id) const; @@ -181,10 +188,12 @@ private: const int64_t in_read_size, const bool need_read_block_header, ReadBuf &read_buf, - int64_t &out_read_size); + int64_t &out_read_size, + LogIOContext &io_ctx); void reset_log_tail_for_last_block_(const LSN &lsn, bool last_block_exist); int update_manifest_(const block_id_t expected_next_block_id, const bool in_restart = false); int check_read_integrity_(const block_id_t &block_id); + bool is_log_cache_inited_(); private: // Used to perform IO tasks in the background LogBlockMgr block_mgr_; @@ -205,11 +214,7 @@ private: UpdateManifestCallback update_manifest_cb_; LogPlugins *plugins_; char block_header_serialize_buf_[MAX_INFO_BLOCK_SIZE]; - LogHotCache *hot_cache_; - int64_t last_accum_read_statistic_time_; - int64_t accum_read_io_count_; - int64_t accum_read_log_size_; - int64_t accum_read_cost_ts_; + LogCache *log_cache_; int64_t flashback_version_; bool is_inited_; }; @@ -231,7 +236,7 @@ int LogStorage::load(const char *base_dir, const UpdateManifestCallback &update_manifest_cb, ILogBlockPool *log_block_pool, LogPlugins *plugins, - LogHotCache *hot_cache, + LogCache *log_cache, EntryHeaderType &entry_header, LSN &lsn) { @@ -252,7 +257,7 @@ int LogStorage::load(const char *base_dir, update_manifest_cb, log_block_pool, plugins, - hot_cache))) { + log_cache))) { PALF_LOG(WARN, "LogStorage do_init_ failed", K(ret), K(base_dir), K(sub_dir), K(palf_id)); // NB: if there is no valid data on disk, no need to load last block } else if (OB_FAIL(block_mgr_.get_block_id_range(min_block_id, max_block_id)) @@ -289,6 +294,7 @@ int LogStorage::locate_log_tail_and_last_valid_entry_header_(const block_id_t mi // the last block may has not valid data, we need iterate prev block // for GC, we must ensure that the block which include 'max_committed_lsn' will no be reused const bool need_print_error = false; + const bool enable_fill_cache = false; while (OB_SUCC(ret) && true == is_valid_block_id(iterate_block_id) && iterate_block_id >= min_block_id) { // NB: 'log_tail_' need point to the tail of 'iterate_block_id', because 'pread' interface @@ -298,7 +304,7 @@ int LogStorage::locate_log_tail_and_last_valid_entry_header_(const block_id_t mi PalfIterator iterator; auto get_file_end_lsn = []() { return LSN(LOG_MAX_LSN_VAL); }; LSN start_lsn(iterate_block_id * logical_block_size_); - if (OB_FAIL(iterator.init(start_lsn, get_file_end_lsn, this))) { + if (OB_FAIL(iterator.init(start_lsn, get_file_end_lsn, this, enable_fill_cache))) { PALF_LOG(WARN, "PalfGroupBufferIterator init failed", K(ret), K(start_lsn)); } else { iterator.set_need_print_error(need_print_error); diff --git a/src/logservice/palf/palf_env_impl.cpp b/src/logservice/palf/palf_env_impl.cpp index 0157f66e9f..e5d5749cff 100644 --- a/src/logservice/palf/palf_env_impl.cpp +++ b/src/logservice/palf/palf_env_impl.cpp @@ -193,6 +193,7 @@ PalfEnvImpl::PalfEnvImpl() : palf_meta_lock_(common::ObLatchIds::PALF_ENV_LOCK), palf_handle_impl_map_(64), // 指定min_size=64 last_palf_epoch_(0), rebuild_replica_log_lag_threshold_(0), + enable_log_cache_(false), diskspace_enough_(true), tenant_id_(0), is_inited_(false), @@ -366,6 +367,7 @@ void PalfEnvImpl::destroy() tmp_log_dir_[0] = '\0'; disk_options_wrapper_.reset(); rebuild_replica_log_lag_threshold_ = 0; + enable_log_cache_ = false; } // NB: not thread safe @@ -923,6 +925,7 @@ int PalfEnvImpl::update_options(const PalfOptions &options) } else if (OB_FAIL(disk_options_wrapper_.update_disk_options(options.disk_options_))) { PALF_LOG(WARN, "update_disk_options failed", K(ret), K(options)); } else { + enable_log_cache_ = options.enable_log_cache_; PALF_LOG(INFO, "update_options successs", K(options), KPC(this)); } return ret; @@ -937,6 +940,7 @@ int PalfEnvImpl::get_options(PalfOptions &options) options.disk_options_ = disk_options_wrapper_.get_disk_opts_for_recycling_blocks(); options.compress_options_ = log_rpc_.get_compress_opts(); options.rebuild_replica_log_lag_threshold_ = rebuild_replica_log_lag_threshold_; + options.enable_log_cache_ = enable_log_cache_; } return ret; } diff --git a/src/logservice/palf/palf_env_impl.h b/src/logservice/palf/palf_env_impl.h index e5ac098d86..3c17eddd8b 100644 --- a/src/logservice/palf/palf_env_impl.h +++ b/src/logservice/palf/palf_env_impl.h @@ -204,6 +204,7 @@ public: virtual int update_replayable_point(const SCN &replayable_scn) = 0; virtual int get_throttling_options(PalfThrottleOptions &option) = 0; virtual void period_calc_disk_usage() = 0; + virtual int get_options(PalfOptions &options) = 0; VIRTUAL_TO_STRING_KV("IPalfEnvImpl", "Dummy"); }; @@ -388,6 +389,7 @@ private: // last_palf_epoch_ is used to assign increasing epoch for each palf instance. int64_t last_palf_epoch_; int64_t rebuild_replica_log_lag_threshold_;//for rebuild test + bool enable_log_cache_; LogIOWorkerConfig log_io_worker_config_; bool diskspace_enough_; diff --git a/src/logservice/palf/palf_handle_impl.cpp b/src/logservice/palf/palf_handle_impl.cpp index 2284729fff..dc48f9a09b 100755 --- a/src/logservice/palf/palf_handle_impl.cpp +++ b/src/logservice/palf/palf_handle_impl.cpp @@ -45,7 +45,7 @@ PalfHandleImpl::PalfHandleImpl() log_engine_(), election_msg_sender_(log_engine_.log_net_service_), election_(), - hot_cache_(), + log_cache_(), fetch_log_engine_(NULL), allocator_(NULL), palf_id_(INVALID_PALF_ID), @@ -148,7 +148,7 @@ int PalfHandleImpl::init(const int64_t palf_id, } else if ((pret = snprintf(log_dir_, MAX_PATH_SIZE, "%s", log_dir)) && false) { ret = OB_ERR_UNEXPECTED; PALF_LOG(ERROR, "error unexpected", K(ret), K(palf_id)); - } else if (OB_FAIL(log_engine_.init(palf_id, log_dir, log_meta, alloc_mgr, log_block_pool, &hot_cache_, \ + } else if (OB_FAIL(log_engine_.init(palf_id, log_dir, log_meta, alloc_mgr, log_block_pool, &log_cache_, \ log_rpc, log_io_worker, log_shared_queue_th, &plugins_, palf_epoch, PALF_BLOCK_SIZE, PALF_META_BLOCK_SIZE))) { PALF_LOG(WARN, "LogEngine init failed", K(ret), K(palf_id), K(log_dir), K(alloc_mgr), K(log_rpc), K(log_io_worker), K(log_shared_queue_th)); @@ -204,7 +204,7 @@ int PalfHandleImpl::load(const int64_t palf_id, ret = OB_INVALID_ARGUMENT; PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(log_dir), K(alloc_mgr), K(log_rpc), K(log_io_worker), K(log_shared_queue_th)); - } else if (OB_FAIL(log_engine_.load(palf_id, log_dir, alloc_mgr, log_block_pool, &hot_cache_, log_rpc, + } else if (OB_FAIL(log_engine_.load(palf_id, log_dir, alloc_mgr, log_block_pool, &log_cache_, log_rpc, log_io_worker, log_shared_queue_th, &plugins_, entry_header, palf_epoch, is_integrity, PALF_BLOCK_SIZE, PALF_META_BLOCK_SIZE))) { PALF_LOG(WARN, "LogEngine load failed", K(ret), K(palf_id)); @@ -242,7 +242,7 @@ void PalfHandleImpl::destroy() fetch_log_engine_ = NULL; allocator_ = NULL; election_.stop(); - hot_cache_.destroy(); + log_cache_.destroy(); log_engine_.destroy(); reconfirm_.destroy(); state_mgr_.destroy(); @@ -2350,7 +2350,7 @@ int PalfHandleImpl::alloc_palf_group_buffer_iterator(const SCN &scn, sw_.get_committed_end_lsn(committed_end_lsn); return MIN(committed_end_lsn, max_flushed_end_lsn); }; - PalfGroupBufferIterator local_iter; + PalfGroupBufferIterator local_iter(palf_id_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); @@ -2924,8 +2924,8 @@ int PalfHandleImpl::do_init_mem_( return role_change_cb_wrpper.on_need_change_leader(id, dest_addr); }))) { PALF_LOG(WARN, "election_ init failed", K(ret), K(palf_id)); - } else if (OB_FAIL(hot_cache_.init(palf_id, this))) { - PALF_LOG(WARN, "hot_cache_ init failed", K(ret), K(palf_id)); + } else if (OB_FAIL(log_cache_.init(palf_id, this, palf_env_impl, log_engine_.get_log_storage()))) { + PALF_LOG(WARN, "log_cache_ init failed", K(ret), K(palf_id)); } else if (OB_FAIL(state_mgr_.init(palf_id, self, log_meta.get_log_prepare_meta(), log_meta.get_log_replica_property_meta(), &election_, &sw_, &reconfirm_, &log_engine_, &config_mgr_, &mode_mgr_, &role_change_cb_wrpper_, &plugins_))) { PALF_LOG(WARN, "state_mgr_ init failed", K(ret), K(palf_id)); @@ -3164,7 +3164,7 @@ int PalfHandleImpl::receive_batch_log(const common::ObAddr &server, { int ret = OB_SUCCESS; int64_t start_ts = ObTimeUtility::current_time(); - MemPalfGroupBufferIterator iterator; + MemPalfGroupBufferIterator iterator(palf_id_); MemoryStorage storage; auto get_file_end_lsn = [curr_lsn, buf_len] () { return curr_lsn + buf_len; }; if (OB_FAIL(iterator.init(curr_lsn, get_file_end_lsn, &storage))) { @@ -3569,7 +3569,7 @@ int PalfHandleImpl::fetch_log_from_storage_(const common::ObAddr &server, LSN prev_lsn_each_round = prev_lsn; LSN prev_end_lsn_each_round = prev_lsn; int64_t prev_log_proposal_id_each_round = INVALID_PROPOSAL_ID; - PalfGroupBufferIterator iterator; + PalfGroupBufferIterator iterator(palf_id_, LogIOUser::FETCHLOG); const LSN fetch_end_lsn = fetch_start_lsn + fetch_log_size; int64_t fetched_count = 0; const bool need_check_prev_log = (prev_lsn.is_valid() && PALF_INITIAL_LSN_VAL < fetch_start_lsn.val_); @@ -3614,6 +3614,7 @@ int PalfHandleImpl::fetch_log_from_storage_(const common::ObAddr &server, // Rpc delay increases enormously when it's size exceeds 2M. const int64_t MAX_BATCH_LOG_SIZE_EACH_ROUND = 2 * 1024 * 1024 - 1024; char *batch_log_buf = NULL; + bool enable_fill_cache = false; if (no_need_fetch_log) { PALF_LOG(INFO, "no need fetch_log_from_storage", K(ret), KPC(this), K(server), K(fetch_start_lsn), K(prev_lsn), K(max_flushed_end_lsn), K(access_mode)); @@ -3632,7 +3633,7 @@ int PalfHandleImpl::fetch_log_from_storage_(const common::ObAddr &server, } else if (check_need_hook_fetch_log_(fetch_type, fetch_start_lsn)) { ret = OB_ERR_OUT_OF_LOWER_BOUND; } else if (FALSE_IT(prev_log_proposal_id_each_round = prev_log_info.log_proposal_id_)) { - } else if (OB_FAIL(iterator.init(fetch_start_lsn, get_file_end_lsn, log_engine_.get_log_storage()))) { + } else if (OB_FAIL(iterator.init(fetch_start_lsn, get_file_end_lsn, log_engine_.get_log_storage(), enable_fill_cache))) { PALF_LOG(ERROR, "init iterator failed", K(ret), K_(palf_id)); } else if (FALSE_IT(iterator.set_need_print_error(false))) { // NB: Fetch log will be concurrent with truncate, the content on disk will not integrity, need igore @@ -4355,7 +4356,7 @@ int PalfHandleImpl::get_prev_log_info_for_fetch_(const LSN &prev_lsn, LogInfo &prev_log_info) { int ret = OB_SUCCESS; - PalfGroupBufferIterator iterator; + PalfGroupBufferIterator iterator(palf_id_, LogIOUser::FETCHLOG); auto get_file_end_lsn = [&]() { return curr_lsn; }; auto get_mode_version = [this]() -> int64_t { int64_t mode_version = INVALID_PROPOSAL_ID; @@ -4366,7 +4367,7 @@ int PalfHandleImpl::get_prev_log_info_for_fetch_(const LSN &prev_lsn, } return mode_version; }; - if (OB_FAIL(iterator.init(prev_lsn, get_file_end_lsn, get_mode_version, log_engine_.get_log_storage()))) { + if (OB_FAIL(iterator.init(prev_lsn, get_file_end_lsn, get_mode_version, log_engine_.get_log_storage(), false))) { PALF_LOG(WARN, "LogGroupEntryIterator init failed", K(ret), K(iterator), K(prev_lsn), K(curr_lsn)); } else if (FALSE_IT(iterator.set_need_print_error(false))) { } else { @@ -4400,15 +4401,16 @@ int PalfHandleImpl::get_prev_log_info_(const LSN &lsn, LSN start_lsn; start_lsn.val_ = (0ul == lsn_block_offset ? (lsn_block_id-1) * PALF_BLOCK_SIZE : lsn_block_id * PALF_BLOCK_SIZE); - PalfGroupBufferIterator iterator; + PalfGroupBufferIterator iterator(palf_id_); const LogSnapshotMeta log_snapshot_meta = log_engine_.get_log_meta().get_log_snapshot_meta(); const LSN base_lsn = log_snapshot_meta.base_lsn_; LogInfo log_info; auto get_file_end_lsn = [&]() { return lsn; }; + bool enable_fill_cache = false; if (lsn == base_lsn && OB_SUCC(log_snapshot_meta.get_prev_log_info(log_info))) { prev_log_info = log_info; PALF_LOG(INFO, "lsn is same as base_lsn, and log_snapshot_meta is valid", K(lsn), K(log_snapshot_meta)); - } else if (OB_FAIL(iterator.init(start_lsn, get_file_end_lsn, log_engine_.get_log_storage()))) { + } else if (OB_FAIL(iterator.init(start_lsn, get_file_end_lsn, log_engine_.get_log_storage(), enable_fill_cache))) { PALF_LOG(WARN, "LogGroupEntryIterator init failed", K(ret), K(start_lsn), K(lsn)); } else { LSN curr_lsn; @@ -4562,14 +4564,15 @@ int PalfHandleImpl::construct_palf_base_info_for_flashback_(const LSN &start_lsn int PalfHandleImpl::append_disk_log_to_sw_(const LSN &start_lsn) { int ret = OB_SUCCESS; - PalfGroupBufferIterator iterator; + PalfGroupBufferIterator iterator(palf_id_); + bool enable_fill_cache = false; // NB: we need append each log locate in [start_lsn, log_tail_); auto get_file_end_lsn = [&](){ return LSN(LOG_MAX_LSN_VAL); }; // if there is no valid data on disk, the 'start_lsn' which meash max committed log offset // of this log stream may be invalid. if (false == start_lsn.is_valid()) { ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(iterator.init(start_lsn, get_file_end_lsn, log_engine_.get_log_storage()))) { + } else if (OB_FAIL(iterator.init(start_lsn, get_file_end_lsn, log_engine_.get_log_storage(), enable_fill_cache))) { PALF_LOG(WARN, "PalfGroupBufferIterator init failed", K(ret), K(log_engine_), K(start_lsn)); } else { LogGroupEntry group_entry; @@ -4879,7 +4882,8 @@ int PalfHandleImpl::read_and_append_log_group_entry_before_ts_( LogGroupEntryHeader prev_entry_header; LogGroupEntry curr_group_entry; LSN prev_log_lsn, curr_log_lsn; - PalfGroupBufferIterator iterator; + PalfGroupBufferIterator iterator(palf_id_, LogIOUser::FLASHBACK); + bool enable_fill_cache = false; auto get_file_end_lsn = [&](){ LSN max_flushed_end_lsn; (void)sw_.get_max_flushed_end_lsn(max_flushed_end_lsn); @@ -4891,7 +4895,7 @@ int PalfHandleImpl::read_and_append_log_group_entry_before_ts_( if (!read_buf_guard.read_buf_.is_valid()) { ret = OB_ALLOCATE_MEMORY_FAILED; PALF_LOG(WARN, "allocate memory failed", KPC(this)); - } else if (OB_FAIL(iterator.init(start_lsn, get_file_end_lsn, log_engine_.get_log_storage()))) { + } else if (OB_FAIL(iterator.init(start_lsn, get_file_end_lsn, log_engine_.get_log_storage(), enable_fill_cache))) { PALF_LOG(WARN, "iterator init failed", K(ret), KPC(this), K(start_lsn), K(flashback_scn)); } else { const int64_t read_buf_len = read_buf_guard.read_buf_.buf_len_; diff --git a/src/logservice/palf/palf_handle_impl.h b/src/logservice/palf/palf_handle_impl.h index d9f83d99ce..e94b5cbb44 100755 --- a/src/logservice/palf/palf_handle_impl.h +++ b/src/logservice/palf/palf_handle_impl.h @@ -1410,7 +1410,7 @@ private: LogEngine log_engine_; ElectionMsgSender election_msg_sender_; election::ElectionImpl election_; - LogHotCache hot_cache_; + LogCache log_cache_; FetchLogEngine *fetch_log_engine_; common::ObILogAllocator *allocator_; int64_t palf_id_; diff --git a/src/logservice/palf/palf_iterator.h b/src/logservice/palf/palf_iterator.h index 06b485409d..bde5c0c2fe 100644 --- a/src/logservice/palf/palf_iterator.h +++ b/src/logservice/palf/palf_iterator.h @@ -27,18 +27,23 @@ template class PalfIterator { public: - PalfIterator() : iterator_storage_(), iterator_impl_(), need_print_error_(true), is_inited_(false) {} + PalfIterator() + : iterator_storage_(), iterator_impl_(), need_print_error_(true), + is_inited_(false), io_ctx_(LogIOUser::DEFAULT), last_print_time_(0) {} + PalfIterator(const int64_t palf_id, const LogIOUser io_user = LogIOUser::DEFAULT) + :iterator_storage_(), iterator_impl_(), need_print_error_(true), + is_inited_(false), io_ctx_(palf_id, io_user), last_print_time_(0) {} ~PalfIterator() {destroy();} - int init(const LSN &start_offset, const GetFileEndLSN &get_file_end_lsn, - ILogStorage *log_storage) + ILogStorage *log_storage, + const bool allow_filling_cache = true) { int ret = OB_SUCCESS; auto get_mode_version = []() { return PALF_INITIAL_PROPOSAL_ID; }; if (IS_INIT) { ret = OB_INIT_TWICE; - } else if (OB_FAIL(do_init_(start_offset, get_file_end_lsn, get_mode_version, log_storage))) { + } else if (OB_FAIL(do_init_(start_offset, get_file_end_lsn, get_mode_version, log_storage, allow_filling_cache))) { PALF_LOG(WARN, "PalfIterator init failed", K(ret)); } else { PALF_LOG(TRACE, "PalfIterator init success", K(ret), K(start_offset), KPC(this)); @@ -46,16 +51,16 @@ public: } return ret; } - int init(const LSN &start_offset, const GetFileEndLSN &get_file_end_lsn, const GetModeVersion &get_mode_version, - ILogStorage *log_storage) + ILogStorage *log_storage, + const bool allow_filling_cache = true) { int ret = OB_SUCCESS; if (IS_INIT) { ret = OB_INIT_TWICE; - } else if (OB_FAIL(do_init_(start_offset, get_file_end_lsn, get_mode_version, log_storage))) { + } else if (OB_FAIL(do_init_(start_offset, get_file_end_lsn, get_mode_version, log_storage, allow_filling_cache))) { PALF_LOG(WARN, "PalfIterator init failed", K(ret)); } else { PALF_LOG(TRACE, "PalfIterator init success", K(ret), K(start_offset), KPC(this)); @@ -81,6 +86,7 @@ public: is_inited_ = false; iterator_impl_.destroy(); iterator_storage_.destroy(); + io_ctx_.destroy(); } } @@ -155,11 +161,14 @@ public: int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - } else if (OB_FAIL(iterator_impl_.next(replayable_point_scn, next_min_scn, iterate_end_by_replayable_point)) + } else if (OB_FAIL(iterator_impl_.next(replayable_point_scn, next_min_scn, iterate_end_by_replayable_point, io_ctx_)) && OB_ITER_END != ret) { PALF_LOG(WARN, "PalfIterator next failed", K(ret), KPC(this)); print_error_log(ret); } else { + if (palf_reach_time_interval(PALF_STAT_PRINT_INTERVAL_US, last_print_time_)) { + PALF_LOG(INFO, "[PALF STAT ITERATOR INFO]", K(io_ctx_)); + } PALF_LOG(TRACE, "PalfIterator next success", K(iterator_impl_), K(ret), KPC(this), K(replayable_point_scn), K(next_min_scn), K(iterate_end_by_replayable_point)); } @@ -229,7 +238,7 @@ public: } bool check_is_the_last_entry() { - return iterator_impl_.check_is_the_last_entry(); + return iterator_impl_.check_is_the_last_entry(io_ctx_); } LSN get_curr_read_lsn() const { @@ -245,13 +254,30 @@ public: { need_print_error_ = need_print_error; } - TO_STRING_KV(K_(iterator_impl)); + void enable_fill_cache() + { + io_ctx_.set_allow_filling_cache(true); + } + void disable_fill_cache() + { + io_ctx_.set_allow_filling_cache(false); + } + void set_palf_id(const int64_t palf_id) + { + io_ctx_.set_palf_id(palf_id); + } + void set_type(const LogIOUser user_type) + { + io_ctx_.set_user_type(user_type); + } + TO_STRING_KV(K_(iterator_impl), K_(io_ctx)); private: int do_init_(const LSN &start_offset, const GetFileEndLSN &get_file_end_lsn, const GetModeVersion &get_mode_version, - ILogStorage *log_storage) + ILogStorage *log_storage, + const bool allow_filling_cache) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -264,6 +290,8 @@ private: } else if (OB_FAIL(iterator_impl_.init(get_mode_version, &iterator_storage_))) { PALF_LOG(WARN, "PalfIterator init failed", K(ret)); } else { + io_ctx_.set_allow_filling_cache(allow_filling_cache); + io_ctx_.set_start_lsn(start_offset); PALF_LOG(TRACE, "PalfIterator init success", K(ret), K(start_offset), KPC(this)); is_inited_ = true; } @@ -313,6 +341,8 @@ private: LogIteratorImpl iterator_impl_; bool need_print_error_; bool is_inited_; + LogIOContext io_ctx_; + int64_t last_print_time_; }; typedef PalfIterator MemPalfBufferIterator; diff --git a/src/logservice/palf/palf_options.cpp b/src/logservice/palf/palf_options.cpp index 61e9838f57..8743b680fa 100644 --- a/src/logservice/palf/palf_options.cpp +++ b/src/logservice/palf/palf_options.cpp @@ -25,6 +25,7 @@ void PalfOptions::reset() disk_options_.reset(); compress_options_.reset(); rebuild_replica_log_lag_threshold_ = 0; + enable_log_cache_ = false; } bool PalfOptions::is_valid() const diff --git a/src/logservice/palf/palf_options.h b/src/logservice/palf/palf_options.h index bd2d972aac..5e77d11f27 100644 --- a/src/logservice/palf/palf_options.h +++ b/src/logservice/palf/palf_options.h @@ -173,18 +173,21 @@ struct PalfOptions { PalfOptions() : disk_options_(), compress_options_(), - rebuild_replica_log_lag_threshold_(0) + rebuild_replica_log_lag_threshold_(0), + enable_log_cache_(false) {} ~PalfOptions() { reset(); } void reset(); bool is_valid() const; TO_STRING_KV(K(disk_options_), K(compress_options_), - K(rebuild_replica_log_lag_threshold_)); + K(rebuild_replica_log_lag_threshold_), + K(enable_log_cache_)); public: PalfDiskOptions disk_options_; PalfTransportCompressOptions compress_options_; int64_t rebuild_replica_log_lag_threshold_; + bool enable_log_cache_; }; struct PalfThrottleOptions diff --git a/src/logservice/replayservice/ob_replay_status.cpp b/src/logservice/replayservice/ob_replay_status.cpp index fa0dbf05e0..cf69435ae7 100644 --- a/src/logservice/replayservice/ob_replay_status.cpp +++ b/src/logservice/replayservice/ob_replay_status.cpp @@ -100,6 +100,7 @@ int ObReplayServiceSubmitTask::init(const palf::LSN &base_lsn, { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; + share::ObLSID ls_id; if (OB_ISNULL(replay_status) || OB_ISNULL(palf_handle)) { ret = OB_INVALID_ARGUMENT; CLOG_LOG(WARN, "invalid argument", K(type_), K(ret), K(replay_status), K(palf_handle)); @@ -108,6 +109,8 @@ int ObReplayServiceSubmitTask::init(const palf::LSN &base_lsn, } else if (OB_UNLIKELY(!base_scn.is_valid())) { ret = OB_ERR_UNEXPECTED; CLOG_LOG(ERROR, "base_scn is invalid", K(type_), K(base_lsn), K(base_scn), KR(ret)); + } else if (OB_FAIL(replay_status->get_ls_id(ls_id))) { + CLOG_LOG(WARN, "get ls_id failed", K(ret), K(type_)); } else { replay_status_ = replay_status; next_to_submit_lsn_ = base_lsn; @@ -115,6 +118,8 @@ int ObReplayServiceSubmitTask::init(const palf::LSN &base_lsn, base_lsn_ = base_lsn; base_scn_ = base_scn; type_ = ObReplayServiceTaskType::SUBMIT_LOG_TASK; + iterator_.set_palf_id(ls_id.id()); + iterator_.set_type(palf::LogIOUser::REPLAY); if (OB_SUCCESS != (tmp_ret = iterator_.next())) { // 在没有写入的情况下有可能已经到达边界 CLOG_LOG(WARN, "iterator next failed", K(iterator_), K(tmp_ret)); diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index 58037b2aa9..518f16fda4 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -112,6 +112,7 @@ #include "share/detect/ob_detect_manager.h" #include "observer/table/ttl/ob_table_ttl_task.h" #include "storage/high_availability/ob_storage_ha_diagnose_service.h" +#include "logservice/palf/log_cache.h" #ifdef OB_BUILD_ARBITRATION #include "logservice/arbserver/palf_env_lite_mgr.h" #include "logservice/arbserver/ob_arb_srv_network_frame.h" @@ -407,6 +408,8 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) LOG_ERROR("init storage failed", KR(ret)); } else if (OB_FAIL(init_tx_data_cache())) { LOG_ERROR("init tx data cache failed", KR(ret)); + } else if (OB_FAIL(init_log_kv_cache())) { + LOG_ERROR("init log kv cache failed", KR(ret)); } else if (OB_FAIL(locality_manager_.init(self_addr_, &sql_proxy_))) { LOG_ERROR("init locality manager failed", KR(ret)); @@ -699,6 +702,10 @@ void ObServer::destroy() OB_TX_DATA_KV_CACHE.destroy(); FLOG_INFO("tx data kv cache destroyed"); + FLOG_INFO("begin to destroy log kv cache"); + OB_LOG_KV_CACHE.destroy(); + FLOG_INFO("log kv cache destroyed"); + FLOG_INFO("begin to destroy location service"); location_service_.destroy(); FLOG_INFO("location service destroyed"); @@ -2838,6 +2845,15 @@ int ObServer::init_tx_data_cache() return ret; } +int ObServer::init_log_kv_cache() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(OB_LOG_KV_CACHE.init(palf::OB_LOG_KV_CACHE_NAME, 1, palf::LOG_CACHE_MEMORY_LIMIT))) { + LOG_WARN("init OB_LOG_KV_CACHE failed", KR(ret)); + } + return ret; +} + int ObServer::get_network_speed_from_sysfs(int64_t &network_speed) { int ret = OB_SUCCESS; diff --git a/src/observer/ob_server.h b/src/observer/ob_server.h index 2911944a2e..d1da8edbee 100644 --- a/src/observer/ob_server.h +++ b/src/observer/ob_server.h @@ -296,6 +296,7 @@ private: int init_px_target_mgr(); int init_storage(); int init_tx_data_cache(); + int init_log_kv_cache(); int init_gc_partition_adapter(); int init_loaddata_global_stat(); int init_bandwidth_throttle(); diff --git a/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp b/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp index 0f8325a286..e7a490c9f0 100644 --- a/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp @@ -452,6 +452,9 @@ int ObAllVirtualSysStat::get_cache_size_(const int64_t tenant_id, ObStatEventSet } else if (0 == STRNCMP(inst->status_.config_->cache_name_, "bf_cache", MAX_CACHE_NAME_LENGTH)) { stat_events.get(ObStatEventIds::BLOOM_FILTER_CACHE_SIZE - ObStatEventIds::STAT_EVENT_ADD_END -1)->stat_value_ = inst->status_.map_size_ + inst->status_.store_size_; + } else if (0 == STRNCMP(inst->status_.config_->cache_name_, "log_kv_cache", MAX_CACHE_NAME_LENGTH)) { + stat_events.get(ObStatEventIds::LOG_KV_CACHE_SIZE - ObStatEventIds::STAT_EVENT_ADD_END -1)->stat_value_ + = inst->status_.map_size_ + inst->status_.store_size_; } else { //do nothing } diff --git a/src/observer/virtual_table/ob_information_kvcache_table.cpp b/src/observer/virtual_table/ob_information_kvcache_table.cpp index d06a1701c5..a51cebbd9b 100644 --- a/src/observer/virtual_table/ob_information_kvcache_table.cpp +++ b/src/observer/virtual_table/ob_information_kvcache_table.cpp @@ -195,6 +195,8 @@ int ObInfoSchemaKvCacheTable::set_diagnose_info(ObKVCacheInst *inst, ObDiagnoseT inst->status_.total_miss_cnt_ = GLOBAL_EVENT_GET(ObStatEventIds::OPT_COLUMN_STAT_CACHE_MISS); } else if (0 == strcmp(inst->status_.config_->cache_name_,"opt_ds_stat_cache")) { inst->status_.total_miss_cnt_ = GLOBAL_EVENT_GET(ObStatEventIds::OPT_DS_STAT_CACHE_MISS); + } else if (0 == strcmp(inst->status_.config_->cache_name_,"log_kv_cache")) { + inst->status_.total_miss_cnt_ = GLOBAL_EVENT_GET(ObStatEventIds::LOG_KV_CACHE_MISS); } return ret; diff --git a/src/rootserver/ob_recovery_ls_service.cpp b/src/rootserver/ob_recovery_ls_service.cpp index 3a8533955d..8b2b2fcbd0 100755 --- a/src/rootserver/ob_recovery_ls_service.cpp +++ b/src/rootserver/ob_recovery_ls_service.cpp @@ -149,7 +149,7 @@ void ObRecoveryLSService::do_work() ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), K(inited_), KP(proxy_)); } else { - palf::PalfBufferIterator iterator;//can not use without palf_guard + palf::PalfBufferIterator iterator(SYS_LS.id(), palf::LogIOUser::RECOVERY);//can not use without palf_guard int64_t idle_time_us = 100 * 1000L; SCN start_scn; last_report_ts_ = OB_INVALID_TIMESTAMP; diff --git a/src/share/allocator/ob_tenant_mutil_allocator.cpp b/src/share/allocator/ob_tenant_mutil_allocator.cpp index 40cc3d5bb4..a547f7629d 100644 --- a/src/share/allocator/ob_tenant_mutil_allocator.cpp +++ b/src/share/allocator/ob_tenant_mutil_allocator.cpp @@ -37,6 +37,7 @@ ObTenantMutilAllocator::ObTenantMutilAllocator(uint64_t tenant_id) PALF_FETCH_LOG_TASK_SIZE(sizeof(palf::FetchLogTask)), LOG_IO_FLASHBACK_TASK_SIZE(sizeof(palf::LogIOFlashbackTask)), LOG_IO_PURGE_THROTTLING_TASK_SIZE(sizeof(palf::LogIOPurgeThrottlingTask)), + LOG_FILL_CACHE_TASK_SIZE(sizeof(palf::LogFillCacheTask)), clog_blk_alloc_(), replay_log_task_blk_alloc_(REPLAY_MEM_LIMIT_THRESHOLD), clog_compressing_blk_alloc_(CLOG_COMPRESSION_MEM_LIMIT_THRESHOLD), @@ -50,6 +51,7 @@ ObTenantMutilAllocator::ObTenantMutilAllocator(uint64_t tenant_id) replay_log_task_alloc_(ObMemAttr(tenant_id, ObModIds::OB_LOG_REPLAY_TASK), common::OB_MALLOC_BIG_BLOCK_SIZE, replay_log_task_blk_alloc_), log_io_flashback_task_alloc_(LOG_IO_FLASHBACK_TASK_SIZE, ObMemAttr(tenant_id, "Flashback"), choose_blk_size(LOG_IO_FLASHBACK_TASK_SIZE), clog_blk_alloc_, this), log_io_purge_throttling_task_alloc_(LOG_IO_PURGE_THROTTLING_TASK_SIZE, ObMemAttr(tenant_id, "PurgeThrottle"), choose_blk_size(LOG_IO_PURGE_THROTTLING_TASK_SIZE), clog_blk_alloc_, this), + log_fill_cache_task_alloc_(LOG_FILL_CACHE_TASK_SIZE, ObMemAttr(tenant_id, "FillCache"), choose_blk_size(LOG_FILL_CACHE_TASK_SIZE), clog_blk_alloc_, this), clog_compression_buf_alloc_(ObMemAttr(tenant_id, "LogComBuf"), common::OB_MALLOC_BIG_BLOCK_SIZE, clog_compressing_blk_alloc_) { // set_nway according to tenant's max_cpu @@ -83,6 +85,7 @@ void ObTenantMutilAllocator::destroy() log_io_purge_throttling_task_alloc_.destroy(); palf_fetch_log_task_alloc_.destroy(); replay_log_task_alloc_.destroy(); + log_fill_cache_task_alloc_.destroy(); clog_compression_buf_alloc_.destroy(); } @@ -112,6 +115,7 @@ void ObTenantMutilAllocator::try_purge() log_io_purge_throttling_task_alloc_.purge_extra_cached_block(0); palf_fetch_log_task_alloc_.purge_extra_cached_block(0); replay_log_task_alloc_.purge_extra_cached_block(0); + log_fill_cache_task_alloc_.purge_extra_cached_block(0); clog_compression_buf_alloc_.purge_extra_cached_block(0); } @@ -327,6 +331,24 @@ void ObTenantMutilAllocator::free_log_io_purge_throttling_task(palf::LogIOPurgeT } } +LogFillCacheTask *ObTenantMutilAllocator::alloc_log_fill_cache_task(const int64_t palf_id, const int64_t palf_epoch) +{ + LogFillCacheTask *ret_ptr = NULL; + void *ptr = log_fill_cache_task_alloc_.alloc(); + if (NULL != ptr) { + ret_ptr = new (ptr) LogFillCacheTask(palf_id, palf_epoch); + } + return ret_ptr; +} + +void ObTenantMutilAllocator::free_log_fill_cache_task(palf::LogFillCacheTask *ptr) +{ + if (OB_LIKELY(NULL != ptr)) { + ptr->~LogFillCacheTask(); + log_fill_cache_task_alloc_.free(ptr); + } +} + void ObTenantMutilAllocator::set_nway(const int32_t nway) { diff --git a/src/share/allocator/ob_tenant_mutil_allocator.h b/src/share/allocator/ob_tenant_mutil_allocator.h index 3ed16264f3..4b015b43ff 100644 --- a/src/share/allocator/ob_tenant_mutil_allocator.h +++ b/src/share/allocator/ob_tenant_mutil_allocator.h @@ -33,6 +33,7 @@ class LogIOFlushMetaTask; class LogIOFlashbackTask; class LogIOPurgeThrottlingTask; class FetchLogTask; +class LogFillCacheTask; } namespace logservice { @@ -75,6 +76,8 @@ public: virtual void free_log_io_flashback_task(palf::LogIOFlashbackTask *ptr) = 0; virtual palf::LogIOPurgeThrottlingTask *alloc_log_io_purge_throttling_task(const int64_t palf_id, const int64_t palf_epoch) = 0; virtual void free_log_io_purge_throttling_task(palf::LogIOPurgeThrottlingTask *ptr) = 0; + virtual palf::LogFillCacheTask *alloc_log_fill_cache_task(const int64_t palf_id, const int64_t palf_epoch) = 0; + virtual void free_log_fill_cache_task(palf::LogFillCacheTask *ptr) = 0; virtual void *alloc_append_compression_buf(const int64_t size) = 0; virtual void free_append_compression_buf(void *ptr) = 0; virtual void *alloc_replay_decompression_buf(const int64_t size) = 0; @@ -150,6 +153,9 @@ public: void free_log_io_flashback_task(palf::LogIOFlashbackTask *ptr); palf::LogIOPurgeThrottlingTask *alloc_log_io_purge_throttling_task(const int64_t palf_id, const int64_t palf_epoch); void free_log_io_purge_throttling_task(palf::LogIOPurgeThrottlingTask *ptr); + palf::LogFillCacheTask *alloc_log_fill_cache_task(const int64_t palf_id, const int64_t palf_epoch); + void free_log_fill_cache_task(palf::LogFillCacheTask *ptr); + void *alloc_append_compression_buf(const int64_t size); void free_append_compression_buf(void *ptr); //alloc buf from replay_log_task_alloc @@ -169,6 +175,7 @@ private: const int PALF_FETCH_LOG_TASK_SIZE; const int LOG_IO_FLASHBACK_TASK_SIZE; const int LOG_IO_PURGE_THROTTLING_TASK_SIZE; + const int LOG_FILL_CACHE_TASK_SIZE; ObBlockAllocMgr clog_blk_alloc_; ObBlockAllocMgr replay_log_task_blk_alloc_; ObBlockAllocMgr clog_compressing_blk_alloc_; @@ -182,6 +189,7 @@ private: ObVSliceAlloc replay_log_task_alloc_; ObSliceAlloc log_io_flashback_task_alloc_; ObSliceAlloc log_io_purge_throttling_task_alloc_; + ObSliceAlloc log_fill_cache_task_alloc_; ObVSliceAlloc clog_compression_buf_alloc_; }; diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index bfa3e19fd1..e7f7c25e84 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -699,6 +699,11 @@ DEF_STR(standby_db_preferred_upstream_log_region, OB_TENANT_PARAMETER, "", "when the preferred upstream log region can not fetch log because of exception etc.", ObParameterAttr(Section::LOGSERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_BOOL(_enable_log_cache, OB_TENANT_PARAMETER, "True", + "specifies whether allow to fill log kv cache. " + "Value: True:turned on False: turned off", + ObParameterAttr(Section::LOGSERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + // ========================= LogService Config End ===================== DEF_INT(resource_hard_limit, OB_CLUSTER_PARAMETER, "100", "[100, 10000]", "system utilization should not be large than resource_hard_limit", diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index 53b17a2ce6..81dddac587 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -298,6 +298,7 @@ _enable_easy_keepalive _enable_hash_join_hasher _enable_hash_join_processor _enable_in_range_optimization +_enable_log_cache _enable_memleak_light_backtrace _enable_newsort _enable_new_sql_nio diff --git a/unittest/logservice/CMakeLists.txt b/unittest/logservice/CMakeLists.txt index d0074e17af..82330cd269 100644 --- a/unittest/logservice/CMakeLists.txt +++ b/unittest/logservice/CMakeLists.txt @@ -40,6 +40,7 @@ ob_unittest(test_palf_throttling) ob_unittest(test_net_standby_restore_source) ob_unittest(test_log_external_storage_handler) ob_unittest(test_log_external_storage_io_task) +ob_unittest(test_log_cache) ob_unittest(test_log_io_utils) if(OB_BUILD_CLOSE_MODULES) ob_unittest(test_arb_gc_utils) diff --git a/unittest/logservice/test_log_cache.cpp b/unittest/logservice/test_log_cache.cpp new file mode 100644 index 0000000000..d940626fd0 --- /dev/null +++ b/unittest/logservice/test_log_cache.cpp @@ -0,0 +1,217 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#include +#define private public +#include "logservice/palf/log_cache.h" +#undef private +#include "logservice/common_util/ob_log_time_utils.h" +#include "share/ob_tenant_mem_limit_getter.h" +#include "share/rc/ob_tenant_base.h" +#include "logservice/palf/log_reader_utils.h" + +namespace oceanbase +{ +namespace unittest +{ +using namespace common; +using namespace palf; +using namespace logservice; +using namespace share; + +static const int64_t KV_CACHE_WASH_TIMER_INTERVAL_US = 60 * _SEC_; +static const int64_t DEFAULT_BUCKET_NUM = 10000000L; +static const int64_t DEFAULT_MAX_CACHE_SIZE = 1024L * 1024L * 1024L * 1024L; + +class TestLogCache : public ::testing::Test +{ +public: + TestLogCache(); + ~TestLogCache(); + virtual void SetUp(); + virtual void TearDown(); +}; + +TestLogCache::TestLogCache() +{ +} + +TestLogCache::~TestLogCache() +{ +} + +void TestLogCache::SetUp() +{ + // init cache + ObKVGlobalCache::get_instance().init(&ObTenantMemLimitGetter::get_instance(), + DEFAULT_BUCKET_NUM, + DEFAULT_MAX_CACHE_SIZE, + lib::ACHUNK_SIZE, + KV_CACHE_WASH_TIMER_INTERVAL_US); + OB_LOG_KV_CACHE.init(OB_LOG_KV_CACHE_NAME, 1); + + // init MTL + ObMallocAllocator::get_instance()->create_and_add_tenant_allocator(1001); + ObTenantBase tbase(1001); + ObTenantEnv::set_tenant(&tbase); +} + +void TestLogCache::TearDown() +{ + PALF_LOG(INFO, "recycle_tenant_allocator start"); + OB_LOG_KV_CACHE.destroy(); + ObMallocAllocator::get_instance()->recycle_tenant_allocator(1001); +} + +LogStorage log_storage; +IPalfEnvImpl *palf_env_impl; + +TEST_F(TestLogCache, test_basic_func) +{ + log_storage.is_inited_ = true; + log_storage.logical_block_size_ = PALF_BLOCK_SIZE; + const int64_t flashback_version = 0; + int64_t palf_id = 1; + LogColdCache cold_cache; + cold_cache.init(palf_id, palf_env_impl, &log_storage); + LSN lsn(0); + int64_t in_read_size = MAX_LOG_BODY_SIZE; + char *buf = reinterpret_cast(ob_malloc(MAX_LOG_BUFFER_SIZE, "LOG_KV_CACHE")); + LogIteratorInfo iterator_info; + { + + int64_t out_read_size = 0; + EXPECT_EQ(OB_ENTRY_NOT_EXIST, cold_cache.get_cache_lines_(lsn, flashback_version, in_read_size, buf, out_read_size, &iterator_info)); + ReadBuf read_buf(buf, MAX_LOG_BUFFER_SIZE); + EXPECT_EQ(OB_SUCCESS, cold_cache.fill_cache_lines_(flashback_version, lsn, MAX_LOG_BODY_SIZE, read_buf.buf_)); + } + + { + iterator_info.reset(); + int64_t out_read_size = 0; + in_read_size = 64 * 1024; + EXPECT_EQ(OB_SUCCESS, cold_cache.get_cache_lines_(lsn, flashback_version, in_read_size, buf, out_read_size, &iterator_info)); + EXPECT_EQ(in_read_size, out_read_size); + } + + { + iterator_info.reset(); + int64_t out_read_size = 0; + lsn.val_ = 1 * 1024 * 1024; + EXPECT_EQ(OB_SUCCESS, cold_cache.get_cache_lines_(lsn, flashback_version, in_read_size, buf, out_read_size, &iterator_info)); + EXPECT_EQ(in_read_size, out_read_size); + } + + { + iterator_info.reset(); + int64_t out_read_size = 0; + in_read_size = MAX_LOG_BODY_SIZE; + EXPECT_EQ(OB_ENTRY_NOT_EXIST, cold_cache.get_cache_lines_(lsn, flashback_version, in_read_size, buf, out_read_size, &iterator_info)); + } + + ob_free(buf); + buf = NULL; +} + +TEST_F(TestLogCache, test_miss) +{ + log_storage.is_inited_ = true; + log_storage.logical_block_size_ = PALF_BLOCK_SIZE; + const int64_t flashback_version = 0; + int64_t palf_id = 1; + LogColdCache cold_cache; + cold_cache.init(palf_id, palf_env_impl, &log_storage); + LogIteratorInfo iterator_info; + // test miss when has_read_size != 0 + { + LSN lsn(1024); + LSN old_lsn(lsn); + int64_t in_read_size = MAX_LOG_BODY_SIZE; + int64_t out_read_size = 0; + int64_t has_read_size = 5000; + int64_t real_read_size = lower_align(has_read_size, LOG_DIO_ALIGN_SIZE); + EXPECT_EQ(OB_SUCCESS, cold_cache.deal_with_miss_(true, has_read_size,lsn, in_read_size , out_read_size, &iterator_info)); + EXPECT_EQ(old_lsn.val_ + real_read_size, lsn.val_); + EXPECT_EQ(MAX_LOG_BODY_SIZE - real_read_size, in_read_size); + EXPECT_EQ(real_read_size, out_read_size); + } + + // test miss for first read + { + iterator_info.reset(); + LSN lsn(67108864); + LSN old_lsn(lsn); + int64_t in_read_size = MAX_LOG_BODY_SIZE; + int64_t has_read_size = 0; + int64_t out_read_size = 0; + EXPECT_EQ(OB_SUCCESS, cold_cache.deal_with_miss_(true, has_read_size, lsn, in_read_size , out_read_size, &iterator_info)); + EXPECT_EQ(PALF_BLOCK_SIZE, lsn.val_); + EXPECT_EQ(MAX_LOG_BODY_SIZE + (old_lsn.val_ - lsn.val_), in_read_size); + } +} + +TEST_F(TestLogCache, test_flashback) +{ + PALF_LOG(INFO, "begin flashback"); + log_storage.is_inited_ = true; + log_storage.logical_block_size_ = PALF_BLOCK_SIZE; + int64_t flashback_version = 0; + int64_t palf_id = 1; + LogColdCache cold_cache; + cold_cache.init(palf_id, palf_env_impl, &log_storage); + LSN lsn(0); + int64_t in_read_size = MAX_LOG_BODY_SIZE; + char *buf = reinterpret_cast(ob_malloc(MAX_LOG_BUFFER_SIZE, "LOG_KV_CACHE")); + ReadBuf read_buf(buf, MAX_LOG_BUFFER_SIZE); + LogIteratorInfo iterator_info; + { + int64_t out_read_size = 0; + EXPECT_EQ(OB_ENTRY_NOT_EXIST, cold_cache.get_cache_lines_(lsn, flashback_version, in_read_size, buf, out_read_size, &iterator_info)); + EXPECT_EQ(OB_SUCCESS, cold_cache.fill_cache_lines_(flashback_version, lsn, MAX_LOG_BODY_SIZE, read_buf.buf_)); + } + + { + iterator_info.reset(); + int64_t out_read_size = 0; + flashback_version++; + int64_t hit_cnt = cold_cache.log_cache_stat_.hit_cnt_; + EXPECT_EQ(OB_ENTRY_NOT_EXIST, cold_cache.get_cache_lines_(lsn, flashback_version, in_read_size, buf, out_read_size, &iterator_info)); + EXPECT_EQ(hit_cnt, cold_cache.log_cache_stat_.hit_cnt_); + } + + { + iterator_info.reset(); + int64_t out_read_size = 0; + EXPECT_EQ(OB_SUCCESS, cold_cache.fill_cache_lines_(flashback_version, lsn, MAX_LOG_BODY_SIZE, read_buf.buf_)); + in_read_size = 64 * 1024; + int hit_cnt = cold_cache.log_cache_stat_.hit_cnt_; + EXPECT_EQ(OB_SUCCESS, cold_cache.get_cache_lines_(lsn, flashback_version, in_read_size, buf, out_read_size, &iterator_info)); + EXPECT_LT(hit_cnt, cold_cache.log_cache_stat_.hit_cnt_); + } + + ob_free(buf); + buf = NULL; +} + +} // end namespace unittest +} // end namespace oceanbase + +int main(int argc, char **argv) +{ + system("rm -f ./test_log_cache.log"); + OB_LOGGER.set_file_name("test_log_cache.log", true, true, "test_log_cache.rs.log"); + OB_LOGGER.set_log_level("TRACE"); + PALF_LOG(INFO, "begin unittest::test_log_cache"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}