From 5d3d5c7cb73acaaeb64117bc97a8e19d45fcc99b Mon Sep 17 00:00:00 2001 From: obdev Date: Fri, 12 Aug 2022 17:11:51 +0800 Subject: [PATCH] [CP] Reduce tmp file block cache memory usage. --- src/sql/executor/ob_interm_result_pool.h | 2 +- .../blocksstable/ob_tmp_file_cache.cpp | 6 +- .../blocksstable/ob_tmp_file_store.cpp | 112 ++++++++++-------- src/storage/blocksstable/ob_tmp_file_store.h | 9 +- .../storage/blocksstable/test_tmp_file.cpp | 22 +++- 5 files changed, 95 insertions(+), 56 deletions(-) diff --git a/src/sql/executor/ob_interm_result_pool.h b/src/sql/executor/ob_interm_result_pool.h index 410bf0f26..0edc47a40 100644 --- a/src/sql/executor/ob_interm_result_pool.h +++ b/src/sql/executor/ob_interm_result_pool.h @@ -32,7 +32,7 @@ public: static const int64_t SCANNER_CAPACITY = 256L << 10; // 256K // adapt to block size in tmp file. - static const int64_t SCANNER_MEM_LIMIT = (8 << 20) - (32 << 10); // 8MB - 32K + static const int64_t SCANNER_MEM_LIMIT = (8 << 20) - (128 << 10); // 8MB - 128K ObIntermResultPool(); virtual ~ObIntermResultPool(); diff --git a/src/storage/blocksstable/ob_tmp_file_cache.cpp b/src/storage/blocksstable/ob_tmp_file_cache.cpp index d78786834..7a5597586 100644 --- a/src/storage/blocksstable/ob_tmp_file_cache.cpp +++ b/src/storage/blocksstable/ob_tmp_file_cache.cpp @@ -150,12 +150,12 @@ int ObTmpPageCache::prefetch( callback.offset_ = info.offset_; callback.buf_size_ = info.size_; callback.allocator_ = &allocator_; - void* buf = allocator_.alloc(sizeof(common::ObSEArray)); + void* buf = allocator_.alloc(sizeof(common::ObSEArray)); if (NULL == buf) { ret = OB_ALLOCATE_MEMORY_FAILED; STORAGE_LOG(WARN, "fail to alloc a buf", K(ret), K(info)); } else { - callback.page_io_infos_ = new (buf) common::ObSEArray(); + callback.page_io_infos_ = new (buf) common::ObSEArray(); callback.page_io_infos_->assign(page_io_infos); if (OB_FAIL(read_io(info, callback, mb_handle))) { if (mb_handle.get_io_handle().is_empty()) { @@ -798,7 +798,7 @@ int ObTmpTenantMemBlockManager::free_extent(const int64_t free_page_nums, const STORAGE_LOG(WARN, "ObTmpBlockCache has not been inited", K(ret)); } else if (free_page_nums < 0 || free_page_nums > mblk_page_nums_ || NULL == t_mblk) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid argument", K(ret), K(free_page_nums), K(*t_mblk)); + STORAGE_LOG(WARN, "invalid argument", K(ret), K(free_page_nums), KPC(t_mblk)); } else if (OB_FAIL(refresh_dir_to_blk_map(t_mblk->get_dir_id(), t_mblk))) { STORAGE_LOG(WARN, "fail to refresh dir_to_blk_map", K(ret), K(*t_mblk)); } else { diff --git a/src/storage/blocksstable/ob_tmp_file_store.cpp b/src/storage/blocksstable/ob_tmp_file_store.cpp index 17430c97a..91e9fec30 100644 --- a/src/storage/blocksstable/ob_tmp_file_store.cpp +++ b/src/storage/blocksstable/ob_tmp_file_store.cpp @@ -39,13 +39,13 @@ int ObTmpFilePageBuddy::init(common::ObIAllocator& allocator) STORAGE_LOG(WARN, "ObTmpFilePageBuddy has not been inited", K(ret)); } else { allocator_ = &allocator; - buf_ = reinterpret_cast( - allocator_->alloc(sizeof(ObTmpFileArea) * std::pow(2, ObTmpFilePageBuddy::MAX_ORDER) - 1)); + buf_ = reinterpret_cast( + allocator_->alloc(sizeof(ObTmpFileArea) * (std::pow(2, MAX_ORDER) - std::pow(2, MIN_ORDER)))); if (NULL == buf_) { ret = OB_ALLOCATE_MEMORY_FAILED; STORAGE_LOG(WARN, "fail to alloc a buf", K(ret)); } else { - max_cont_page_nums_ = std::pow(2, ObTmpFilePageBuddy::MAX_ORDER - 1); + max_cont_page_nums_ = std::pow(2, MAX_ORDER - 1); /** * page buddy free_list for a new block: * -------------- --------- --------- --------- --------- --------- --------- --------- ------- @@ -55,8 +55,11 @@ int ObTmpFilePageBuddy::init(common::ObIAllocator& allocator) * -------------- --------- --------- --------- --------- --------- --------- --------- ------- */ int64_t nums = max_cont_page_nums_; - for (int32_t i = MAX_ORDER - 1; i >= 0; --i) { - char* buf = reinterpret_cast(&(buf_[start_id])); + for (int32_t i = MIN_ORDER - 1; i >= 0; --i) { + free_area_[i] = NULL; + } + for (int32_t i = MAX_ORDER - 1; i >= MIN_ORDER; --i) { + char *buf = reinterpret_cast(&(buf_[start_id])); free_area_[i] = new (buf) ObTmpFileArea(start_id, nums); start_id += nums; nums /= 2; @@ -89,13 +92,14 @@ int ObTmpFilePageBuddy::alloc_all_pages() ret = OB_NOT_INIT; STORAGE_LOG(WARN, "ObTmpFilePageBuddy has not been inited", K(ret)); } else if (is_empty()) { - for (int32_t i = 0; i < ObTmpFilePageBuddy::MAX_ORDER; ++i) { + for (int32_t i = 0; i < MAX_ORDER; ++i) { while (NULL != free_area_[i]) { tmp = free_area_[i]; free_area_[i] = tmp->next_; tmp->~ObTmpFileArea(); } } + max_cont_page_nums_ = 0; } else { ret = OB_ERR_UNEXPECTED; @@ -118,7 +122,7 @@ int ObTmpFilePageBuddy::alloc(const int32_t page_nums, } else { int32_t index = std::ceil(std::log(page_nums) / std::log(2)); bool is_alloced = false; - for (int32_t i = index; i < ObTmpFilePageBuddy::MAX_ORDER && !is_alloced; ++i) { + for (int32_t i = index; i < MAX_ORDER && !is_alloced; ++i) { if (NULL != free_area_[i]) { int64_t num = i - index; ObTmpFileArea* tmp = free_area_[i]; @@ -185,7 +189,13 @@ void ObTmpFilePageBuddy::free_align(const int32_t start_page_id, const int32_t p bool ObTmpFilePageBuddy::is_empty() const { bool is_empty = true; - for (int32_t i = 0; i < ObTmpFilePageBuddy::MAX_ORDER; ++i) { + for (int32_t i = 0; i < MIN_ORDER && is_empty; ++i) { + if (NULL != free_area_[i]) { + is_empty = false; + break; + } + } + for (int32_t i = MIN_ORDER; i < MAX_ORDER && is_empty; ++i) { if (NULL == free_area_[i]) { is_empty = false; break; @@ -200,7 +210,7 @@ int64_t ObTmpFilePageBuddy::to_string(char* buf, const int64_t buf_len) const bool first = true; ObTmpFileArea* area = NULL; common::databuff_printf(buf, buf_len, pos, "{"); - for (int32_t i = 0; i < ObTmpFilePageBuddy::MAX_ORDER; ++i) { + for (int32_t i = 0; i < MAX_ORDER; ++i) { area = free_area_[i]; if (NULL != area) { common::databuff_print_kv(buf, buf_len, pos, "page_nums", static_cast(std::pow(2, i))); @@ -231,45 +241,51 @@ int64_t ObTmpFilePageBuddy::to_string(char* buf, const int64_t buf_len) const void ObTmpFilePageBuddy::free(const int32_t start_page_id, const int32_t page_nums) { - int32_t start_id = start_page_id; - int32_t nums = page_nums; - int32_t length = 0; - while (nums > 0) { - /** - * PURPOSE: align free area into power of 2. - * - * The probable value of alloc_start_id: - * page nums start page id - * 128 0 ---------------- 128 ------------------- 256 - * 64 0 ------ 64 ------ 128 ------- 192 ------- 256 - * 32 0 - 32 - 64 - 96 - 128 - 160 - 192 - 224 - 256 - * ... ... - * So, the maximum number of consecutive pages from a start_page_id is the - * gcd(greatest common divisor) between it and 512, except 0. The maximum - * consecutive page nums of 0 is 256. - * - * The layout of free area in alocated area : - * |<---------------alloc_page_nums--------------->| - * <---- |<--free_page_nums-->| - * |==========================|====================| - * alloc_start free_page_id alloc_end - * - * So, free_end always equal to alloc_end. - * - * Based on two observations above, the algorithm is designed as follows: - */ - length = 2; - while (0 == start_id % length && length <= nums) { - length *= 2; - } - length = std::min(length / 2, nums); + if (OB_UNLIKELY(start_page_id + page_nums >= std::pow(2, MAX_ORDER))) { + STORAGE_LOG(ERROR, "page id more than max numbers in block", K(start_page_id), K(page_nums)); + ob_abort(); + } else { + int32_t start_id = start_page_id; + int32_t nums = page_nums; + int32_t length = 0; + while (nums > 0) { + /** + * PURPOSE: align free area into power of 2. + * + * The probable value of alloc_start_id: + * page nums start page id + * 128 0 ---------------- 128 ------------------- 256 + * 64 0 ------ 64 ------ 128 ------- 192 ------- 256 + * 32 0 - 32 - 64 - 96 - 128 - 160 - 192 - 224 - 256 + * ... ... + * So, the maximum number of consecutive pages from a start_page_id is the + * gcd(greatest common divisor) between it and 512, except 0. The maximum + * consecutive page nums of 0 is 256. + * + * The layout of free area in alocated area : + * |<---------------alloc_page_nums--------------->| + * <---- |<--free_page_nums-->| + * |==========================|====================| + * alloc_start free_page_id alloc_end + * + * So, free_end always equal to alloc_end. + * + * Based on two observations above, the algorithm is designed as follows: + */ + length = 2; + while (0 == start_id % length && length <= nums) { + length *= 2; + } + length = std::min(length / 2, nums); - char* buf = reinterpret_cast(&(buf_[start_id])); - ObTmpFileArea* area = new (buf) ObTmpFileArea(start_id, length); - free_align(area->start_page_id_, area->page_nums_, area); - start_id += length; - nums -= length; + char* buf = reinterpret_cast(&(buf_[start_id])); + ObTmpFileArea* area = new (buf) ObTmpFileArea(start_id, length); + free_align(area->start_page_id_, area->page_nums_, area); + start_id += length; + nums -= length; + } } + } ObTmpFileArea* ObTmpFilePageBuddy::find_buddy(const int32_t page_nums, const int32_t start_page_id) @@ -704,7 +720,7 @@ int64_t ObTmpTenantMacroBlockManager::get_next_blk_id() } ObTmpTenantFileStore::ObTmpTenantFileStore() - : tmp_block_manager_(), tmp_mem_block_manager_(), page_cache_(NULL), file_handle_(), allocator_(), is_inited_(false) + : tmp_block_manager_(), page_cache_(NULL), tmp_mem_block_manager_(), file_handle_(), allocator_(), is_inited_(false) {} ObTmpTenantFileStore::~ObTmpTenantFileStore() @@ -1036,7 +1052,7 @@ int ObTmpTenantFileStore::read_page(ObTmpMacroBlock* block, ObTmpBlockIOInfo& io int64_t remain_size = io_info.size_; int64_t size = std::min(ObTmpMacroBlock::get_default_page_size() - offset, remain_size); int32_t page_nums = 0; - common::ObSEArray page_io_infos; + common::ObSEArray page_io_infos; do { ObTmpPageCacheKey key(io_info.block_id_, page_start_id, io_info.tenant_id_); ObTmpPageValueHandle p_handle; diff --git a/src/storage/blocksstable/ob_tmp_file_store.h b/src/storage/blocksstable/ob_tmp_file_store.h index 5dc765ecd..b26306ce7 100644 --- a/src/storage/blocksstable/ob_tmp_file_store.h +++ b/src/storage/blocksstable/ob_tmp_file_store.h @@ -67,6 +67,8 @@ public: bool is_empty() const; int64_t to_string(char* buf, const int64_t buf_len) const; + static const int64_t MAX_PAGE_NUMS = 252; // 2^MAX_ORDER - 2^MIN_ORDER + private: void free_align(const int32_t start_page_id, const int32_t page_nums, ObTmpFileArea*& area); ObTmpFileArea* find_buddy(const int32_t page_nums, const int32_t start_page_id); @@ -74,6 +76,7 @@ private: { return std::pow(2, ObTmpFilePageBuddy::MAX_ORDER) - 1; } + static const int MIN_ORDER = 2; static const int MAX_ORDER = 8; ObTmpFileArea* free_area_[ObTmpFilePageBuddy::MAX_ORDER]; int64_t max_cont_page_nums_; @@ -295,8 +298,8 @@ private: static constexpr double DEFAULT_PAGE_IO_MERGE_RATIO = 0.5; ObTmpTenantMacroBlockManager tmp_block_manager_; - ObTmpTenantMemBlockManager tmp_mem_block_manager_; ObTmpPageCache* page_cache_; + ObTmpTenantMemBlockManager tmp_mem_block_manager_; ObStorageFileHandle file_handle_; common::ObConcurrentFIFOAllocator allocator_; common::SpinRWLock lock_; @@ -325,11 +328,11 @@ public: OB_INLINE int64_t get_mblk_page_nums() const { - return OB_FILE_SYSTEM.get_macro_block_size() / ObTmpMacroBlock::get_default_page_size() - 1; + return OB_FILE_SYSTEM.get_macro_block_size() / ObTmpMacroBlock::get_default_page_size() - 4; } OB_INLINE int64_t get_block_size() const { - return OB_FILE_SYSTEM.get_macro_block_size() - ObTmpMacroBlock::get_default_page_size(); + return OB_FILE_SYSTEM.get_macro_block_size() - 4 * ObTmpMacroBlock::get_default_page_size(); } private: diff --git a/unittest/storage/blocksstable/test_tmp_file.cpp b/unittest/storage/blocksstable/test_tmp_file.cpp index 05b88c553..361d241f1 100644 --- a/unittest/storage/blocksstable/test_tmp_file.cpp +++ b/unittest/storage/blocksstable/test_tmp_file.cpp @@ -1602,12 +1602,15 @@ TEST_F(TestTmpFile, test_page_buddy) int32_t page_nums = 64; int32_t alloced_page_nums = 64; int32_t start_page_id = -1; + ASSERT_EQ(true, page_buddy_1.is_empty()); ret = page_buddy_1.alloc(page_nums, start_page_id, alloced_page_nums); ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, page_buddy_1.is_empty()); int32_t start_page_id_2 = -1; ret = page_buddy_1.alloc(page_nums, start_page_id_2, alloced_page_nums); ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, page_buddy_1.is_empty()); page_buddy_1.free(start_page_id + 63, page_nums - 63); page_buddy_1.free(start_page_id_2 + 1, page_nums - 1); @@ -1620,11 +1623,14 @@ TEST_F(TestTmpFile, test_page_buddy) ObTmpFilePageBuddy page_buddy_2; ret = page_buddy_2.init(allocator); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, page_buddy_2.is_empty()); start_page_id = 0; ret = page_buddy_2.alloc_all_pages(); ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(false, page_buddy_2.is_empty()); - int32_t free_nums = 511 - 129; + int32_t free_nums = 252 - 129; page_buddy_2.free(start_page_id + 129, free_nums); free_nums = 127; page_buddy_2.free(start_page_id + 2, free_nums); @@ -1645,6 +1651,20 @@ TEST_F(TestTmpFile, test_page_buddy) ASSERT_EQ(true, page_buddy_3.is_empty()); STORAGE_LOG(INFO, "page buddy", K(page_buddy_3)); } + + ObTmpFilePageBuddy page_buddy_4; + ret = page_buddy_4.init(allocator); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(true, page_buddy_4.is_empty()); + + page_nums = 2; + alloced_page_nums = -1; + start_page_id = -1; + ASSERT_EQ(true, page_buddy_4.is_empty()); + ret = page_buddy_4.alloc(page_nums, start_page_id, alloced_page_nums); + ASSERT_EQ(OB_SUCCESS, ret); + ASSERT_EQ(alloced_page_nums, page_nums); + ASSERT_EQ(false, page_buddy_4.is_empty()); } } // end namespace unittest