From 05643dc40340f5028ee0f86757dcf51ebd9a4147 Mon Sep 17 00:00:00 2001 From: ZHAO Chun Date: Sat, 19 Oct 2019 15:53:24 +0800 Subject: [PATCH] Replace Arena with MemPool (#2012) After replacing Arena with MemPool, we can achieve one copy for string value read from segment v2. We can exchange MemPool's chunk between RowBlockV2 and RowBlock. This change only replace Arena, this work will be done in other change list. --- LICENSE.txt | 16 ---- be/src/olap/aggregate_func.h | 4 +- be/src/olap/column_block.h | 13 ++- be/src/olap/field.h | 40 +++------ be/src/olap/generic_iterators.cpp | 3 +- be/src/olap/key_coder.h | 26 +++--- be/src/olap/row.h | 10 --- be/src/olap/row_block2.cpp | 1 + be/src/olap/row_block2.h | 17 ++-- .../rowset/segment_v2/binary_dict_page.cpp | 11 +-- .../olap/rowset/segment_v2/binary_dict_page.h | 7 +- .../rowset/segment_v2/binary_plain_page.h | 4 +- be/src/olap/rowset/segment_v2/column_reader.h | 7 +- .../rowset/segment_v2/column_zone_map.cpp | 10 +-- .../olap/rowset/segment_v2/column_zone_map.h | 9 ++- .../rowset/segment_v2/segment_iterator.cpp | 4 +- be/src/olap/skiplist.h | 2 +- be/src/olap/types.cpp | 1 - be/src/olap/types.h | 20 ----- be/src/util/CMakeLists.txt | 1 - be/src/util/arena.cpp | 68 ---------------- be/src/util/arena.h | 74 ----------------- be/test/olap/delta_writer_test.cpp | 15 ++-- be/test/olap/key_coder_test.cpp | 29 +++---- .../segment_v2/binary_dict_page_test.cpp | 16 ++-- .../segment_v2/binary_plain_page_test.cpp | 18 +++-- .../segment_v2/bitshuffle_page_test.cpp | 17 ++-- .../segment_v2/column_reader_writer_test.cpp | 24 +++--- .../frame_of_reference_page_test.cpp | 17 ++-- .../rowset/segment_v2/plain_page_test.cpp | 17 ++-- .../olap/rowset/segment_v2/rle_page_test.cpp | 18 +++-- .../olap/rowset/segment_v2/segment_test.cpp | 17 ++-- be/test/olap/tablet_schema_helper.h | 8 +- be/test/util/CMakeLists.txt | 1 - be/test/util/arena_test.cpp | 81 ------------------- run-ut.sh | 1 - 36 files changed, 190 insertions(+), 437 deletions(-) delete mode 100644 be/src/util/arena.cpp delete mode 100644 be/src/util/arena.h delete mode 100644 be/test/util/arena_test.cpp diff --git a/LICENSE.txt b/LICENSE.txt index 636a4194a9..e224581f63 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -414,22 +414,6 @@ be/src/olap/skiplist.h : BSD-style license -------------------------------------------------------------------------------- -be/src/util/arena.h : BSD-style license - - Copyright (c) 2011 The LevelDB Authors. All rights reserved. - Use of this source code is governed by a BSD-style license that can be - found in the LICENSE file. See the AUTHORS file for names of contributors. - --------------------------------------------------------------------------------- - -be/src/util/arena.cpp : BSD-style license - - Copyright (c) 2011 The LevelDB Authors. All rights reserved. - Use of this source code is governed by a BSD-style license that can be - found in the LICENSE file. See the AUTHORS file for names of contributors. - --------------------------------------------------------------------------------- - be/src/olap/new_status.h : BSD-style license Copyright (c) 2011 The LevelDB Authors. All rights reserved. diff --git a/be/src/olap/aggregate_func.h b/be/src/olap/aggregate_func.h index 17485baf1b..3540581750 100644 --- a/be/src/olap/aggregate_func.h +++ b/be/src/olap/aggregate_func.h @@ -42,7 +42,7 @@ public: // Init function will initialize aggregation execute environment in dst with source // and convert the source raw data to storage aggregate format // - // Memory Note: For plain memory can be allocated from arena, whose lifetime + // Memory Note: For plain memory can be allocated from *mem_pool, whose lifetime // will last util finalize function is called. Memory allocated from heap should // be freed in finalize functioin to avoid memory leak. inline void init(RowCursorCell* dst, const char* src, bool src_null, MemPool* mem_pool, ObjectPool* agg_pool) const { @@ -66,7 +66,7 @@ public: // saved in src. // // Memory Note: All heap memory allocated in init and update function should be freed - // before this function return. Memory allocated from arena will be still available + // before this function return. Memory allocated from *mem_pool will be still available // and will be freed by client. inline void finalize(RowCursorCell* src, MemPool* mem_pool) const { _finalize_fn(src, mem_pool); diff --git a/be/src/olap/column_block.h b/be/src/olap/column_block.h index 0b82617df0..4c0d36a66e 100644 --- a/be/src/olap/column_block.h +++ b/be/src/olap/column_block.h @@ -24,23 +24,22 @@ namespace doris { -class Arena; +class MemPool; class TypeInfo; - class ColumnBlockCell; // Block of data belong to a single column. // It doesn't own any data, user should keep the life of input data. class ColumnBlock { public: - ColumnBlock(const TypeInfo* type_info, uint8_t* data, uint8_t* null_bitmap, size_t nrows, Arena* arena) - : _type_info(type_info), _data(data), _null_bitmap(null_bitmap), _nrows(nrows), _arena(arena) { } + ColumnBlock(const TypeInfo* type_info, uint8_t* data, uint8_t* null_bitmap, size_t nrows, MemPool* pool) + : _type_info(type_info), _data(data), _null_bitmap(null_bitmap), _nrows(nrows), _pool(pool) { } const TypeInfo* type_info() const { return _type_info; } uint8_t* data() const { return _data; } uint8_t* null_bitmap() const { return _null_bitmap; } bool is_nullable() const { return _null_bitmap != nullptr; } - Arena* arena() const { return _arena; } + MemPool* pool() const { return _pool; } const uint8_t* cell_ptr(size_t idx) const { return _data + idx * _type_info->size(); } uint8_t* mutable_cell_ptr(size_t idx) const { return _data + idx * _type_info->size(); } bool is_null(size_t idx) const { @@ -61,7 +60,7 @@ private: uint8_t* _data; uint8_t* _null_bitmap; size_t _nrows; - Arena* _arena; + MemPool* _pool; }; struct ColumnBlockCell { @@ -90,7 +89,7 @@ public: void advance(size_t skip) { _row_offset += skip; } size_t first_row_index() const { return _row_offset; } ColumnBlock* column_block() { return _block; } - Arena* arena() const { return _block->arena(); } + MemPool* pool() const { return _block->pool(); } void set_null_bits(size_t num_rows, bool val) { BitmapChangeBits(_block->null_bitmap(), _row_offset, num_rows, val); } diff --git a/be/src/olap/field.h b/be/src/olap/field.h index ad513778b7..d1287118a9 100644 --- a/be/src/olap/field.h +++ b/be/src/olap/field.h @@ -57,7 +57,10 @@ public: virtual inline void set_to_max(char* buf) const { return _type_info->set_to_max(buf); } inline void set_to_min(char* buf) const { return _type_info->set_to_min(buf); } - virtual inline char* allocate_value_from_arena(Arena* arena) const { return arena->Allocate(_type_info->size()); } + + // This function allocate memory from pool, other than allocate_memory + // reserve memory from continuous memory. + virtual inline char* allocate_value(MemPool* pool) const { return (char*)pool->allocate(_type_info->size()); } inline void agg_update(RowCursorCell* dest, const RowCursorCell& src, MemPool* mem_pool = nullptr) const { _agg_info->update(dest, src, mem_pool); @@ -172,18 +175,6 @@ public: _type_info->deep_copy(dst->mutable_cell_ptr(), src.cell_ptr(), pool); } - template - void deep_copy(DstCellType* dst, - const SrcCellType& src, - Arena* arena) const { - bool is_null = src.is_null(); - dst->set_is_null(is_null); - if (is_null) { - return; - } - _type_info->deep_copy_with_arena(dst->mutable_cell_ptr(), src.cell_ptr(), arena); - } - // deep copy filed content from `src` to `dst` without null-byte inline void deep_copy_content(char* dst, const char* src, MemPool* mem_pool) const { _type_info->deep_copy(dst, src, mem_pool); @@ -195,11 +186,6 @@ public: _type_info->shallow_copy(dst, src); } - // copy filed content from src to dest without nullbyte - inline void deep_copy_content(char* dest, const char* src, Arena* arena) const { - _type_info->deep_copy_with_arena(dest, src, arena); - } - // Copy srouce content to destination in index format. template void to_index(DstCellType* dst, const SrcCellType& src) const; @@ -240,8 +226,8 @@ public: _key_coder->encode_ascending(value, _index_size, buf); } - Status decode_ascending(Slice* encoded_key, uint8_t* cell_ptr, Arena* arena) const { - return _key_coder->decode_ascending(encoded_key, _index_size, cell_ptr, arena); + Status decode_ascending(Slice* encoded_key, uint8_t* cell_ptr, MemPool* pool) const { + return _key_coder->decode_ascending(encoded_key, _index_size, cell_ptr, pool); } private: // Field的最大长度，单位为字节，通常等于length， 变长字符串不同 @@ -256,11 +242,11 @@ protected: // 除字符串外，其它类型都是确定的 uint32_t _length; - char* allocate_string_value_from_arena(Arena* arena) const { - char* type_value = arena->Allocate(sizeof(Slice)); + char* allocate_string_value(MemPool* pool) const { + char* type_value = (char*)pool->allocate(sizeof(Slice)); auto slice = reinterpret_cast(type_value); slice->size = _length; - slice->data = arena->Allocate(slice->size); + slice->data = (char*)pool->allocate(slice->size); return type_value; }; }; @@ -383,8 +369,8 @@ public: return new CharField(*this); } - char* allocate_value_from_arena(Arena* arena) const override { - return Field::allocate_string_value_from_arena(arena); + char* allocate_value(MemPool* pool) const override { + return Field::allocate_string_value(pool); } void set_to_max(char* ch) const override { @@ -416,8 +402,8 @@ public: return new VarcharField(*this); } - char* allocate_value_from_arena(Arena* arena) const override { - return Field::allocate_string_value_from_arena(arena); + char* allocate_value(MemPool* pool) const override { + return Field::allocate_string_value(pool); } void set_to_max(char* ch) const override { diff --git a/be/src/olap/generic_iterators.cpp b/be/src/olap/generic_iterators.cpp index 2d540cfecc..746d3668b7 100644 --- a/be/src/olap/generic_iterators.cpp +++ b/be/src/olap/generic_iterators.cpp @@ -22,7 +22,6 @@ #include "olap/row_block2.h" #include "olap/row_cursor_cell.h" #include "olap/row.h" -#include "util/arena.h" namespace doris { @@ -261,7 +260,7 @@ Status MergeIterator::next_batch(RowBlockV2* block) { RowBlockRow dst_row = block->row(row_idx); // copy current row to block - copy_row(&dst_row, ctx->current_row(), block->arena()); + copy_row(&dst_row, ctx->current_row(), block->pool()); RETURN_IF_ERROR(ctx->advance()); if (ctx->valid()) { diff --git a/be/src/olap/key_coder.h b/be/src/olap/key_coder.h index 773c4e9ddb..5c79fec95e 100644 --- a/be/src/olap/key_coder.h +++ b/be/src/olap/key_coder.h @@ -24,14 +24,14 @@ #include "gutil/endian.h" #include "gutil/strings/substitute.h" #include "olap/types.h" -#include "util/arena.h" +#include "runtime/mem_pool.h" namespace doris { using strings::Substitute; using EncodeAscendingFunc = void (*)(const void* value, size_t index_size, std::string* buf); -using DecodeAscendingFunc = Status (*)(Slice* encoded_key, size_t index_size, uint8_t* cell_ptr, Arena* arena); +using DecodeAscendingFunc = Status (*)(Slice* encoded_key, size_t index_size, uint8_t* cell_ptr, MemPool* pool); // Helper class that is used to encode types of value in memory format // into a sorted binary. For example, this class will encode unsigned @@ -44,8 +44,8 @@ public: void encode_ascending(const void* value, size_t index_size, std::string* buf) const { _encode_ascending(value, index_size, buf); } - Status decode_ascending(Slice* encoded_key, size_t index_size, uint8_t* cell_ptr, Arena* arena) const { - return _decode_ascending(encoded_key, index_size, cell_ptr, arena); + Status decode_ascending(Slice* encoded_key, size_t index_size, uint8_t* cell_ptr, MemPool* pool) const { + return _decode_ascending(encoded_key, index_size, cell_ptr, pool); } private: @@ -97,7 +97,7 @@ public: } static Status decode_ascending(Slice* encoded_key, size_t index_size, - uint8_t* cell_ptr, Arena* arena) { + uint8_t* cell_ptr, MemPool* pool) { if (encoded_key->size < sizeof(UnsignedCppType)) { return Status::InvalidArgument( Substitute("Key too short, need=$0 vs real=$1", @@ -131,7 +131,7 @@ public: } static Status decode_ascending(Slice* encoded_key, size_t index_size, - uint8_t* cell_ptr, Arena* arena) { + uint8_t* cell_ptr, MemPool* pool) { if (encoded_key->size < sizeof(UnsignedCppType)) { return Status::InvalidArgument( Substitute("Key too short, need=$0 vs real=$1", @@ -161,12 +161,12 @@ public: } static Status decode_ascending(Slice* encoded_key, size_t index_size, - uint8_t* cell_ptr, Arena* arena) { + uint8_t* cell_ptr, MemPool* pool) { decimal12_t decimal_val; RETURN_IF_ERROR(KeyCoderTraits::decode_ascending( - encoded_key, sizeof(decimal_val.integer), (uint8_t*)&decimal_val.integer, arena)); + encoded_key, sizeof(decimal_val.integer), (uint8_t*)&decimal_val.integer, pool)); RETURN_IF_ERROR(KeyCoderTraits::decode_ascending( - encoded_key, sizeof(decimal_val.fraction), (uint8_t*)&decimal_val.fraction, arena)); + encoded_key, sizeof(decimal_val.fraction), (uint8_t*)&decimal_val.fraction, pool)); memcpy(cell_ptr, &decimal_val, sizeof(decimal12_t)); return Status::OK(); } @@ -182,14 +182,14 @@ public: } static Status decode_ascending(Slice* encoded_key, size_t index_size, - uint8_t* cell_ptr, Arena* arena) { + uint8_t* cell_ptr, MemPool* pool) { if (encoded_key->size < index_size) { return Status::InvalidArgument( Substitute("Key too short, need=$0 vs real=$1", index_size, encoded_key->size)); } Slice* slice = (Slice*)cell_ptr; - slice->data = arena->Allocate(index_size); + slice->data = (char*)pool->allocate(index_size); slice->size = index_size; memcpy(slice->data, encoded_key->data, index_size); encoded_key->remove_prefix(index_size); @@ -207,13 +207,13 @@ public: } static Status decode_ascending(Slice* encoded_key, size_t index_size, - uint8_t* cell_ptr, Arena* arena) { + uint8_t* cell_ptr, MemPool* pool) { CHECK(encoded_key->size <= index_size) << "encoded_key size is larger than index_size, key_size=" << encoded_key->size << ", index_size=" << index_size; auto copy_size = encoded_key->size; Slice* slice = (Slice*)cell_ptr; - slice->data = arena->Allocate(copy_size); + slice->data = (char*)pool->allocate(copy_size); slice->size = copy_size; memcpy(slice->data, encoded_key->data, copy_size); encoded_key->remove_prefix(copy_size); diff --git a/be/src/olap/row.h b/be/src/olap/row.h index b6b2926836..760941c074 100644 --- a/be/src/olap/row.h +++ b/be/src/olap/row.h @@ -134,16 +134,6 @@ void copy_row(DstRowType* dst, const SrcRowType& src, MemPool* pool) { } } -// Deep copy src row to dst row. Schema of src and dst row must be same. -template -void copy_row(DstRowType* dst, const SrcRowType& src, Arena* arena) { - for (auto cid : dst->schema()->column_ids()) { - auto dst_cell = dst->cell(cid); - auto src_cell = src.cell(cid); - dst->schema()->column(cid)->deep_copy(&dst_cell, src_cell, arena); - } -} - template void agg_update_row(DstRowType* dst, const SrcRowType& src, MemPool* mem_pool) { for (uint32_t cid = dst->schema()->num_key_columns(); cid < dst->schema()->num_columns(); ++cid) { diff --git a/be/src/olap/row_block2.cpp b/be/src/olap/row_block2.cpp index da612eecf6..69d5c1ddcd 100644 --- a/be/src/olap/row_block2.cpp +++ b/be/src/olap/row_block2.cpp @@ -31,6 +31,7 @@ RowBlockV2::RowBlockV2(const Schema& schema, uint16_t capacity) _capacity(capacity), _column_datas(_schema.num_columns(), nullptr), _column_null_bitmaps(_schema.num_columns(), nullptr), + _pool(new MemPool(&_tracker)), _selection_vector(nullptr) { auto bitmap_size = BitmapSize(capacity); for (auto cid : _schema.column_ids()) { diff --git a/be/src/olap/row_block2.h b/be/src/olap/row_block2.h index e867958548..ce3d5b65bd 100644 --- a/be/src/olap/row_block2.h +++ b/be/src/olap/row_block2.h @@ -26,10 +26,12 @@ #include "olap/schema.h" #include "olap/types.h" #include "olap/selection_vector.h" +#include "runtime/mem_pool.h" +#include "runtime/mem_tracker.h" namespace doris { -class Arena; +class MemPool; class RowCursor; // This struct contains a block of rows, in which each column's data is stored @@ -47,14 +49,14 @@ public: // return the maximum number of rows that can be contained in this block. // invariant: 0 <= num_rows() <= capacity() size_t capacity() const { return _capacity; } - Arena* arena() const { return _arena.get(); } + MemPool* pool() const { return _pool.get(); } // reset the state of the block so that it can be reused for write. // all previously returned ColumnBlocks are invalidated after clear(), accessing them // will result in undefined behavior. void clear() { _num_rows = 0; - _arena.reset(new Arena); + _pool->clear(); _selected_size = _capacity; for (int i = 0; i < _selected_size; ++i) { _selection_vector[i] = i; @@ -77,7 +79,7 @@ public: const TypeInfo* type_info = _schema.column(cid)->type_info(); uint8_t* data = _column_datas[cid]; uint8_t* null_bitmap = _column_null_bitmaps[cid]; - return ColumnBlock(type_info, data, null_bitmap, _capacity, _arena.get()); + return ColumnBlock(type_info, data, null_bitmap, _capacity, _pool.get()); } RowBlockRow row(size_t row_idx) const; @@ -101,15 +103,16 @@ private: size_t _capacity; // keeps fixed-size (field_size x capacity) data vector for each column, // _column_datas[cid] == null if cid is not in `_schema`. - // memory are not allocated from `_arena` because we don't wan't to reallocate them in clear() + // memory are not allocated from `_pool` because we don't wan't to reallocate them in clear() std::vector _column_datas; // keeps null bitmap for each column, // _column_null_bitmaps[cid] == null if cid is not in `_schema` or the column is not null. - // memory are not allocated from `_arena` because we don't wan't to reallocate them in clear() + // memory are not allocated from `_pool` because we don't wan't to reallocate them in clear() std::vector _column_null_bitmaps; size_t _num_rows; // manages the memory for slice's data - std::unique_ptr _arena; + MemTracker _tracker; + std::unique_ptr _pool; // index of selected rows for rows passed the predicate uint16_t* _selection_vector; diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp index 706de61752..61c9e360c6 100644 --- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp +++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp @@ -34,7 +34,8 @@ BinaryDictPageBuilder::BinaryDictPageBuilder(const PageBuilderOptions& options) _finished(false), _data_page_builder(nullptr), _dict_builder(nullptr), - _encoding_type(DICT_ENCODING) { + _encoding_type(DICT_ENCODING), + _pool(&_tracker) { // initially use DICT_ENCODING // TODO: the data page builder type can be created by Factory according to user config _data_page_builder.reset(new BitshufflePageBuilder(options)); @@ -71,7 +72,7 @@ Status BinaryDictPageBuilder::add(const uint8_t* vals, size_t* count) { } Slice dict_item(src->data, src->size); if (src->size > 0) { - char* item_mem = _arena.Allocate(src->size); + char* item_mem = (char*)_pool.allocate(src->size); if (item_mem == nullptr) { return Status::MemoryAllocFailed(Substitute("memory allocate failed, size:$0", src->size)); } @@ -127,7 +128,7 @@ size_t BinaryDictPageBuilder::count() const { } uint64_t BinaryDictPageBuilder::size() const { - return _arena.MemoryUsage() + _data_page_builder->size(); + return _pool.total_allocated_bytes() + _data_page_builder->size(); } Status BinaryDictPageBuilder::get_dictionary_page(Slice* dictionary_page) { @@ -206,7 +207,7 @@ Status BinaryDictPageDecoder::next_batch(size_t* n, ColumnBlockView* dst) { // And then copy the strings corresponding to the codewords to the destination buffer TypeInfo *type_info = get_type_info(OLAP_FIELD_TYPE_INT); // the data in page is not null - ColumnBlock column_block(type_info, _code_buf.data(), nullptr, *n, dst->column_block()->arena()); + ColumnBlock column_block(type_info, _code_buf.data(), nullptr, *n, dst->column_block()->pool()); ColumnBlockView tmp_block_view(&column_block); RETURN_IF_ERROR(_data_page_decoder->next_batch(n, &tmp_block_view)); for (int i = 0; i < *n; ++i) { @@ -214,7 +215,7 @@ Status BinaryDictPageDecoder::next_batch(size_t* n, ColumnBlockView* dst) { // get the string from the dict decoder Slice element = _dict_decoder->string_at_index(codeword); if (element.size > 0) { - char* destination = dst->column_block()->arena()->Allocate(element.size); + char* destination = (char*)dst->column_block()->pool()->allocate(element.size); if (destination == nullptr) { return Status::MemoryAllocFailed(Substitute( "memory allocate failed, size:$0", element.size)); diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.h b/be/src/olap/rowset/segment_v2/binary_dict_page.h index 7951efd7bf..6a2ef32a92 100644 --- a/be/src/olap/rowset/segment_v2/binary_dict_page.h +++ b/be/src/olap/rowset/segment_v2/binary_dict_page.h @@ -28,7 +28,8 @@ #include "olap/rowset/segment_v2/options.h" #include "olap/rowset/segment_v2/common.h" #include "olap/column_block.h" -#include "util/arena.h" +#include "runtime/mem_pool.h" +#include "runtime/mem_tracker.h" #include "gen_cpp/segment_v2.pb.h" #include "gutil/hash/string_hash.h" @@ -94,7 +95,9 @@ private: std::unordered_map _dictionary; // used to remember the insertion order of dict keys std::vector _dict_items; - Arena _arena; + // TODO(zc): rethink about this mem pool + MemTracker _tracker; + MemPool _pool; faststring _buffer; }; diff --git a/be/src/olap/rowset/segment_v2/binary_plain_page.h b/be/src/olap/rowset/segment_v2/binary_plain_page.h index d085164947..b442eb85e3 100644 --- a/be/src/olap/rowset/segment_v2/binary_plain_page.h +++ b/be/src/olap/rowset/segment_v2/binary_plain_page.h @@ -28,7 +28,6 @@ #pragma once -#include "util/arena.h" #include "util/coding.h" #include "util/faststring.h" #include "olap/olap_common.h" @@ -36,6 +35,7 @@ #include "olap/rowset/segment_v2/page_decoder.h" #include "olap/rowset/segment_v2/options.h" #include "olap/types.h" +#include "runtime/mem_pool.h" namespace doris { namespace segment_v2 { @@ -183,7 +183,7 @@ public: Slice elem(string_at_index(_cur_idx)); out->size = elem.size; if (elem.size != 0) { - out->data = reinterpret_cast(dst->arena()->Allocate(elem.size * sizeof(uint8_t))); + out->data = reinterpret_cast(dst->pool()->allocate(elem.size * sizeof(uint8_t))); memcpy(out->data, elem.data, elem.size); } } diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index af34db2903..af0de1f9ef 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -34,7 +34,6 @@ namespace doris { class ColumnBlock; -class Arena; class RandomAccessFile; class TypeInfo; class BlockCompressionCodec; @@ -134,7 +133,7 @@ public: // After one seek, we can call this function many times to read data // into ColumnBlock. when read string type data, memory will allocated - // from Arena + // from MemPool virtual Status next_batch(size_t* n, ColumnBlock* dst) = 0; virtual rowid_t get_current_ordinal() const = 0; @@ -151,8 +150,8 @@ public: // // In the case that the values are themselves references // to other memory (eg Slices), the referred-to memory is - // allocated in the dst column vector's arena. - Status scan(size_t* n, ColumnBlock* dst, Arena* arena); + // allocated in the dst column vector's MemPool. + Status scan(size_t* n, ColumnBlock* dst, MemPool* pool); // release next_batch related resource Status finish_batch(); diff --git a/be/src/olap/rowset/segment_v2/column_zone_map.cpp b/be/src/olap/rowset/segment_v2/column_zone_map.cpp index 88609f10ab..659880c219 100644 --- a/be/src/olap/rowset/segment_v2/column_zone_map.cpp +++ b/be/src/olap/rowset/segment_v2/column_zone_map.cpp @@ -23,15 +23,15 @@ namespace doris { namespace segment_v2 { -ColumnZoneMapBuilder::ColumnZoneMapBuilder(Field* field) : _field(field) { +ColumnZoneMapBuilder::ColumnZoneMapBuilder(Field* field) : _field(field), _pool(&_tracker) { PageBuilderOptions options; options.data_page_size = 0; _page_builder.reset(new BinaryPlainPageBuilder(options)); - _zone_map.min_value = _field->allocate_value_from_arena(&_arena); - _zone_map.max_value = _field->allocate_value_from_arena(&_arena); + _zone_map.min_value = _field->allocate_value(&_pool); + _zone_map.max_value = _field->allocate_value(&_pool); _reset_page_zone_map(); - _segment_zone_map.min_value = _field->allocate_value_from_arena(&_arena); - _segment_zone_map.max_value = _field->allocate_value_from_arena(&_arena); + _segment_zone_map.min_value = _field->allocate_value(&_pool); + _segment_zone_map.max_value = _field->allocate_value(&_pool); _reset_segment_zone_map(); } diff --git a/be/src/olap/rowset/segment_v2/column_zone_map.h b/be/src/olap/rowset/segment_v2/column_zone_map.h index 3ed3f0ceba..f4aaa45f50 100644 --- a/be/src/olap/rowset/segment_v2/column_zone_map.h +++ b/be/src/olap/rowset/segment_v2/column_zone_map.h @@ -25,6 +25,8 @@ #include "olap/field.h" #include "gen_cpp/segment_v2.pb.h" #include "olap/rowset/segment_v2/binary_plain_page.h" +#include "runtime/mem_pool.h" +#include "runtime/mem_tracker.h" namespace doris { @@ -75,10 +77,13 @@ private: private: std::unique_ptr _page_builder; Field* _field; - // memory will be managed by arena + // memory will be managed by MemPool ZoneMap _zone_map; ZoneMap _segment_zone_map; - Arena _arena; + // TODO(zc): we should replace this memory pool later, we only allocate min/max + // for field. But MemPool allocate 4KB least, it will a waste for most cases. + MemTracker _tracker; + MemPool _pool; }; // ColumnZoneMap diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 62043f5b4f..d3667e6c44 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -271,9 +271,9 @@ Status SegmentIterator::_seek_and_peek(rowid_t rowid) { _column_iterators[cid]->seek_to_ordinal(rowid); } size_t num_rows = 1; - // please note that usually RowBlockV2.clear() is called to free arena memory before reading the next block, + // please note that usually RowBlockV2.clear() is called to free MemPool memory before reading the next block, // but here since there won't be too many keys to seek, we don't call RowBlockV2.clear() so that we can use - // a single arena for all seeked keys. + // a single MemPool for all seeked keys. RETURN_IF_ERROR(_next_batch(_seek_block.get(), &num_rows)); _seek_block->set_num_rows(num_rows); return Status::OK(); diff --git a/be/src/olap/skiplist.h b/be/src/olap/skiplist.h index fb1da641e6..9f1a5d0f87 100644 --- a/be/src/olap/skiplist.h +++ b/be/src/olap/skiplist.h @@ -44,7 +44,7 @@ private: public: // Create a new SkipList object that will use "cmp" for comparing keys, - // and will allocate memory using "*arena". Objects allocated in the arena + // and will allocate memory using "*mem_pool". Objects allocated in the mem_pool // must remain allocated for the lifetime of the skiplist object. explicit SkipList(Comparator cmp, MemPool* mem_pool); diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp index d7f976851e..9ff8afcc62 100644 --- a/be/src/olap/types.cpp +++ b/be/src/olap/types.cpp @@ -27,7 +27,6 @@ TypeInfo::TypeInfo(TypeTraitsClass t) _cmp(TypeTraitsClass::cmp), _shallow_copy(TypeTraitsClass::shallow_copy), _deep_copy(TypeTraitsClass::deep_copy), - _deep_copy_with_arena(TypeTraitsClass::deep_copy_with_arena), _direct_copy(TypeTraitsClass::direct_copy), _from_string(TypeTraitsClass::from_string), _to_string(TypeTraitsClass::to_string), diff --git a/be/src/olap/types.h b/be/src/olap/types.h index 78bc09315c..3bb0aa22ea 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -34,7 +34,6 @@ #include "util/mem_util.hpp" #include "util/slice.h" #include "util/types.h" -#include "util/arena.h" namespace doris { @@ -56,10 +55,6 @@ public: _deep_copy(dest, src, mem_pool); } - inline void deep_copy_with_arena(void* dest, const void* src, Arena* arena) const { - _deep_copy_with_arena(dest, src, arena); - } - inline void direct_copy(void* dest, const void* src) const { _direct_copy(dest, src); } @@ -83,7 +78,6 @@ private: void (*_shallow_copy)(void* dest, const void* src); void (*_deep_copy)(void* dest, const void* src, MemPool* mem_pool); - void (*_deep_copy_with_arena)(void* dest, const void* src, Arena* arena); void (*_direct_copy)(void* dest, const void* src); OLAPStatus (*_from_string)(void* buf, const std::string& scan_key); @@ -193,10 +187,6 @@ struct BaseFieldtypeTraits : public CppTypeTraits { *reinterpret_cast(dest) = *reinterpret_cast(src); } - static inline void deep_copy_with_arena(void* dest, const void* src, Arena* arena) { - *reinterpret_cast(dest) = *reinterpret_cast(src); - } - static inline void direct_copy(void* dest, const void* src) { *reinterpret_cast(dest) = *reinterpret_cast(src); } @@ -343,9 +333,6 @@ struct FieldTypeTraits : public BaseFieldtypeTraits
    (dest) = *reinterpret_cast(src); } - static void deep_copy_with_arena(void* dest, const void* src, Arena* arena) { - *reinterpret_cast(dest) = *reinterpret_cast(src); - } static void direct_copy(void* dest, const void* src) { *reinterpret_cast(dest) = *reinterpret_cast(src); } @@ -541,13 +528,6 @@ struct FieldTypeTraits : public BaseFieldtypeTraitsdata, r_slice->data, r_slice->size); l_slice->size = r_slice->size; } - static void deep_copy_with_arena(void* dest, const void* src, Arena* arena) { - auto l_slice = reinterpret_cast(dest); - auto r_slice = reinterpret_cast(src); - l_slice->data = reinterpret_cast(arena->Allocate(r_slice->size)); - memory_copy(l_slice->data, r_slice->data, r_slice->size); - l_slice->size = r_slice->size; - } static void direct_copy(void* dest, const void* src) { auto l_slice = reinterpret_cast(dest); auto r_slice = reinterpret_cast(src); diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt index 66e8a9238d..a69afcc3f0 100644 --- a/be/src/util/CMakeLists.txt +++ b/be/src/util/CMakeLists.txt @@ -25,7 +25,6 @@ set(UTIL_FILES arrow/row_batch.cpp arrow/row_block.cpp arrow/utils.cpp - arena.cpp bfd_parser.cpp bitmap.cpp block_compression.cpp diff --git a/be/src/util/arena.cpp b/be/src/util/arena.cpp deleted file mode 100644 index cf72db82b8..0000000000 --- a/be/src/util/arena.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#include "util/arena.h" -#include - -namespace doris { - -static const int kBlockSize = 4096; - -Arena::Arena() : memory_usage_(0) { - alloc_ptr_ = NULL; // First allocation will allocate a block - alloc_bytes_remaining_ = 0; -} - -Arena::~Arena() { - for (size_t i = 0; i < blocks_.size(); i++) { - delete[] blocks_[i]; - } -} - -char* Arena::AllocateFallback(size_t bytes) { - if (bytes > kBlockSize / 4) { - // Object is more than a quarter of our block size. Allocate it separately - // to avoid wasting too much space in leftover bytes. - char* result = AllocateNewBlock(bytes); - return result; - } - - // We waste the remaining space in the current block. - alloc_ptr_ = AllocateNewBlock(kBlockSize); - alloc_bytes_remaining_ = kBlockSize; - - char* result = alloc_ptr_; - alloc_ptr_ += bytes; - alloc_bytes_remaining_ -= bytes; - return result; -} - -char* Arena::AllocateAligned(size_t bytes) { - const int align = (sizeof(void*) > 8) ? sizeof(void*) : 8; - assert((align & (align-1)) == 0); // Pointer size should be a power of 2 - size_t current_mod = reinterpret_cast(alloc_ptr_) & (align-1); - size_t slop = (current_mod == 0 ? 0 : align - current_mod); - size_t needed = bytes + slop; - char* result; - if (needed <= alloc_bytes_remaining_) { - result = alloc_ptr_ + slop; - alloc_ptr_ += needed; - alloc_bytes_remaining_ -= needed; - } else { - // AllocateFallback always returned aligned memory - result = AllocateFallback(bytes); - } - assert((reinterpret_cast(result) & (align-1)) == 0); - return result; -} - -char* Arena::AllocateNewBlock(size_t block_bytes) { - char* result = new char[block_bytes]; - blocks_.push_back(result); - memory_usage_.store(MemoryUsage() + block_bytes + sizeof(char*), - std::memory_order_relaxed); - return result; -} - -} // namespace doris diff --git a/be/src/util/arena.h b/be/src/util/arena.h deleted file mode 100644 index ef042eb768..0000000000 --- a/be/src/util/arena.h +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - -#ifndef DORIS_BE_SRC_COMMON_UTIL_ARENA_H -#define DORIS_BE_SRC_COMMON_UTIL_ARENA_H - -#include -#include -#include - -#include -#include - -#include "common/compiler_util.h" - -namespace doris { - -class Arena { -public: - Arena(); - ~Arena(); - - // Return a pointer to a newly allocated memory block of "bytes" bytes. - char* Allocate(size_t bytes); - - // Allocate memory with the normal alignment guarantees provided by malloc - char* AllocateAligned(size_t bytes); - - // Returns an estimate of the total memory usage of data allocated - // by the arena. - size_t MemoryUsage() const { - //return reinterpret_cast(memory_usage_.NoBarrier_Load()); - return memory_usage_.load(std::memory_order_relaxed); - } - - // For the object wasn't allocated from Arena, but need to - // collect and control the object memory usage. - void track_memory(size_t bytes) { - memory_usage_.store(MemoryUsage() + bytes,std::memory_order_relaxed); - } -private: - char* AllocateFallback(size_t bytes); - char* AllocateNewBlock(size_t block_bytes); - - // Allocation state - char* alloc_ptr_; - size_t alloc_bytes_remaining_; - - // Array of new[] allocated memory blocks - std::vector blocks_; - - // Total memory usage of the arena. - std::atomic memory_usage_; - - // No copying allowed - Arena(const Arena&); - void operator=(const Arena&); -}; - -inline char* Arena::Allocate(size_t bytes) { - if (UNLIKELY(bytes == 0)) { return nullptr; } - if (bytes <= alloc_bytes_remaining_) { - char* result = alloc_ptr_; - alloc_ptr_ += bytes; - alloc_bytes_remaining_ -= bytes; - return result; - } - return AllocateFallback(bytes); -} - -} // namespace doris - -#endif // DORIS_BE_SRC_COMMON_UTIL_ARENA_H diff --git a/be/test/olap/delta_writer_test.cpp b/be/test/olap/delta_writer_test.cpp index 495d011437..134f09ba78 100644 --- a/be/test/olap/delta_writer_test.cpp +++ b/be/test/olap/delta_writer_test.cpp @@ -31,6 +31,8 @@ #include "runtime/tuple.h" #include "runtime/descriptor_helper.h" #include "runtime/exec_env.h" +#include "runtime/mem_pool.h" +#include "runtime/mem_tracker.h" #include "util/logging.h" #include "olap/options.h" #include "olap/tablet_meta_manager.h" @@ -345,10 +347,11 @@ TEST_F(TestDeltaWriter, write) { DeltaWriter::open(&write_req, k_mem_tracker, &delta_writer); ASSERT_NE(delta_writer, nullptr); - Arena arena; + MemTracker tracker; + MemPool pool(&tracker); // Tuple 1 { - Tuple* tuple = reinterpret_cast(arena.Allocate(tuple_desc->byte_size())); + Tuple* tuple = reinterpret_cast(pool.allocate(tuple_desc->byte_size())); memset(tuple, 0, tuple_desc->byte_size()); *(int8_t*)(tuple->get_slot(slots[0]->tuple_offset())) = -127; *(int16_t*)(tuple->get_slot(slots[1]->tuple_offset())) = -32767; @@ -362,12 +365,12 @@ TEST_F(TestDeltaWriter, write) { ((DateTimeValue*)(tuple->get_slot(slots[6]->tuple_offset())))->from_date_str("2636-08-16 19:39:43", 19); StringValue* char_ptr = (StringValue*)(tuple->get_slot(slots[7]->tuple_offset())); - char_ptr->ptr = arena.Allocate(4); + char_ptr->ptr = (char*)pool.allocate(4); memcpy(char_ptr->ptr, "abcd", 4); char_ptr->len = 4; StringValue* var_ptr = (StringValue*)(tuple->get_slot(slots[8]->tuple_offset())); - var_ptr->ptr = arena.Allocate(5); + var_ptr->ptr = (char*)pool.allocate(5); memcpy(var_ptr->ptr, "abcde", 5); var_ptr->len = 5; @@ -385,12 +388,12 @@ TEST_F(TestDeltaWriter, write) { ((DateTimeValue*)(tuple->get_slot(slots[16]->tuple_offset())))->from_date_str("2636-08-16 19:39:43", 19); char_ptr = (StringValue*)(tuple->get_slot(slots[17]->tuple_offset())); - char_ptr->ptr = arena.Allocate(4); + char_ptr->ptr = (char*)pool.allocate(4); memcpy(char_ptr->ptr, "abcd", 4); char_ptr->len = 4; var_ptr = (StringValue*)(tuple->get_slot(slots[18]->tuple_offset())); - var_ptr->ptr = arena.Allocate(5); + var_ptr->ptr = (char*)pool.allocate(5); memcpy(var_ptr->ptr, "abcde", 5); var_ptr->len = 5; diff --git a/be/test/olap/key_coder_test.cpp b/be/test/olap/key_coder_test.cpp index 9c6ddd6c34..8fb22c9d03 100644 --- a/be/test/olap/key_coder_test.cpp +++ b/be/test/olap/key_coder_test.cpp @@ -22,14 +22,19 @@ #include #include "util/debug_util.h" +#include "runtime/mem_pool.h" +#include "runtime/mem_tracker.h" namespace doris { class KeyCoderTest : public testing::Test { public: - KeyCoderTest() { } + KeyCoderTest() : _pool(&_tracker) { } virtual ~KeyCoderTest() { } +private: + MemTracker _tracker; + MemPool _pool; }; template @@ -97,7 +102,7 @@ void test_integer_encode() { } } -TEST(KeyCoderTest, test_int) { +TEST_F(KeyCoderTest, test_int) { test_integer_encode(); test_integer_encode(); test_integer_encode(); @@ -108,7 +113,7 @@ TEST(KeyCoderTest, test_int) { test_integer_encode(); } -TEST(KeyCoderTest, test_date) { +TEST_F(KeyCoderTest, test_date) { using CppType = uint24_t; auto key_coder = get_key_coder(OLAP_FIELD_TYPE_DATE); @@ -168,7 +173,7 @@ TEST(KeyCoderTest, test_date) { } } -TEST(KeyCoderTest, test_decimal) { +TEST_F(KeyCoderTest, test_decimal) { auto key_coder = get_key_coder(OLAP_FIELD_TYPE_DECIMAL); decimal12_t val1(1, 100000000); @@ -212,7 +217,7 @@ TEST(KeyCoderTest, test_decimal) { } } -TEST(KeyCoderTest, test_char) { +TEST_F(KeyCoderTest, test_char) { auto key_coder = get_key_coder(OLAP_FIELD_TYPE_CHAR); char buf[] = "1234567890"; @@ -223,9 +228,8 @@ TEST(KeyCoderTest, test_char) { key_coder->encode_ascending(&slice, 10, &key); Slice encoded_key(key); - Arena arena; Slice check_slice; - auto st = key_coder->decode_ascending(&encoded_key, 10, (uint8_t*)&check_slice, &arena); + auto st = key_coder->decode_ascending(&encoded_key, 10, (uint8_t*)&check_slice, &_pool); ASSERT_TRUE(st.ok()); ASSERT_STREQ("1234567890", check_slice.data); @@ -236,16 +240,15 @@ TEST(KeyCoderTest, test_char) { key_coder->encode_ascending(&slice, 5, &key); Slice encoded_key(key); - Arena arena; Slice check_slice; - auto st = key_coder->decode_ascending(&encoded_key, 5, (uint8_t*)&check_slice, &arena); + auto st = key_coder->decode_ascending(&encoded_key, 5, (uint8_t*)&check_slice, &_pool); ASSERT_TRUE(st.ok()); ASSERT_STREQ("12345", check_slice.data); } } -TEST(KeyCoderTest, test_varchar) { +TEST_F(KeyCoderTest, test_varchar) { auto key_coder = get_key_coder(OLAP_FIELD_TYPE_VARCHAR); char buf[] = "1234567890"; @@ -256,9 +259,8 @@ TEST(KeyCoderTest, test_varchar) { key_coder->encode_ascending(&slice, 15, &key); Slice encoded_key(key); - Arena arena; Slice check_slice; - auto st = key_coder->decode_ascending(&encoded_key, 15, (uint8_t*)&check_slice, &arena); + auto st = key_coder->decode_ascending(&encoded_key, 15, (uint8_t*)&check_slice, &_pool); ASSERT_TRUE(st.ok()); ASSERT_STREQ("1234567890", check_slice.data); @@ -269,9 +271,8 @@ TEST(KeyCoderTest, test_varchar) { key_coder->encode_ascending(&slice, 5, &key); Slice encoded_key(key); - Arena arena; Slice check_slice; - auto st = key_coder->decode_ascending(&encoded_key, 5, (uint8_t*)&check_slice, &arena); + auto st = key_coder->decode_ascending(&encoded_key, 5, (uint8_t*)&check_slice, &_pool); ASSERT_TRUE(st.ok()); ASSERT_STREQ("12345", check_slice.data); diff --git a/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp b/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp index 119640e0c7..fb6b2e55a0 100644 --- a/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp +++ b/be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp @@ -27,6 +27,8 @@ #include "olap/olap_common.h" #include "olap/types.h" #include "util/debug_util.h" +#include "runtime/mem_tracker.h" +#include "runtime/mem_pool.h" namespace doris { namespace segment_v2 { @@ -70,11 +72,12 @@ public: ASSERT_EQ(slices.size(), page_decoder.count()); //check values - Arena arena; + MemTracker tracker; + MemPool pool(&tracker); TypeInfo* type_info = get_type_info(OLAP_FIELD_TYPE_VARCHAR); size_t size = slices.size(); - Slice* values = reinterpret_cast(arena.Allocate(size * sizeof(Slice))); - ColumnBlock column_block(type_info, (uint8_t*)values, nullptr, size, &arena); + Slice* values = reinterpret_cast(pool.allocate(size * sizeof(Slice))); + ColumnBlock column_block(type_info, (uint8_t*)values, nullptr, size, &pool); ColumnBlockView block_view(&column_block); status = page_decoder.next_batch(&size, &block_view); @@ -161,10 +164,11 @@ public: ASSERT_TRUE(status.ok()); //check values - Arena arena; + MemTracker tracker; + MemPool pool(&tracker); TypeInfo* type_info = get_type_info(OLAP_FIELD_TYPE_VARCHAR); - Slice* values = reinterpret_cast(arena.Allocate(sizeof(Slice))); - ColumnBlock column_block(type_info, (uint8_t*)values, nullptr, 1, &arena); + Slice* values = reinterpret_cast(pool.allocate(sizeof(Slice))); + ColumnBlock column_block(type_info, (uint8_t*)values, nullptr, 1, &pool); ColumnBlockView block_view(&column_block); size_t num = 1; diff --git a/be/test/olap/rowset/segment_v2/binary_plain_page_test.cpp b/be/test/olap/rowset/segment_v2/binary_plain_page_test.cpp index 112bd0d6f0..b93964bd79 100644 --- a/be/test/olap/rowset/segment_v2/binary_plain_page_test.cpp +++ b/be/test/olap/rowset/segment_v2/binary_plain_page_test.cpp @@ -25,6 +25,8 @@ #include "olap/rowset/segment_v2/binary_plain_page.h" #include "olap/olap_common.h" #include "olap/types.h" +#include "runtime/mem_pool.h" +#include "runtime/mem_tracker.h" namespace doris { namespace segment_v2 { @@ -59,13 +61,13 @@ public: //test1 - size_t size = 3; - - Arena arena; - Slice* values = reinterpret_cast(arena.Allocate(size * sizeof(Slice))); - uint8_t* null_bitmap = reinterpret_cast(arena.Allocate(BitmapSize(size))); - ColumnBlock block(get_type_info(OLAP_FIELD_TYPE_VARCHAR), (uint8_t*)values, null_bitmap, size, &arena); + MemTracker tracker; + MemPool pool(&tracker); + size_t size = 3; + Slice* values = reinterpret_cast(pool.allocate(size * sizeof(Slice))); + uint8_t* null_bitmap = reinterpret_cast(pool.allocate(BitmapSize(size))); + ColumnBlock block(get_type_info(OLAP_FIELD_TYPE_VARCHAR), (uint8_t*)values, null_bitmap, size, &pool); ColumnBlockView column_block_view(&block); status = page_decoder.next_batch(&size, &column_block_view); @@ -77,8 +79,8 @@ public: ASSERT_EQ (",", value[1].to_string()); ASSERT_EQ ("Doris", value[2].to_string()); - Slice* values2 = reinterpret_cast(arena.Allocate(size * sizeof(Slice))); - ColumnBlock block2(get_type_info(OLAP_FIELD_TYPE_VARCHAR), (uint8_t*)values2, null_bitmap, 1, &arena); + Slice* values2 = reinterpret_cast(pool.allocate(size * sizeof(Slice))); + ColumnBlock block2(get_type_info(OLAP_FIELD_TYPE_VARCHAR), (uint8_t*)values2, null_bitmap, 1, &pool); ColumnBlockView column_block_view2(&block2); size_t fetch_num = 1; diff --git a/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp b/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp index 5ed3e138f7..aefeee6f70 100644 --- a/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp +++ b/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp @@ -22,7 +22,8 @@ #include "olap/rowset/segment_v2/page_builder.h" #include "olap/rowset/segment_v2/page_decoder.h" #include "olap/rowset/segment_v2/bitshuffle_page.h" -#include "util/arena.h" +#include "runtime/mem_pool.h" +#include "runtime/mem_tracker.h" #include "util/logging.h" using doris::segment_v2::PageBuilderOptions; @@ -35,9 +36,10 @@ public: template void copy_one(PageDecoderType* decoder, typename TypeTraits::CppType* ret) { - Arena arena; + MemTracker tracker; + MemPool pool(&tracker); uint8_t null_bitmap = 0; - ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &arena); + ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &pool); ColumnBlockView column_block_view(&block); size_t n = 1; @@ -64,11 +66,12 @@ public: ASSERT_TRUE(status.ok()); ASSERT_EQ(0, page_decoder.current_index()); - Arena arena; + MemTracker tracker; + MemPool pool(&tracker); - CppType* values = reinterpret_cast(arena.Allocate(size * sizeof(CppType))); - uint8_t* null_bitmap = reinterpret_cast(arena.Allocate(BitmapSize(size))); - ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &arena); + CppType* values = reinterpret_cast(pool.allocate(size * sizeof(CppType))); + uint8_t* null_bitmap = reinterpret_cast(pool.allocate(BitmapSize(size))); + ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &pool); ColumnBlockView column_block_view(&block); status = page_decoder.next_batch(&size, &column_block_view); diff --git a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp index f88c35b749..3c4fb63598 100644 --- a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp +++ b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp @@ -29,16 +29,20 @@ #include "olap/types.h" #include "olap/column_block.h" #include "util/file_utils.h" -#include "util/arena.h" +#include "runtime/mem_tracker.h" +#include "runtime/mem_pool.h" namespace doris { namespace segment_v2 { class ColumnReaderWriterTest : public testing::Test { public: - ColumnReaderWriterTest() { } + ColumnReaderWriterTest() : _pool(&_tracker) { } virtual ~ColumnReaderWriterTest() { } +private: + MemTracker _tracker; + MemPool _pool; }; template @@ -119,11 +123,12 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s st = iter->seek_to_first(); ASSERT_TRUE(st.ok()) << st.to_string(); - Arena arena; + MemTracker tracker; + MemPool pool(&tracker); Type vals[1024]; Type* vals_ = vals; uint8_t is_null[1024]; - ColumnBlock col(type_info, (uint8_t*)vals, is_null, 1024, &arena); + ColumnBlock col(type_info, (uint8_t*)vals, is_null, 1024, &pool); int idx = 0; while (true) { @@ -152,10 +157,11 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s } // random read { - Arena arena; + MemTracker tracker; + MemPool pool(&tracker); Type vals[1024]; uint8_t is_null[1024]; - ColumnBlock col(type_info, (uint8_t*)vals, is_null, 1024, &arena); + ColumnBlock col(type_info, (uint8_t*)vals, is_null, 1024, &pool); for (int rowid = 0; rowid < num_rows; rowid += 4025) { st = iter->seek_to_ordinal(rowid); @@ -224,8 +230,6 @@ TEST_F(ColumnReaderWriterTest, test_nullable) { } TEST_F(ColumnReaderWriterTest, test_types) { - Arena arena; - size_t num_uint8_rows = 1024 * 1024; uint8_t* is_null = new uint8_t[num_uint8_rows]; @@ -241,8 +245,8 @@ TEST_F(ColumnReaderWriterTest, test_types) { datetime_vals[i] = i + 33; decimal_vals[i] = decimal12_t(i, i); // 1.000000001 - set_column_value_by_type(OLAP_FIELD_TYPE_VARCHAR, i, (char*)&varchar_vals[i], &arena); - set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, i, (char*)&char_vals[i], &arena, 8); + set_column_value_by_type(OLAP_FIELD_TYPE_VARCHAR, i, (char*)&varchar_vals[i], &_pool); + set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, i, (char*)&char_vals[i], &_pool, 8); BitmapChange(is_null, i, (i % 4) == 0); } diff --git a/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp b/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp index 641353682f..6806408513 100644 --- a/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp +++ b/be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp @@ -22,7 +22,8 @@ #include "olap/rowset/segment_v2/page_builder.h" #include "olap/rowset/segment_v2/page_decoder.h" #include "olap/rowset/segment_v2/frame_of_reference_page.h" -#include "util/arena.h" +#include "runtime/mem_tracker.h" +#include "runtime/mem_pool.h" #include "util/logging.h" using doris::segment_v2::PageBuilderOptions; @@ -33,9 +34,10 @@ class FrameOfReferencePageTest : public testing::Test { public: template void copy_one(PageDecoderType* decoder, typename TypeTraits::CppType* ret) { - Arena arena; + MemTracker tracker; + MemPool pool(&tracker); uint8_t null_bitmap = 0; - ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &arena); + ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &pool); ColumnBlockView column_block_view(&block); size_t n = 1; @@ -63,10 +65,11 @@ public: ASSERT_EQ(0, for_page_decoder.current_index()); ASSERT_EQ(size, for_page_decoder.count()); - Arena arena; - CppType* values = reinterpret_cast(arena.Allocate(size * sizeof(CppType))); - uint8_t* null_bitmap = reinterpret_cast(arena.Allocate(BitmapSize(size))); - ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &arena); + MemTracker tracker; + MemPool pool(&tracker); + CppType* values = reinterpret_cast(pool.allocate(size * sizeof(CppType))); + uint8_t* null_bitmap = reinterpret_cast(pool.allocate(BitmapSize(size))); + ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &pool); ColumnBlockView column_block_view(&block); size_t size_to_fetch = size; status = for_page_decoder.next_batch(&size_to_fetch, &column_block_view); diff --git a/be/test/olap/rowset/segment_v2/plain_page_test.cpp b/be/test/olap/rowset/segment_v2/plain_page_test.cpp index 635a5d4e6a..ea276a3816 100644 --- a/be/test/olap/rowset/segment_v2/plain_page_test.cpp +++ b/be/test/olap/rowset/segment_v2/plain_page_test.cpp @@ -25,7 +25,8 @@ #include "common/logging.h" #include "olap/olap_common.h" #include "olap/types.h" -#include "util/arena.h" +#include "runtime/mem_pool.h" +#include "runtime/mem_tracker.h" namespace doris { namespace segment_v2 { @@ -45,9 +46,10 @@ public: template void copy_one(PageDecoderType* decoder, typename TypeTraits::CppType* ret) { - Arena arena; + MemTracker tracker; + MemPool pool(&tracker); uint8_t null_bitmap = 0; - ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &arena); + ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &pool); ColumnBlockView column_block_view(&block); size_t n = 1; @@ -74,11 +76,12 @@ public: ASSERT_EQ(0, page_decoder.current_index()); - Arena arena; + MemTracker tracker; + MemPool pool(&tracker); - CppType* values = reinterpret_cast(arena.Allocate(size * sizeof(CppType))); - uint8_t* null_bitmap = reinterpret_cast(arena.Allocate(BitmapSize(size))); - ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &arena); + CppType* values = reinterpret_cast(pool.allocate(size * sizeof(CppType))); + uint8_t* null_bitmap = reinterpret_cast(pool.allocate(BitmapSize(size))); + ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &pool); ColumnBlockView column_block_view(&block); status = page_decoder.next_batch(&size, &column_block_view); ASSERT_TRUE(status.ok()); diff --git a/be/test/olap/rowset/segment_v2/rle_page_test.cpp b/be/test/olap/rowset/segment_v2/rle_page_test.cpp index d355cd5e15..efb6282f7b 100644 --- a/be/test/olap/rowset/segment_v2/rle_page_test.cpp +++ b/be/test/olap/rowset/segment_v2/rle_page_test.cpp @@ -22,8 +22,9 @@ #include "olap/rowset/segment_v2/page_builder.h" #include "olap/rowset/segment_v2/page_decoder.h" #include "olap/rowset/segment_v2/rle_page.h" -#include "util/arena.h" #include "util/logging.h" +#include "runtime/mem_tracker.h" +#include "runtime/mem_pool.h" using doris::segment_v2::PageBuilderOptions; using doris::segment_v2::PageDecoderOptions; @@ -36,9 +37,10 @@ public: template void copy_one(PageDecoderType* decoder, typename TypeTraits::CppType* ret) { - Arena arena; + MemTracker tracker; + MemPool pool(&tracker); uint8_t null_bitmap = 0; - ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &arena); + ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &pool); ColumnBlockView column_block_view(&block); size_t n = 1; @@ -66,11 +68,11 @@ public: ASSERT_EQ(0, rle_page_decoder.current_index()); ASSERT_EQ(size, rle_page_decoder.count()); - Arena arena; - - CppType* values = reinterpret_cast(arena.Allocate(size * sizeof(CppType))); - uint8_t* null_bitmap = reinterpret_cast(arena.Allocate(BitmapSize(size))); - ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &arena); + MemTracker tracker; + MemPool pool(&tracker); + CppType* values = reinterpret_cast(pool.allocate(size * sizeof(CppType))); + uint8_t* null_bitmap = reinterpret_cast(pool.allocate(BitmapSize(size))); + ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &pool); ColumnBlockView column_block_view(&block); size_t size_to_fetch = size; status = rle_page_decoder.next_batch(&size_to_fetch, &column_block_view); diff --git a/be/test/olap/rowset/segment_v2/segment_test.cpp b/be/test/olap/rowset/segment_v2/segment_test.cpp index 698e3be7d2..d798137ea2 100644 --- a/be/test/olap/rowset/segment_v2/segment_test.cpp +++ b/be/test/olap/rowset/segment_v2/segment_test.cpp @@ -32,6 +32,8 @@ #include "olap/types.h" #include "olap/tablet_schema_helper.h" #include "util/file_utils.h" +#include "runtime/mem_pool.h" +#include "runtime/mem_tracker.h" namespace doris { namespace segment_v2 { @@ -640,7 +642,8 @@ TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) { TEST_F(SegmentReaderWriterTest, TestStringDict) { size_t num_rows_per_block = 10; - Arena _arena; + MemTracker tracker; + MemPool pool(&tracker); std::shared_ptr tablet_schema(new TabletSchema()); tablet_schema->_num_columns = 4; @@ -677,7 +680,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { for (int j = 0; j < 4; ++j) { auto cell = row.cell(j); cell.set_not_null(); - set_column_value_by_type(tablet_schema->_cols[j]._type, i * 10 + j, (char*)cell.mutable_cell_ptr(), &_arena, tablet_schema->_cols[j]._length); + set_column_value_by_type(tablet_schema->_cols[j]._type, i * 10 + j, (char*)cell.mutable_cell_ptr(), &pool, tablet_schema->_cols[j]._length); } Status status = writer.append_row(row); ASSERT_TRUE(status.ok()); @@ -722,7 +725,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { const Slice* actual = reinterpret_cast(column_block.cell_ptr(i)); Slice expect; - set_column_value_by_type(tablet_schema->_cols[j]._type, rid * 10 + cid, reinterpret_cast(&expect), &_arena, tablet_schema->_cols[j]._length); + set_column_value_by_type(tablet_schema->_cols[j]._type, rid * 10 + cid, reinterpret_cast(&expect), &pool, tablet_schema->_cols[j]._length); ASSERT_EQ(expect.to_string(), actual->to_string()); } } @@ -738,7 +741,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { { auto cell = lower_bound->cell(0); cell.set_not_null(); - set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, 40970, (char*)cell.mutable_cell_ptr(), &_arena, tablet_schema->_cols[0]._length); + set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, 40970, (char*)cell.mutable_cell_ptr(), &pool, tablet_schema->_cols[0]._length); } StorageReadOptions read_opts; @@ -760,7 +763,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { { auto cell = lower_bound->cell(0); cell.set_not_null(); - set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, -2, (char*)cell.mutable_cell_ptr(), &_arena, tablet_schema->_cols[0]._length); + set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, -2, (char*)cell.mutable_cell_ptr(), &pool, tablet_schema->_cols[0]._length); } std::unique_ptr upper_bound(new RowCursor()); @@ -768,7 +771,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { { auto cell = upper_bound->cell(0); cell.set_not_null(); - set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, -1, (char*)cell.mutable_cell_ptr(), &_arena, tablet_schema->_cols[0]._length); + set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, -1, (char*)cell.mutable_cell_ptr(), &pool, tablet_schema->_cols[0]._length); } StorageReadOptions read_opts; @@ -820,7 +823,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) { const Slice* actual = reinterpret_cast(column_block.cell_ptr(i)); Slice expect; - set_column_value_by_type(tablet_schema->_cols[j]._type, rid * 10 + cid, reinterpret_cast(&expect), &_arena, tablet_schema->_cols[j]._length); + set_column_value_by_type(tablet_schema->_cols[j]._type, rid * 10 + cid, reinterpret_cast(&expect), &pool, tablet_schema->_cols[j]._length); ASSERT_EQ(expect.to_string(), actual->to_string()) << "rid:" << rid << ", i:" << i;; } } diff --git a/be/test/olap/tablet_schema_helper.h b/be/test/olap/tablet_schema_helper.h index 7e7082f7d2..d0a03b7cfa 100644 --- a/be/test/olap/tablet_schema_helper.h +++ b/be/test/olap/tablet_schema_helper.h @@ -20,6 +20,8 @@ #include #include "olap/tablet_schema.h" +#include "runtime/mem_pool.h" + namespace doris { TabletColumn create_int_key(int32_t id, bool is_nullable = true) { @@ -79,14 +81,14 @@ TabletColumn create_varchar_key(int32_t id, bool is_nullable = true) { return column; } -void set_column_value_by_type(FieldType fieldType, int src, char* target, Arena* _arena, size_t _length = 0) { +void set_column_value_by_type(FieldType fieldType, int src, char* target, MemPool* pool, size_t _length = 0) { if (fieldType == OLAP_FIELD_TYPE_CHAR) { char* src_value = &std::to_string(src)[0]; int src_len = strlen(src_value); auto* dest_slice = (Slice*)target; dest_slice->size = _length; - dest_slice->data = _arena->Allocate(dest_slice->size); + dest_slice->data = (char*)pool->allocate(dest_slice->size); memcpy(dest_slice->data, src_value, src_len); memset(dest_slice->data + src_len, 0, dest_slice->size - src_len); } else if (fieldType == OLAP_FIELD_TYPE_VARCHAR) { @@ -95,7 +97,7 @@ void set_column_value_by_type(FieldType fieldType, int src, char* target, Arena* auto* dest_slice = (Slice*)target; dest_slice->size = src_len; - dest_slice->data = _arena->Allocate(src_len); + dest_slice->data = (char*)pool->allocate(src_len); std::memcpy(dest_slice->data, src_value, src_len); } else { *(int*)target = src; diff --git a/be/test/util/CMakeLists.txt b/be/test/util/CMakeLists.txt index c9cfb5eb0b..26cd532796 100644 --- a/be/test/util/CMakeLists.txt +++ b/be/test/util/CMakeLists.txt @@ -37,7 +37,6 @@ ADD_BE_TEST(types_test) ADD_BE_TEST(json_util_test) ADD_BE_TEST(byte_buffer_test2) ADD_BE_TEST(uid_util_test) -ADD_BE_TEST(arena_test) ADD_BE_TEST(aes_util_test) ADD_BE_TEST(md5_test) ADD_BE_TEST(bitmap_test) diff --git a/be/test/util/arena_test.cpp b/be/test/util/arena_test.cpp deleted file mode 100644 index 7194fb2fab..0000000000 --- a/be/test/util/arena_test.cpp +++ /dev/null @@ -1,81 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include "util/arena.h" -#include "util/random.h" - -namespace doris { - -class ArenaTest : public testing::Test { }; - -TEST_F(ArenaTest, Empty) { - Arena arena; -} - -TEST_F(ArenaTest, Simple) { - std::vector > allocated; - Arena arena; - const int N = 100000; - size_t bytes = 0; - Random rnd(301); - for (int i = 0; i < N; i++) { - size_t s; - if (i % (N / 10) == 0) { - s = i; - } else { - s = rnd.OneIn(4000) ? rnd.Uniform(6000) : - (rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20)); - } - if (s == 0) { - // Our arena disallows size 0 allocations. - s = 1; - } - char* r; - if (rnd.OneIn(10)) { - r = arena.AllocateAligned(s); - } else { - r = arena.Allocate(s); - } - - for (size_t b = 0; b < s; b++) { - // Fill the "i"th allocation with a known bit pattern - r[b] = i % 256; - } - bytes += s; - allocated.push_back(std::make_pair(s, r)); - ASSERT_GE(arena.MemoryUsage(), bytes); - if (i > N / 10) { - ASSERT_LE(arena.MemoryUsage(), bytes * 1.10); - } - } - for (size_t i = 0; i < allocated.size(); i++) { - size_t num_bytes = allocated[i].first; - const char* p = allocated[i].second; - for (size_t b = 0; b < num_bytes; b++) { - // Check the "i"th allocation for the known bit pattern - ASSERT_EQ(int(p[b]) & 0xff, i % 256); - } - } -} - -} // namespace doris - -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/run-ut.sh b/run-ut.sh index 80b0bf49fd..c8f77ca467 100755 --- a/run-ut.sh +++ b/run-ut.sh @@ -145,7 +145,6 @@ ${DORIS_TEST_BINARY_DIR}/util/new_metrics_test ${DORIS_TEST_BINARY_DIR}/util/doris_metrics_test ${DORIS_TEST_BINARY_DIR}/util/system_metrics_test ${DORIS_TEST_BINARY_DIR}/util/core_local_test -${DORIS_TEST_BINARY_DIR}/util/arena_test ${DORIS_TEST_BINARY_DIR}/util/types_test ${DORIS_TEST_BINARY_DIR}/util/json_util_test ${DORIS_TEST_BINARY_DIR}/util/byte_buffer_test2