database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Replace Arena with MemPool (#2012)

Browse Source 
After replacing Arena with MemPool, we can achieve one copy for string
value read from segment v2. We can exchange MemPool's chunk between
RowBlockV2 and RowBlock. This change only replace Arena, this work will
be done in other change list.
This commit is contained in:
ZHAO Chun
2019-10-19 15:53:24 +08:00
committed by Mingyu Chen
parent 292273be2e
commit 05643dc403
36 changed files with 190 additions and 437 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
LICENSE.txt
View File

@ -414,22 +414,6 @@ be/src/olap/skiplist.h : BSD-style license
--------------------------------------------------------------------------------
be/src/util/arena.h : BSD-style license
Copyright (c) 2011 The LevelDB Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file. See the AUTHORS file for names of contributors.
--------------------------------------------------------------------------------
be/src/util/arena.cpp : BSD-style license
Copyright (c) 2011 The LevelDB Authors. All rights reserved.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file. See the AUTHORS file for names of contributors.
--------------------------------------------------------------------------------
be/src/olap/new_status.h : BSD-style license
Copyright (c) 2011 The LevelDB Authors. All rights reserved.

4
be/src/olap/aggregate_func.h
View File

@ -42,7 +42,7 @@ public:
// Init function will initialize aggregation execute environment in dst with source
// and convert the source raw data to storage aggregate format
//
// Memory Note: For plain memory can be allocated from arena, whose lifetime
// Memory Note: For plain memory can be allocated from *mem_pool, whose lifetime
// will last util finalize function is called. Memory allocated from heap should
// be freed in finalize functioin to avoid memory leak.
inline void init(RowCursorCell* dst, const char* src, bool src_null, MemPool* mem_pool, ObjectPool* agg_pool) const {
@ -66,7 +66,7 @@ public:
// saved in src.
//
// Memory Note: All heap memory allocated in init and update function should be freed
// before this function return. Memory allocated from arena will be still available
// before this function return. Memory allocated from *mem_pool will be still available
// and will be freed by client.
inline void finalize(RowCursorCell* src, MemPool* mem_pool) const {
_finalize_fn(src, mem_pool);

13
be/src/olap/column_block.h
View File

@ -24,23 +24,22 @@
namespace doris {
class Arena;
class MemPool;
class TypeInfo;
class ColumnBlockCell;
// Block of data belong to a single column.
// It doesn't own any data, user should keep the life of input data.
class ColumnBlock {
public:
ColumnBlock(const TypeInfo* type_info, uint8_t* data, uint8_t* null_bitmap, size_t nrows, Arena* arena)
: _type_info(type_info), _data(data), _null_bitmap(null_bitmap), _nrows(nrows), _arena(arena) { }
ColumnBlock(const TypeInfo* type_info, uint8_t* data, uint8_t* null_bitmap, size_t nrows, MemPool* pool)
: _type_info(type_info), _data(data), _null_bitmap(null_bitmap), _nrows(nrows), _pool(pool) { }
const TypeInfo* type_info() const { return _type_info; }
uint8_t* data() const { return _data; }
uint8_t* null_bitmap() const { return _null_bitmap; }
bool is_nullable() const { return _null_bitmap != nullptr; }
Arena* arena() const { return _arena; }
MemPool* pool() const { return _pool; }
const uint8_t* cell_ptr(size_t idx) const { return _data + idx * _type_info->size(); }
uint8_t* mutable_cell_ptr(size_t idx) const { return _data + idx * _type_info->size(); }
bool is_null(size_t idx) const {
@ -61,7 +60,7 @@ private:
uint8_t* _data;
uint8_t* _null_bitmap;
size_t _nrows;
Arena* _arena;
MemPool* _pool;
};
struct ColumnBlockCell {
@ -90,7 +89,7 @@ public:
void advance(size_t skip) { _row_offset += skip; }
size_t first_row_index() const { return _row_offset; }
ColumnBlock* column_block() { return _block; }
Arena* arena() const { return _block->arena(); }
MemPool* pool() const { return _block->pool(); }
void set_null_bits(size_t num_rows, bool val) {
BitmapChangeBits(_block->null_bitmap(), _row_offset, num_rows, val);
}

40
be/src/olap/field.h
View File

@ -57,7 +57,10 @@ public:
virtual inline void set_to_max(char* buf) const { return _type_info->set_to_max(buf); }
inline void set_to_min(char* buf) const { return _type_info->set_to_min(buf); }
virtual inline char* allocate_value_from_arena(Arena* arena) const { return arena->Allocate(_type_info->size()); }
// This function allocate memory from pool, other than allocate_memory
// reserve memory from continuous memory.
virtual inline char* allocate_value(MemPool* pool) const { return (char*)pool->allocate(_type_info->size()); }
inline void agg_update(RowCursorCell* dest, const RowCursorCell& src, MemPool* mem_pool = nullptr) const {
_agg_info->update(dest, src, mem_pool);
@ -172,18 +175,6 @@ public:
_type_info->deep_copy(dst->mutable_cell_ptr(), src.cell_ptr(), pool);
}
template<typename DstCellType, typename SrcCellType>
void deep_copy(DstCellType* dst,
const SrcCellType& src,
Arena* arena) const {
bool is_null = src.is_null();
dst->set_is_null(is_null);
if (is_null) {
return;
}
_type_info->deep_copy_with_arena(dst->mutable_cell_ptr(), src.cell_ptr(), arena);
}
// deep copy filed content from `src` to `dst` without null-byte
inline void deep_copy_content(char* dst, const char* src, MemPool* mem_pool) const {
_type_info->deep_copy(dst, src, mem_pool);
@ -195,11 +186,6 @@ public:
_type_info->shallow_copy(dst, src);
}
// copy filed content from src to dest without nullbyte
inline void deep_copy_content(char* dest, const char* src, Arena* arena) const {
_type_info->deep_copy_with_arena(dest, src, arena);
}
// Copy srouce content to destination in index format.
template<typename DstCellType, typename SrcCellType>
void to_index(DstCellType* dst, const SrcCellType& src) const;
@ -240,8 +226,8 @@ public:
_key_coder->encode_ascending(value, _index_size, buf);
}
Status decode_ascending(Slice* encoded_key, uint8_t* cell_ptr, Arena* arena) const {
return _key_coder->decode_ascending(encoded_key, _index_size, cell_ptr, arena);
Status decode_ascending(Slice* encoded_key, uint8_t* cell_ptr, MemPool* pool) const {
return _key_coder->decode_ascending(encoded_key, _index_size, cell_ptr, pool);
}
private:
// Field的最大长度，单位为字节，通常等于length， 变长字符串不同
@ -256,11 +242,11 @@ protected:
// 除字符串外，其它类型都是确定的
uint32_t _length;
char* allocate_string_value_from_arena(Arena* arena) const {
char* type_value = arena->Allocate(sizeof(Slice));
char* allocate_string_value(MemPool* pool) const {
char* type_value = (char*)pool->allocate(sizeof(Slice));
auto slice = reinterpret_cast<Slice*>(type_value);
slice->size = _length;
slice->data = arena->Allocate(slice->size);
slice->data = (char*)pool->allocate(slice->size);
return type_value;
};
};
@ -383,8 +369,8 @@ public:
return new CharField(*this);
}
char* allocate_value_from_arena(Arena* arena) const override {
return Field::allocate_string_value_from_arena(arena);
char* allocate_value(MemPool* pool) const override {
return Field::allocate_string_value(pool);
}
void set_to_max(char* ch) const override {
@ -416,8 +402,8 @@ public:
return new VarcharField(*this);
}
char* allocate_value_from_arena(Arena* arena) const override {
return Field::allocate_string_value_from_arena(arena);
char* allocate_value(MemPool* pool) const override {
return Field::allocate_string_value(pool);
}
void set_to_max(char* ch) const override {

3
be/src/olap/generic_iterators.cpp
View File

@ -22,7 +22,6 @@
#include "olap/row_block2.h"
#include "olap/row_cursor_cell.h"
#include "olap/row.h"
#include "util/arena.h"
namespace doris {
@ -261,7 +260,7 @@ Status MergeIterator::next_batch(RowBlockV2* block) {
RowBlockRow dst_row = block->row(row_idx);
// copy current row to block
copy_row(&dst_row, ctx->current_row(), block->arena());
copy_row(&dst_row, ctx->current_row(), block->pool());
RETURN_IF_ERROR(ctx->advance());
if (ctx->valid()) {

26
be/src/olap/key_coder.h
View File

@ -24,14 +24,14 @@
#include "gutil/endian.h"
#include "gutil/strings/substitute.h"
#include "olap/types.h"
#include "util/arena.h"
#include "runtime/mem_pool.h"
namespace doris {
using strings::Substitute;
using EncodeAscendingFunc = void (*)(const void* value, size_t index_size, std::string* buf);
using DecodeAscendingFunc = Status (*)(Slice* encoded_key, size_t index_size, uint8_t* cell_ptr, Arena* arena);
using DecodeAscendingFunc = Status (*)(Slice* encoded_key, size_t index_size, uint8_t* cell_ptr, MemPool* pool);
// Helper class that is used to encode types of value in memory format
// into a sorted binary. For example, this class will encode unsigned
@ -44,8 +44,8 @@ public:
void encode_ascending(const void* value, size_t index_size, std::string* buf) const {
_encode_ascending(value, index_size, buf);
}
Status decode_ascending(Slice* encoded_key, size_t index_size, uint8_t* cell_ptr, Arena* arena) const {
return _decode_ascending(encoded_key, index_size, cell_ptr, arena);
Status decode_ascending(Slice* encoded_key, size_t index_size, uint8_t* cell_ptr, MemPool* pool) const {
return _decode_ascending(encoded_key, index_size, cell_ptr, pool);
}
private:
@ -97,7 +97,7 @@ public:
}
static Status decode_ascending(Slice* encoded_key, size_t index_size,
uint8_t* cell_ptr, Arena* arena) {
uint8_t* cell_ptr, MemPool* pool) {
if (encoded_key->size < sizeof(UnsignedCppType)) {
return Status::InvalidArgument(
Substitute("Key too short, need=$0 vs real=$1",
@ -131,7 +131,7 @@ public:
}
static Status decode_ascending(Slice* encoded_key, size_t index_size,
uint8_t* cell_ptr, Arena* arena) {
uint8_t* cell_ptr, MemPool* pool) {
if (encoded_key->size < sizeof(UnsignedCppType)) {
return Status::InvalidArgument(
Substitute("Key too short, need=$0 vs real=$1",
@ -161,12 +161,12 @@ public:
}
static Status decode_ascending(Slice* encoded_key, size_t index_size,
uint8_t* cell_ptr, Arena* arena) {
uint8_t* cell_ptr, MemPool* pool) {
decimal12_t decimal_val;
RETURN_IF_ERROR(KeyCoderTraits<OLAP_FIELD_TYPE_BIGINT>::decode_ascending(
encoded_key, sizeof(decimal_val.integer), (uint8_t*)&decimal_val.integer, arena));
encoded_key, sizeof(decimal_val.integer), (uint8_t*)&decimal_val.integer, pool));
RETURN_IF_ERROR(KeyCoderTraits<OLAP_FIELD_TYPE_INT>::decode_ascending(
encoded_key, sizeof(decimal_val.fraction), (uint8_t*)&decimal_val.fraction, arena));
encoded_key, sizeof(decimal_val.fraction), (uint8_t*)&decimal_val.fraction, pool));
memcpy(cell_ptr, &decimal_val, sizeof(decimal12_t));
return Status::OK();
}
@ -182,14 +182,14 @@ public:
}
static Status decode_ascending(Slice* encoded_key, size_t index_size,
uint8_t* cell_ptr, Arena* arena) {
uint8_t* cell_ptr, MemPool* pool) {
if (encoded_key->size < index_size) {
return Status::InvalidArgument(
Substitute("Key too short, need=$0 vs real=$1",
index_size, encoded_key->size));
}
Slice* slice = (Slice*)cell_ptr;
slice->data = arena->Allocate(index_size);
slice->data = (char*)pool->allocate(index_size);
slice->size = index_size;
memcpy(slice->data, encoded_key->data, index_size);
encoded_key->remove_prefix(index_size);
@ -207,13 +207,13 @@ public:
}
static Status decode_ascending(Slice* encoded_key, size_t index_size,
uint8_t* cell_ptr, Arena* arena) {
uint8_t* cell_ptr, MemPool* pool) {
CHECK(encoded_key->size <= index_size)
<< "encoded_key size is larger than index_size, key_size=" << encoded_key->size
<< ", index_size=" << index_size;
auto copy_size = encoded_key->size;
Slice* slice = (Slice*)cell_ptr;
slice->data = arena->Allocate(copy_size);
slice->data = (char*)pool->allocate(copy_size);
slice->size = copy_size;
memcpy(slice->data, encoded_key->data, copy_size);
encoded_key->remove_prefix(copy_size);

10
be/src/olap/row.h
View File

@ -134,16 +134,6 @@ void copy_row(DstRowType* dst, const SrcRowType& src, MemPool* pool) {
}
}
// Deep copy src row to dst row. Schema of src and dst row must be same.
template<typename DstRowType, typename SrcRowType>
void copy_row(DstRowType* dst, const SrcRowType& src, Arena* arena) {
for (auto cid : dst->schema()->column_ids()) {
auto dst_cell = dst->cell(cid);
auto src_cell = src.cell(cid);
dst->schema()->column(cid)->deep_copy(&dst_cell, src_cell, arena);
}
}
template<typename DstRowType, typename SrcRowType>
void agg_update_row(DstRowType* dst, const SrcRowType& src, MemPool* mem_pool) {
for (uint32_t cid = dst->schema()->num_key_columns(); cid < dst->schema()->num_columns(); ++cid) {

1
be/src/olap/row_block2.cpp
View File

@ -31,6 +31,7 @@ RowBlockV2::RowBlockV2(const Schema& schema, uint16_t capacity)
_capacity(capacity),
_column_datas(_schema.num_columns(), nullptr),
_column_null_bitmaps(_schema.num_columns(), nullptr),
_pool(new MemPool(&_tracker)),
_selection_vector(nullptr) {
auto bitmap_size = BitmapSize(capacity);
for (auto cid : _schema.column_ids()) {

17
be/src/olap/row_block2.h
View File

@ -26,10 +26,12 @@
#include "olap/schema.h"
#include "olap/types.h"
#include "olap/selection_vector.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
namespace doris {
class Arena;
class MemPool;
class RowCursor;
// This struct contains a block of rows, in which each column's data is stored
@ -47,14 +49,14 @@ public:
// return the maximum number of rows that can be contained in this block.
// invariant: 0 <= num_rows() <= capacity()
size_t capacity() const { return _capacity; }
Arena* arena() const { return _arena.get(); }
MemPool* pool() const { return _pool.get(); }
// reset the state of the block so that it can be reused for write.
// all previously returned ColumnBlocks are invalidated after clear(), accessing them
// will result in undefined behavior.
void clear() {
_num_rows = 0;
_arena.reset(new Arena);
_pool->clear();
_selected_size = _capacity;
for (int i = 0; i < _selected_size; ++i) {
_selection_vector[i] = i;
@ -77,7 +79,7 @@ public:
const TypeInfo* type_info = _schema.column(cid)->type_info();
uint8_t* data = _column_datas[cid];
uint8_t* null_bitmap = _column_null_bitmaps[cid];
return ColumnBlock(type_info, data, null_bitmap, _capacity, _arena.get());
return ColumnBlock(type_info, data, null_bitmap, _capacity, _pool.get());
}
RowBlockRow row(size_t row_idx) const;
@ -101,15 +103,16 @@ private:
size_t _capacity;
// keeps fixed-size (field_size x capacity) data vector for each column,
// _column_datas[cid] == null if cid is not in `_schema`.
// memory are not allocated from `_arena` because we don't wan't to reallocate them in clear()
// memory are not allocated from `_pool` because we don't wan't to reallocate them in clear()
std::vector<uint8_t*> _column_datas;
// keeps null bitmap for each column,
// _column_null_bitmaps[cid] == null if cid is not in `_schema` or the column is not null.
// memory are not allocated from `_arena` because we don't wan't to reallocate them in clear()
// memory are not allocated from `_pool` because we don't wan't to reallocate them in clear()
std::vector<uint8_t*> _column_null_bitmaps;
size_t _num_rows;
// manages the memory for slice's data
std::unique_ptr<Arena> _arena;
MemTracker _tracker;
std::unique_ptr<MemPool> _pool;
// index of selected rows for rows passed the predicate
uint16_t* _selection_vector;

11
be/src/olap/rowset/segment_v2/binary_dict_page.cpp
View File

@ -34,7 +34,8 @@ BinaryDictPageBuilder::BinaryDictPageBuilder(const PageBuilderOptions& options)
_finished(false),
_data_page_builder(nullptr),
_dict_builder(nullptr),
_encoding_type(DICT_ENCODING) {
_encoding_type(DICT_ENCODING),
_pool(&_tracker) {
// initially use DICT_ENCODING
// TODO: the data page builder type can be created by Factory according to user config
_data_page_builder.reset(new BitshufflePageBuilder<OLAP_FIELD_TYPE_INT>(options));
@ -71,7 +72,7 @@ Status BinaryDictPageBuilder::add(const uint8_t* vals, size_t* count) {
}
Slice dict_item(src->data, src->size);
if (src->size > 0) {
char* item_mem = _arena.Allocate(src->size);
char* item_mem = (char*)_pool.allocate(src->size);
if (item_mem == nullptr) {
return Status::MemoryAllocFailed(Substitute("memory allocate failed, size:$0", src->size));
}
@ -127,7 +128,7 @@ size_t BinaryDictPageBuilder::count() const {
}
uint64_t BinaryDictPageBuilder::size() const {
return _arena.MemoryUsage() + _data_page_builder->size();
return _pool.total_allocated_bytes() + _data_page_builder->size();
}
Status BinaryDictPageBuilder::get_dictionary_page(Slice* dictionary_page) {
@ -206,7 +207,7 @@ Status BinaryDictPageDecoder::next_batch(size_t* n, ColumnBlockView* dst) {
// And then copy the strings corresponding to the codewords to the destination buffer
TypeInfo *type_info = get_type_info(OLAP_FIELD_TYPE_INT);
// the data in page is not null
ColumnBlock column_block(type_info, _code_buf.data(), nullptr, *n, dst->column_block()->arena());
ColumnBlock column_block(type_info, _code_buf.data(), nullptr, *n, dst->column_block()->pool());
ColumnBlockView tmp_block_view(&column_block);
RETURN_IF_ERROR(_data_page_decoder->next_batch(n, &tmp_block_view));
for (int i = 0; i < *n; ++i) {
@ -214,7 +215,7 @@ Status BinaryDictPageDecoder::next_batch(size_t* n, ColumnBlockView* dst) {
// get the string from the dict decoder
Slice element = _dict_decoder->string_at_index(codeword);
if (element.size > 0) {
char* destination = dst->column_block()->arena()->Allocate(element.size);
char* destination = (char*)dst->column_block()->pool()->allocate(element.size);
if (destination == nullptr) {
return Status::MemoryAllocFailed(Substitute(
"memory allocate failed, size:$0", element.size));

7
be/src/olap/rowset/segment_v2/binary_dict_page.h
View File

@ -28,7 +28,8 @@
#include "olap/rowset/segment_v2/options.h"
#include "olap/rowset/segment_v2/common.h"
#include "olap/column_block.h"
#include "util/arena.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
#include "gen_cpp/segment_v2.pb.h"
#include "gutil/hash/string_hash.h"
@ -94,7 +95,9 @@ private:
std::unordered_map<Slice, uint32_t, HashOfSlice> _dictionary;
// used to remember the insertion order of dict keys
std::vector<Slice> _dict_items;
Arena _arena;
// TODO(zc): rethink about this mem pool
MemTracker _tracker;
MemPool _pool;
faststring _buffer;
};

4
be/src/olap/rowset/segment_v2/binary_plain_page.h
View File

@ -28,7 +28,6 @@
#pragma once
#include "util/arena.h"
#include "util/coding.h"
#include "util/faststring.h"
#include "olap/olap_common.h"
@ -36,6 +35,7 @@
#include "olap/rowset/segment_v2/page_decoder.h"
#include "olap/rowset/segment_v2/options.h"
#include "olap/types.h"
#include "runtime/mem_pool.h"
namespace doris {
namespace segment_v2 {
@ -183,7 +183,7 @@ public:
Slice elem(string_at_index(_cur_idx));
out->size = elem.size;
if (elem.size != 0) {
out->data = reinterpret_cast<char*>(dst->arena()->Allocate(elem.size * sizeof(uint8_t)));
out->data = reinterpret_cast<char*>(dst->pool()->allocate(elem.size * sizeof(uint8_t)));
memcpy(out->data, elem.data, elem.size);
}
}

7
be/src/olap/rowset/segment_v2/column_reader.h
View File

@ -34,7 +34,6 @@
namespace doris {
class ColumnBlock;
class Arena;
class RandomAccessFile;
class TypeInfo;
class BlockCompressionCodec;
@ -134,7 +133,7 @@ public:
// After one seek, we can call this function many times to read data
// into ColumnBlock. when read string type data, memory will allocated
// from Arena
// from MemPool
virtual Status next_batch(size_t* n, ColumnBlock* dst) = 0;
virtual rowid_t get_current_ordinal() const = 0;
@ -151,8 +150,8 @@ public:
//
// In the case that the values are themselves references
// to other memory (eg Slices), the referred-to memory is
// allocated in the dst column vector's arena.
Status scan(size_t* n, ColumnBlock* dst, Arena* arena);
// allocated in the dst column vector's MemPool.
Status scan(size_t* n, ColumnBlock* dst, MemPool* pool);
// release next_batch related resource
Status finish_batch();

10
be/src/olap/rowset/segment_v2/column_zone_map.cpp
View File

@ -23,15 +23,15 @@ namespace doris {
namespace segment_v2 {
ColumnZoneMapBuilder::ColumnZoneMapBuilder(Field* field) : _field(field) {
ColumnZoneMapBuilder::ColumnZoneMapBuilder(Field* field) : _field(field), _pool(&_tracker) {
PageBuilderOptions options;
options.data_page_size = 0;
_page_builder.reset(new BinaryPlainPageBuilder(options));
_zone_map.min_value = _field->allocate_value_from_arena(&_arena);
_zone_map.max_value = _field->allocate_value_from_arena(&_arena);
_zone_map.min_value = _field->allocate_value(&_pool);
_zone_map.max_value = _field->allocate_value(&_pool);
_reset_page_zone_map();
_segment_zone_map.min_value = _field->allocate_value_from_arena(&_arena);
_segment_zone_map.max_value = _field->allocate_value_from_arena(&_arena);
_segment_zone_map.min_value = _field->allocate_value(&_pool);
_segment_zone_map.max_value = _field->allocate_value(&_pool);
_reset_segment_zone_map();
}

9
be/src/olap/rowset/segment_v2/column_zone_map.h
View File

@ -25,6 +25,8 @@
#include "olap/field.h"
#include "gen_cpp/segment_v2.pb.h"
#include "olap/rowset/segment_v2/binary_plain_page.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
namespace doris {
@ -75,10 +77,13 @@ private:
private:
std::unique_ptr<BinaryPlainPageBuilder> _page_builder;
Field* _field;
// memory will be managed by arena
// memory will be managed by MemPool
ZoneMap _zone_map;
ZoneMap _segment_zone_map;
Arena _arena;
// TODO(zc): we should replace this memory pool later, we only allocate min/max
// for field. But MemPool allocate 4KB least, it will a waste for most cases.
MemTracker _tracker;
MemPool _pool;
};
// ColumnZoneMap

4
be/src/olap/rowset/segment_v2/segment_iterator.cpp
View File

@ -271,9 +271,9 @@ Status SegmentIterator::_seek_and_peek(rowid_t rowid) {
_column_iterators[cid]->seek_to_ordinal(rowid);
}
size_t num_rows = 1;
// please note that usually RowBlockV2.clear() is called to free arena memory before reading the next block,
// please note that usually RowBlockV2.clear() is called to free MemPool memory before reading the next block,
// but here since there won't be too many keys to seek, we don't call RowBlockV2.clear() so that we can use
// a single arena for all seeked keys.
// a single MemPool for all seeked keys.
RETURN_IF_ERROR(_next_batch(_seek_block.get(), &num_rows));
_seek_block->set_num_rows(num_rows);
return Status::OK();

2
be/src/olap/skiplist.h
View File

@ -44,7 +44,7 @@ private:
public:
// Create a new SkipList object that will use "cmp" for comparing keys,
// and will allocate memory using "*arena". Objects allocated in the arena
// and will allocate memory using "*mem_pool". Objects allocated in the mem_pool
// must remain allocated for the lifetime of the skiplist object.
explicit SkipList(Comparator cmp, MemPool* mem_pool);

1
be/src/olap/types.cpp
View File

@ -27,7 +27,6 @@ TypeInfo::TypeInfo(TypeTraitsClass t)
_cmp(TypeTraitsClass::cmp),
_shallow_copy(TypeTraitsClass::shallow_copy),
_deep_copy(TypeTraitsClass::deep_copy),
_deep_copy_with_arena(TypeTraitsClass::deep_copy_with_arena),
_direct_copy(TypeTraitsClass::direct_copy),
_from_string(TypeTraitsClass::from_string),
_to_string(TypeTraitsClass::to_string),

20
be/src/olap/types.h
View File

@ -34,7 +34,6 @@
#include "util/mem_util.hpp"
#include "util/slice.h"
#include "util/types.h"
#include "util/arena.h"
namespace doris {
@ -56,10 +55,6 @@ public:
_deep_copy(dest, src, mem_pool);
}
inline void deep_copy_with_arena(void* dest, const void* src, Arena* arena) const {
_deep_copy_with_arena(dest, src, arena);
}
inline void direct_copy(void* dest, const void* src) const {
_direct_copy(dest, src);
}
@ -83,7 +78,6 @@ private:
void (*_shallow_copy)(void* dest, const void* src);
void (*_deep_copy)(void* dest, const void* src, MemPool* mem_pool);
void (*_deep_copy_with_arena)(void* dest, const void* src, Arena* arena);
void (*_direct_copy)(void* dest, const void* src);
OLAPStatus (*_from_string)(void* buf, const std::string& scan_key);
@ -193,10 +187,6 @@ struct BaseFieldtypeTraits : public CppTypeTraits<field_type> {
*reinterpret_cast<CppType*>(dest) = *reinterpret_cast<const CppType*>(src);
}
static inline void deep_copy_with_arena(void* dest, const void* src, Arena* arena) {
*reinterpret_cast<CppType*>(dest) = *reinterpret_cast<const CppType*>(src);
}
static inline void direct_copy(void* dest, const void* src) {
*reinterpret_cast<CppType*>(dest) = *reinterpret_cast<const CppType*>(src);
}
@ -343,9 +333,6 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_LARGEINT> : public BaseFieldtypeTraits<OL
static void deep_copy(void* dest, const void* src, MemPool* mem_pool) {
*reinterpret_cast<PackedInt128*>(dest) = *reinterpret_cast<const PackedInt128*>(src);
}
static void deep_copy_with_arena(void* dest, const void* src, Arena* arena) {
*reinterpret_cast<PackedInt128*>(dest) = *reinterpret_cast<const PackedInt128*>(src);
}
static void direct_copy(void* dest, const void* src) {
*reinterpret_cast<PackedInt128*>(dest) = *reinterpret_cast<const PackedInt128*>(src);
}
@ -541,13 +528,6 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_CHAR> : public BaseFieldtypeTraits<OLAP_F
memory_copy(l_slice->data, r_slice->data, r_slice->size);
l_slice->size = r_slice->size;
}
static void deep_copy_with_arena(void* dest, const void* src, Arena* arena) {
auto l_slice = reinterpret_cast<Slice*>(dest);
auto r_slice = reinterpret_cast<const Slice*>(src);
l_slice->data = reinterpret_cast<char*>(arena->Allocate(r_slice->size));
memory_copy(l_slice->data, r_slice->data, r_slice->size);
l_slice->size = r_slice->size;
}
static void direct_copy(void* dest, const void* src) {
auto l_slice = reinterpret_cast<Slice*>(dest);
auto r_slice = reinterpret_cast<const Slice*>(src);

1
be/src/util/CMakeLists.txt
View File

@ -25,7 +25,6 @@ set(UTIL_FILES
arrow/row_batch.cpp
arrow/row_block.cpp
arrow/utils.cpp
arena.cpp
bfd_parser.cpp
bitmap.cpp
block_compression.cpp

68
be/src/util/arena.cpp
View File

@ -1,68 +0,0 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "util/arena.h"
#include <assert.h>
namespace doris {
static const int kBlockSize = 4096;
Arena::Arena() : memory_usage_(0) {
alloc_ptr_ = NULL; // First allocation will allocate a block
alloc_bytes_remaining_ = 0;
}
Arena::~Arena() {
for (size_t i = 0; i < blocks_.size(); i++) {
delete[] blocks_[i];
}
}
char* Arena::AllocateFallback(size_t bytes) {
if (bytes > kBlockSize / 4) {
// Object is more than a quarter of our block size. Allocate it separately
// to avoid wasting too much space in leftover bytes.
char* result = AllocateNewBlock(bytes);
return result;
}
// We waste the remaining space in the current block.
alloc_ptr_ = AllocateNewBlock(kBlockSize);
alloc_bytes_remaining_ = kBlockSize;
char* result = alloc_ptr_;
alloc_ptr_ += bytes;
alloc_bytes_remaining_ -= bytes;
return result;
}
char* Arena::AllocateAligned(size_t bytes) {
const int align = (sizeof(void*) > 8) ? sizeof(void*) : 8;
assert((align & (align-1)) == 0); // Pointer size should be a power of 2
size_t current_mod = reinterpret_cast<uintptr_t>(alloc_ptr_) & (align-1);
size_t slop = (current_mod == 0 ? 0 : align - current_mod);
size_t needed = bytes + slop;
char* result;
if (needed <= alloc_bytes_remaining_) {
result = alloc_ptr_ + slop;
alloc_ptr_ += needed;
alloc_bytes_remaining_ -= needed;
} else {
// AllocateFallback always returned aligned memory
result = AllocateFallback(bytes);
}
assert((reinterpret_cast<uintptr_t>(result) & (align-1)) == 0);
return result;
}
char* Arena::AllocateNewBlock(size_t block_bytes) {
char* result = new char[block_bytes];
blocks_.push_back(result);
memory_usage_.store(MemoryUsage() + block_bytes + sizeof(char*),
std::memory_order_relaxed);
return result;
}
} // namespace doris

74
be/src/util/arena.h
View File

@ -1,74 +0,0 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef DORIS_BE_SRC_COMMON_UTIL_ARENA_H
#define DORIS_BE_SRC_COMMON_UTIL_ARENA_H
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include <atomic>
#include <vector>
#include "common/compiler_util.h"
namespace doris {
class Arena {
public:
Arena();
~Arena();
// Return a pointer to a newly allocated memory block of "bytes" bytes.
char* Allocate(size_t bytes);
// Allocate memory with the normal alignment guarantees provided by malloc
char* AllocateAligned(size_t bytes);
// Returns an estimate of the total memory usage of data allocated
// by the arena.
size_t MemoryUsage() const {
//return reinterpret_cast<uintptr_t>(memory_usage_.NoBarrier_Load());
return memory_usage_.load(std::memory_order_relaxed);
}
// For the object wasn't allocated from Arena, but need to
// collect and control the object memory usage.
void track_memory(size_t bytes) {
memory_usage_.store(MemoryUsage() + bytes,std::memory_order_relaxed);
}
private:
char* AllocateFallback(size_t bytes);
char* AllocateNewBlock(size_t block_bytes);
// Allocation state
char* alloc_ptr_;
size_t alloc_bytes_remaining_;
// Array of new[] allocated memory blocks
std::vector<char*> blocks_;
// Total memory usage of the arena.
std::atomic<size_t> memory_usage_;
// No copying allowed
Arena(const Arena&);
void operator=(const Arena&);
};
inline char* Arena::Allocate(size_t bytes) {
if (UNLIKELY(bytes == 0)) { return nullptr; }
if (bytes <= alloc_bytes_remaining_) {
char* result = alloc_ptr_;
alloc_ptr_ += bytes;
alloc_bytes_remaining_ -= bytes;
return result;
}
return AllocateFallback(bytes);
}
} // namespace doris
#endif // DORIS_BE_SRC_COMMON_UTIL_ARENA_H

15
be/test/olap/delta_writer_test.cpp
View File

@ -31,6 +31,8 @@
#include "runtime/tuple.h"
#include "runtime/descriptor_helper.h"
#include "runtime/exec_env.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
#include "util/logging.h"
#include "olap/options.h"
#include "olap/tablet_meta_manager.h"
@ -345,10 +347,11 @@ TEST_F(TestDeltaWriter, write) {
DeltaWriter::open(&write_req, k_mem_tracker, &delta_writer);
ASSERT_NE(delta_writer, nullptr);
Arena arena;
MemTracker tracker;
MemPool pool(&tracker);
// Tuple 1
{
Tuple* tuple = reinterpret_cast<Tuple*>(arena.Allocate(tuple_desc->byte_size()));
Tuple* tuple = reinterpret_cast<Tuple*>(pool.allocate(tuple_desc->byte_size()));
memset(tuple, 0, tuple_desc->byte_size());
*(int8_t*)(tuple->get_slot(slots[0]->tuple_offset())) = -127;
*(int16_t*)(tuple->get_slot(slots[1]->tuple_offset())) = -32767;
@ -362,12 +365,12 @@ TEST_F(TestDeltaWriter, write) {
((DateTimeValue*)(tuple->get_slot(slots[6]->tuple_offset())))->from_date_str("2636-08-16 19:39:43", 19);
StringValue* char_ptr = (StringValue*)(tuple->get_slot(slots[7]->tuple_offset()));
char_ptr->ptr = arena.Allocate(4);
char_ptr->ptr = (char*)pool.allocate(4);
memcpy(char_ptr->ptr, "abcd", 4);
char_ptr->len = 4;
StringValue* var_ptr = (StringValue*)(tuple->get_slot(slots[8]->tuple_offset()));
var_ptr->ptr = arena.Allocate(5);
var_ptr->ptr = (char*)pool.allocate(5);
memcpy(var_ptr->ptr, "abcde", 5);
var_ptr->len = 5;
@ -385,12 +388,12 @@ TEST_F(TestDeltaWriter, write) {
((DateTimeValue*)(tuple->get_slot(slots[16]->tuple_offset())))->from_date_str("2636-08-16 19:39:43", 19);
char_ptr = (StringValue*)(tuple->get_slot(slots[17]->tuple_offset()));
char_ptr->ptr = arena.Allocate(4);
char_ptr->ptr = (char*)pool.allocate(4);
memcpy(char_ptr->ptr, "abcd", 4);
char_ptr->len = 4;
var_ptr = (StringValue*)(tuple->get_slot(slots[18]->tuple_offset()));
var_ptr->ptr = arena.Allocate(5);
var_ptr->ptr = (char*)pool.allocate(5);
memcpy(var_ptr->ptr, "abcde", 5);
var_ptr->len = 5;

29
be/test/olap/key_coder_test.cpp
View File

@ -22,14 +22,19 @@
#include <string.h>
#include "util/debug_util.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
namespace doris {
class KeyCoderTest : public testing::Test {
public:
KeyCoderTest() { }
KeyCoderTest() : _pool(&_tracker) { }
virtual ~KeyCoderTest() {
}
private:
MemTracker _tracker;
MemPool _pool;
};
template<FieldType type>
@ -97,7 +102,7 @@ void test_integer_encode() {
}
}
TEST(KeyCoderTest, test_int) {
TEST_F(KeyCoderTest, test_int) {
test_integer_encode<OLAP_FIELD_TYPE_TINYINT>();
test_integer_encode<OLAP_FIELD_TYPE_SMALLINT>();
test_integer_encode<OLAP_FIELD_TYPE_INT>();
@ -108,7 +113,7 @@ TEST(KeyCoderTest, test_int) {
test_integer_encode<OLAP_FIELD_TYPE_DATETIME>();
}
TEST(KeyCoderTest, test_date) {
TEST_F(KeyCoderTest, test_date) {
using CppType = uint24_t;
auto key_coder = get_key_coder(OLAP_FIELD_TYPE_DATE);
@ -168,7 +173,7 @@ TEST(KeyCoderTest, test_date) {
}
}
TEST(KeyCoderTest, test_decimal) {
TEST_F(KeyCoderTest, test_decimal) {
auto key_coder = get_key_coder(OLAP_FIELD_TYPE_DECIMAL);
decimal12_t val1(1, 100000000);
@ -212,7 +217,7 @@ TEST(KeyCoderTest, test_decimal) {
}
}
TEST(KeyCoderTest, test_char) {
TEST_F(KeyCoderTest, test_char) {
auto key_coder = get_key_coder(OLAP_FIELD_TYPE_CHAR);
char buf[] = "1234567890";
@ -223,9 +228,8 @@ TEST(KeyCoderTest, test_char) {
key_coder->encode_ascending(&slice, 10, &key);
Slice encoded_key(key);
Arena arena;
Slice check_slice;
auto st = key_coder->decode_ascending(&encoded_key, 10, (uint8_t*)&check_slice, &arena);
auto st = key_coder->decode_ascending(&encoded_key, 10, (uint8_t*)&check_slice, &_pool);
ASSERT_TRUE(st.ok());
ASSERT_STREQ("1234567890", check_slice.data);
@ -236,16 +240,15 @@ TEST(KeyCoderTest, test_char) {
key_coder->encode_ascending(&slice, 5, &key);
Slice encoded_key(key);
Arena arena;
Slice check_slice;
auto st = key_coder->decode_ascending(&encoded_key, 5, (uint8_t*)&check_slice, &arena);
auto st = key_coder->decode_ascending(&encoded_key, 5, (uint8_t*)&check_slice, &_pool);
ASSERT_TRUE(st.ok());
ASSERT_STREQ("12345", check_slice.data);
}
}
TEST(KeyCoderTest, test_varchar) {
TEST_F(KeyCoderTest, test_varchar) {
auto key_coder = get_key_coder(OLAP_FIELD_TYPE_VARCHAR);
char buf[] = "1234567890";
@ -256,9 +259,8 @@ TEST(KeyCoderTest, test_varchar) {
key_coder->encode_ascending(&slice, 15, &key);
Slice encoded_key(key);
Arena arena;
Slice check_slice;
auto st = key_coder->decode_ascending(&encoded_key, 15, (uint8_t*)&check_slice, &arena);
auto st = key_coder->decode_ascending(&encoded_key, 15, (uint8_t*)&check_slice, &_pool);
ASSERT_TRUE(st.ok());
ASSERT_STREQ("1234567890", check_slice.data);
@ -269,9 +271,8 @@ TEST(KeyCoderTest, test_varchar) {
key_coder->encode_ascending(&slice, 5, &key);
Slice encoded_key(key);
Arena arena;
Slice check_slice;
auto st = key_coder->decode_ascending(&encoded_key, 5, (uint8_t*)&check_slice, &arena);
auto st = key_coder->decode_ascending(&encoded_key, 5, (uint8_t*)&check_slice, &_pool);
ASSERT_TRUE(st.ok());
ASSERT_STREQ("12345", check_slice.data);

16
be/test/olap/rowset/segment_v2/binary_dict_page_test.cpp
View File

@ -27,6 +27,8 @@
#include "olap/olap_common.h"
#include "olap/types.h"
#include "util/debug_util.h"
#include "runtime/mem_tracker.h"
#include "runtime/mem_pool.h"
namespace doris {
namespace segment_v2 {
@ -70,11 +72,12 @@ public:
ASSERT_EQ(slices.size(), page_decoder.count());
//check values
Arena arena;
MemTracker tracker;
MemPool pool(&tracker);
TypeInfo* type_info = get_type_info(OLAP_FIELD_TYPE_VARCHAR);
size_t size = slices.size();
Slice* values = reinterpret_cast<Slice*>(arena.Allocate(size * sizeof(Slice)));
ColumnBlock column_block(type_info, (uint8_t*)values, nullptr, size, &arena);
Slice* values = reinterpret_cast<Slice*>(pool.allocate(size * sizeof(Slice)));
ColumnBlock column_block(type_info, (uint8_t*)values, nullptr, size, &pool);
ColumnBlockView block_view(&column_block);
status = page_decoder.next_batch(&size, &block_view);
@ -161,10 +164,11 @@ public:
ASSERT_TRUE(status.ok());
//check values
Arena arena;
MemTracker tracker;
MemPool pool(&tracker);
TypeInfo* type_info = get_type_info(OLAP_FIELD_TYPE_VARCHAR);
Slice* values = reinterpret_cast<Slice*>(arena.Allocate(sizeof(Slice)));
ColumnBlock column_block(type_info, (uint8_t*)values, nullptr, 1, &arena);
Slice* values = reinterpret_cast<Slice*>(pool.allocate(sizeof(Slice)));
ColumnBlock column_block(type_info, (uint8_t*)values, nullptr, 1, &pool);
ColumnBlockView block_view(&column_block);
size_t num = 1;

18
be/test/olap/rowset/segment_v2/binary_plain_page_test.cpp
View File

@ -25,6 +25,8 @@
#include "olap/rowset/segment_v2/binary_plain_page.h"
#include "olap/olap_common.h"
#include "olap/types.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
namespace doris {
namespace segment_v2 {
@ -59,13 +61,13 @@ public:
//test1
size_t size = 3;
Arena arena;
Slice* values = reinterpret_cast<Slice*>(arena.Allocate(size * sizeof(Slice)));
uint8_t* null_bitmap = reinterpret_cast<uint8_t*>(arena.Allocate(BitmapSize(size)));
ColumnBlock block(get_type_info(OLAP_FIELD_TYPE_VARCHAR), (uint8_t*)values, null_bitmap, size, &arena);
MemTracker tracker;
MemPool pool(&tracker);
size_t size = 3;
Slice* values = reinterpret_cast<Slice*>(pool.allocate(size * sizeof(Slice)));
uint8_t* null_bitmap = reinterpret_cast<uint8_t*>(pool.allocate(BitmapSize(size)));
ColumnBlock block(get_type_info(OLAP_FIELD_TYPE_VARCHAR), (uint8_t*)values, null_bitmap, size, &pool);
ColumnBlockView column_block_view(&block);
status = page_decoder.next_batch(&size, &column_block_view);
@ -77,8 +79,8 @@ public:
ASSERT_EQ (",", value[1].to_string());
ASSERT_EQ ("Doris", value[2].to_string());
Slice* values2 = reinterpret_cast<Slice*>(arena.Allocate(size * sizeof(Slice)));
ColumnBlock block2(get_type_info(OLAP_FIELD_TYPE_VARCHAR), (uint8_t*)values2, null_bitmap, 1, &arena);
Slice* values2 = reinterpret_cast<Slice*>(pool.allocate(size * sizeof(Slice)));
ColumnBlock block2(get_type_info(OLAP_FIELD_TYPE_VARCHAR), (uint8_t*)values2, null_bitmap, 1, &pool);
ColumnBlockView column_block_view2(&block2);
size_t fetch_num = 1;

17
be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp
View File

@ -22,7 +22,8 @@
#include "olap/rowset/segment_v2/page_builder.h"
#include "olap/rowset/segment_v2/page_decoder.h"
#include "olap/rowset/segment_v2/bitshuffle_page.h"
#include "util/arena.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
#include "util/logging.h"
using doris::segment_v2::PageBuilderOptions;
@ -35,9 +36,10 @@ public:
template<FieldType type, class PageDecoderType>
void copy_one(PageDecoderType* decoder, typename TypeTraits<type>::CppType* ret) {
Arena arena;
MemTracker tracker;
MemPool pool(&tracker);
uint8_t null_bitmap = 0;
ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &arena);
ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &pool);
ColumnBlockView column_block_view(&block);
size_t n = 1;
@ -64,11 +66,12 @@ public:
ASSERT_TRUE(status.ok());
ASSERT_EQ(0, page_decoder.current_index());
Arena arena;
MemTracker tracker;
MemPool pool(&tracker);
CppType* values = reinterpret_cast<CppType*>(arena.Allocate(size * sizeof(CppType)));
uint8_t* null_bitmap = reinterpret_cast<uint8_t*>(arena.Allocate(BitmapSize(size)));
ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &arena);
CppType* values = reinterpret_cast<CppType*>(pool.allocate(size * sizeof(CppType)));
uint8_t* null_bitmap = reinterpret_cast<uint8_t*>(pool.allocate(BitmapSize(size)));
ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &pool);
ColumnBlockView column_block_view(&block);
status = page_decoder.next_batch(&size, &column_block_view);

24
be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
View File

@ -29,16 +29,20 @@
#include "olap/types.h"
#include "olap/column_block.h"
#include "util/file_utils.h"
#include "util/arena.h"
#include "runtime/mem_tracker.h"
#include "runtime/mem_pool.h"
namespace doris {
namespace segment_v2 {
class ColumnReaderWriterTest : public testing::Test {
public:
ColumnReaderWriterTest() { }
ColumnReaderWriterTest() : _pool(&_tracker) { }
virtual ~ColumnReaderWriterTest() {
}
private:
MemTracker _tracker;
MemPool _pool;
};
template<FieldType type, EncodingTypePB encoding>
@ -119,11 +123,12 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s
st = iter->seek_to_first();
ASSERT_TRUE(st.ok()) << st.to_string();
Arena arena;
MemTracker tracker;
MemPool pool(&tracker);
Type vals[1024];
Type* vals_ = vals;
uint8_t is_null[1024];
ColumnBlock col(type_info, (uint8_t*)vals, is_null, 1024, &arena);
ColumnBlock col(type_info, (uint8_t*)vals, is_null, 1024, &pool);
int idx = 0;
while (true) {
@ -152,10 +157,11 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s
}
// random read
{
Arena arena;
MemTracker tracker;
MemPool pool(&tracker);
Type vals[1024];
uint8_t is_null[1024];
ColumnBlock col(type_info, (uint8_t*)vals, is_null, 1024, &arena);
ColumnBlock col(type_info, (uint8_t*)vals, is_null, 1024, &pool);
for (int rowid = 0; rowid < num_rows; rowid += 4025) {
st = iter->seek_to_ordinal(rowid);
@ -224,8 +230,6 @@ TEST_F(ColumnReaderWriterTest, test_nullable) {
}
TEST_F(ColumnReaderWriterTest, test_types) {
Arena arena;
size_t num_uint8_rows = 1024 * 1024;
uint8_t* is_null = new uint8_t[num_uint8_rows];
@ -241,8 +245,8 @@ TEST_F(ColumnReaderWriterTest, test_types) {
datetime_vals[i] = i + 33;
decimal_vals[i] = decimal12_t(i, i); // 1.000000001
set_column_value_by_type(OLAP_FIELD_TYPE_VARCHAR, i, (char*)&varchar_vals[i], &arena);
set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, i, (char*)&char_vals[i], &arena, 8);
set_column_value_by_type(OLAP_FIELD_TYPE_VARCHAR, i, (char*)&varchar_vals[i], &_pool);
set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, i, (char*)&char_vals[i], &_pool, 8);
BitmapChange(is_null, i, (i % 4) == 0);
}

17
be/test/olap/rowset/segment_v2/frame_of_reference_page_test.cpp
View File

@ -22,7 +22,8 @@
#include "olap/rowset/segment_v2/page_builder.h"
#include "olap/rowset/segment_v2/page_decoder.h"
#include "olap/rowset/segment_v2/frame_of_reference_page.h"
#include "util/arena.h"
#include "runtime/mem_tracker.h"
#include "runtime/mem_pool.h"
#include "util/logging.h"
using doris::segment_v2::PageBuilderOptions;
@ -33,9 +34,10 @@ class FrameOfReferencePageTest : public testing::Test {
public:
template<FieldType type, class PageDecoderType>
void copy_one(PageDecoderType* decoder, typename TypeTraits<type>::CppType* ret) {
Arena arena;
MemTracker tracker;
MemPool pool(&tracker);
uint8_t null_bitmap = 0;
ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &arena);
ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &pool);
ColumnBlockView column_block_view(&block);
size_t n = 1;
@ -63,10 +65,11 @@ public:
ASSERT_EQ(0, for_page_decoder.current_index());
ASSERT_EQ(size, for_page_decoder.count());
Arena arena;
CppType* values = reinterpret_cast<CppType*>(arena.Allocate(size * sizeof(CppType)));
uint8_t* null_bitmap = reinterpret_cast<uint8_t*>(arena.Allocate(BitmapSize(size)));
ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &arena);
MemTracker tracker;
MemPool pool(&tracker);
CppType* values = reinterpret_cast<CppType*>(pool.allocate(size * sizeof(CppType)));
uint8_t* null_bitmap = reinterpret_cast<uint8_t*>(pool.allocate(BitmapSize(size)));
ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &pool);
ColumnBlockView column_block_view(&block);
size_t size_to_fetch = size;
status = for_page_decoder.next_batch(&size_to_fetch, &column_block_view);

17
be/test/olap/rowset/segment_v2/plain_page_test.cpp
View File

@ -25,7 +25,8 @@
#include "common/logging.h"
#include "olap/olap_common.h"
#include "olap/types.h"
#include "util/arena.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
namespace doris {
namespace segment_v2 {
@ -45,9 +46,10 @@ public:
template<FieldType type, class PageDecoderType>
void copy_one(PageDecoderType* decoder, typename TypeTraits<type>::CppType* ret) {
Arena arena;
MemTracker tracker;
MemPool pool(&tracker);
uint8_t null_bitmap = 0;
ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &arena);
ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &pool);
ColumnBlockView column_block_view(&block);
size_t n = 1;
@ -74,11 +76,12 @@ public:
ASSERT_EQ(0, page_decoder.current_index());
Arena arena;
MemTracker tracker;
MemPool pool(&tracker);
CppType* values = reinterpret_cast<CppType*>(arena.Allocate(size * sizeof(CppType)));
uint8_t* null_bitmap = reinterpret_cast<uint8_t*>(arena.Allocate(BitmapSize(size)));
ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &arena);
CppType* values = reinterpret_cast<CppType*>(pool.allocate(size * sizeof(CppType)));
uint8_t* null_bitmap = reinterpret_cast<uint8_t*>(pool.allocate(BitmapSize(size)));
ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &pool);
ColumnBlockView column_block_view(&block);
status = page_decoder.next_batch(&size, &column_block_view);
ASSERT_TRUE(status.ok());

18
be/test/olap/rowset/segment_v2/rle_page_test.cpp
View File

@ -22,8 +22,9 @@
#include "olap/rowset/segment_v2/page_builder.h"
#include "olap/rowset/segment_v2/page_decoder.h"
#include "olap/rowset/segment_v2/rle_page.h"
#include "util/arena.h"
#include "util/logging.h"
#include "runtime/mem_tracker.h"
#include "runtime/mem_pool.h"
using doris::segment_v2::PageBuilderOptions;
using doris::segment_v2::PageDecoderOptions;
@ -36,9 +37,10 @@ public:
template<FieldType type, class PageDecoderType>
void copy_one(PageDecoderType* decoder, typename TypeTraits<type>::CppType* ret) {
Arena arena;
MemTracker tracker;
MemPool pool(&tracker);
uint8_t null_bitmap = 0;
ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &arena);
ColumnBlock block(get_type_info(type), (uint8_t*)ret, &null_bitmap, 1, &pool);
ColumnBlockView column_block_view(&block);
size_t n = 1;
@ -66,11 +68,11 @@ public:
ASSERT_EQ(0, rle_page_decoder.current_index());
ASSERT_EQ(size, rle_page_decoder.count());
Arena arena;
CppType* values = reinterpret_cast<CppType*>(arena.Allocate(size * sizeof(CppType)));
uint8_t* null_bitmap = reinterpret_cast<uint8_t*>(arena.Allocate(BitmapSize(size)));
ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &arena);
MemTracker tracker;
MemPool pool(&tracker);
CppType* values = reinterpret_cast<CppType*>(pool.allocate(size * sizeof(CppType)));
uint8_t* null_bitmap = reinterpret_cast<uint8_t*>(pool.allocate(BitmapSize(size)));
ColumnBlock block(get_type_info(Type), (uint8_t*)values, null_bitmap, size, &pool);
ColumnBlockView column_block_view(&block);
size_t size_to_fetch = size;
status = rle_page_decoder.next_batch(&size_to_fetch, &column_block_view);

17
be/test/olap/rowset/segment_v2/segment_test.cpp
View File

@ -32,6 +32,8 @@
#include "olap/types.h"
#include "olap/tablet_schema_helper.h"
#include "util/file_utils.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
namespace doris {
namespace segment_v2 {
@ -640,7 +642,8 @@ TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) {
TEST_F(SegmentReaderWriterTest, TestStringDict) {
size_t num_rows_per_block = 10;
Arena _arena;
MemTracker tracker;
MemPool pool(&tracker);
std::shared_ptr<TabletSchema> tablet_schema(new TabletSchema());
tablet_schema->_num_columns = 4;
@ -677,7 +680,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
for (int j = 0; j < 4; ++j) {
auto cell = row.cell(j);
cell.set_not_null();
set_column_value_by_type(tablet_schema->_cols[j]._type, i * 10 + j, (char*)cell.mutable_cell_ptr(), &_arena, tablet_schema->_cols[j]._length);
set_column_value_by_type(tablet_schema->_cols[j]._type, i * 10 + j, (char*)cell.mutable_cell_ptr(), &pool, tablet_schema->_cols[j]._length);
}
Status status = writer.append_row(row);
ASSERT_TRUE(status.ok());
@ -722,7 +725,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
const Slice* actual = reinterpret_cast<const Slice*>(column_block.cell_ptr(i));
Slice expect;
set_column_value_by_type(tablet_schema->_cols[j]._type, rid * 10 + cid, reinterpret_cast<char*>(&expect), &_arena, tablet_schema->_cols[j]._length);
set_column_value_by_type(tablet_schema->_cols[j]._type, rid * 10 + cid, reinterpret_cast<char*>(&expect), &pool, tablet_schema->_cols[j]._length);
ASSERT_EQ(expect.to_string(), actual->to_string());
}
}
@ -738,7 +741,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
{
auto cell = lower_bound->cell(0);
cell.set_not_null();
set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, 40970, (char*)cell.mutable_cell_ptr(), &_arena, tablet_schema->_cols[0]._length);
set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, 40970, (char*)cell.mutable_cell_ptr(), &pool, tablet_schema->_cols[0]._length);
}
StorageReadOptions read_opts;
@ -760,7 +763,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
{
auto cell = lower_bound->cell(0);
cell.set_not_null();
set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, -2, (char*)cell.mutable_cell_ptr(), &_arena, tablet_schema->_cols[0]._length);
set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, -2, (char*)cell.mutable_cell_ptr(), &pool, tablet_schema->_cols[0]._length);
}
std::unique_ptr<RowCursor> upper_bound(new RowCursor());
@ -768,7 +771,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
{
auto cell = upper_bound->cell(0);
cell.set_not_null();
set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, -1, (char*)cell.mutable_cell_ptr(), &_arena, tablet_schema->_cols[0]._length);
set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, -1, (char*)cell.mutable_cell_ptr(), &pool, tablet_schema->_cols[0]._length);
}
StorageReadOptions read_opts;
@ -820,7 +823,7 @@ TEST_F(SegmentReaderWriterTest, TestStringDict) {
const Slice* actual = reinterpret_cast<const Slice*>(column_block.cell_ptr(i));
Slice expect;
set_column_value_by_type(tablet_schema->_cols[j]._type, rid * 10 + cid, reinterpret_cast<char*>(&expect), &_arena, tablet_schema->_cols[j]._length);
set_column_value_by_type(tablet_schema->_cols[j]._type, rid * 10 + cid, reinterpret_cast<char*>(&expect), &pool, tablet_schema->_cols[j]._length);
ASSERT_EQ(expect.to_string(), actual->to_string()) << "rid:" << rid << ", i:" << i;;
}
}

8
be/test/olap/tablet_schema_helper.h
View File

@ -20,6 +20,8 @@
#include <string>
#include "olap/tablet_schema.h"
#include "runtime/mem_pool.h"
namespace doris {
TabletColumn create_int_key(int32_t id, bool is_nullable = true) {
@ -79,14 +81,14 @@ TabletColumn create_varchar_key(int32_t id, bool is_nullable = true) {
return column;
}
void set_column_value_by_type(FieldType fieldType, int src, char* target, Arena* _arena, size_t _length = 0) {
void set_column_value_by_type(FieldType fieldType, int src, char* target, MemPool* pool, size_t _length = 0) {
if (fieldType == OLAP_FIELD_TYPE_CHAR) {
char* src_value = &std::to_string(src)[0];
int src_len = strlen(src_value);
auto* dest_slice = (Slice*)target;
dest_slice->size = _length;
dest_slice->data = _arena->Allocate(dest_slice->size);
dest_slice->data = (char*)pool->allocate(dest_slice->size);
memcpy(dest_slice->data, src_value, src_len);
memset(dest_slice->data + src_len, 0, dest_slice->size - src_len);
} else if (fieldType == OLAP_FIELD_TYPE_VARCHAR) {
@ -95,7 +97,7 @@ void set_column_value_by_type(FieldType fieldType, int src, char* target, Arena*
auto* dest_slice = (Slice*)target;
dest_slice->size = src_len;
dest_slice->data = _arena->Allocate(src_len);
dest_slice->data = (char*)pool->allocate(src_len);
std::memcpy(dest_slice->data, src_value, src_len);
} else {
*(int*)target = src;

1
be/test/util/CMakeLists.txt
View File

@ -37,7 +37,6 @@ ADD_BE_TEST(types_test)
ADD_BE_TEST(json_util_test)
ADD_BE_TEST(byte_buffer_test2)
ADD_BE_TEST(uid_util_test)
ADD_BE_TEST(arena_test)
ADD_BE_TEST(aes_util_test)
ADD_BE_TEST(md5_test)
ADD_BE_TEST(bitmap_test)

81
be/test/util/arena_test.cpp
View File

@ -1,81 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include <gtest/gtest.h>
#include "util/arena.h"
#include "util/random.h"
namespace doris {
class ArenaTest : public testing::Test { };
TEST_F(ArenaTest, Empty) {
Arena arena;
}
TEST_F(ArenaTest, Simple) {
std::vector<std::pair<size_t, char*> > allocated;
Arena arena;
const int N = 100000;
size_t bytes = 0;
Random rnd(301);
for (int i = 0; i < N; i++) {
size_t s;
if (i % (N / 10) == 0) {
s = i;
} else {
s = rnd.OneIn(4000) ? rnd.Uniform(6000) :
(rnd.OneIn(10) ? rnd.Uniform(100) : rnd.Uniform(20));
}
if (s == 0) {
// Our arena disallows size 0 allocations.
s = 1;
}
char* r;
if (rnd.OneIn(10)) {
r = arena.AllocateAligned(s);
} else {
r = arena.Allocate(s);
}
for (size_t b = 0; b < s; b++) {
// Fill the "i"th allocation with a known bit pattern
r[b] = i % 256;
}
bytes += s;
allocated.push_back(std::make_pair(s, r));
ASSERT_GE(arena.MemoryUsage(), bytes);
if (i > N / 10) {
ASSERT_LE(arena.MemoryUsage(), bytes * 1.10);
}
}
for (size_t i = 0; i < allocated.size(); i++) {
size_t num_bytes = allocated[i].first;
const char* p = allocated[i].second;
for (size_t b = 0; b < num_bytes; b++) {
// Check the "i"th allocation for the known bit pattern
ASSERT_EQ(int(p[b]) & 0xff, i % 256);
}
}
}
} // namespace doris
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

1
run-ut.sh
View File

@ -145,7 +145,6 @@ ${DORIS_TEST_BINARY_DIR}/util/new_metrics_test
${DORIS_TEST_BINARY_DIR}/util/doris_metrics_test
${DORIS_TEST_BINARY_DIR}/util/system_metrics_test
${DORIS_TEST_BINARY_DIR}/util/core_local_test
${DORIS_TEST_BINARY_DIR}/util/arena_test
${DORIS_TEST_BINARY_DIR}/util/types_test
${DORIS_TEST_BINARY_DIR}/util/json_util_test
${DORIS_TEST_BINARY_DIR}/util/byte_buffer_test2