hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

reduce EncodePivot memory usage with lots of columns and few rows

Browse Source
This commit is contained in:
saltonz
2023-11-16 14:46:15 +00:00
committed by ob-robot
parent cef829bb89
commit 960df4d65c
11 changed files with 47 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
unittest/storage/blocksstable/cs_encoding/test_integer_stream.cpp
View File

@ -283,7 +283,7 @@ public:
K(loop), K(sizeof(T)), K(min), K(max), K(mon), K(use_null_replace_ref), K(actual_uint_width));
ObIntegerStreamEncoderCtx ctx;
ObCSEncodingOpt encoding_opt;
ObArenaAllocator alloctor;
ObArenaAllocator allocator;
const ObCompressorType compress_type = ObCompressorType::ZSTD_1_3_8_COMPRESSOR;
ctx.meta_.width_ = actual_uint_width;
ctx.meta_.type_ = type;
@ -302,10 +302,10 @@ public:
if (ctx.meta_.is_use_null_replace_value()) {
ctx.meta_.set_null_replaced_value(null_replace_value);
}
ctx.build_stream_encoder_info(has_null, false, &encoding_opt, nullptr, -1, compress_type, &alloctor);
ctx.build_stream_encoder_info(has_null, false, &encoding_opt, nullptr, -1, compress_type, &allocator);
ObIntegerStreamEncoder encoder;
ObColDatums *datums = new ObColDatums();
ObColDatums *datums = new ObColDatums(allocator);
datums->reserve(1 << 20);
generate_datums<T>(datums, size, has_null, min, max, mon);
int64_t bitmap_size = pad8(size);

5
unittest/storage/blocksstable/cs_encoding/test_string_stream.cpp
View File

@ -122,6 +122,7 @@ public:
{
LOG_INFO("test_and_check_string_encoding", K(size), K(type), K(use_zero_len_as_null), K(has_null), K(is_fix_len),
K(use_nullbitmap), K(all_null), K(half_null_half_empty), K(use_null_replaced_ref));
ObArenaAllocator local_arena;
ObStringStreamEncoderCtx ctx;
ObCSEncodingOpt encoding_opt;
bool is_use_zero_len_as_null = use_zero_len_as_null;
@ -140,7 +141,7 @@ public:
ObStringStreamEncoder encoder;
uint32_t *data = nullptr;
ObColDatums *datums = new ObColDatums();
ObColDatums *datums = new ObColDatums(local_arena);
ASSERT_EQ(OB_SUCCESS, datums->resize(max_count));
datums->reuse();
if (half_null_half_empty) {
@ -203,7 +204,7 @@ public:
str_data.set(all_string_writer.data(), all_string_writer.length());
// 3. decode str
ObColDatums *datums2 = new ObColDatums();
ObColDatums *datums2 = new ObColDatums(local_arena);
ASSERT_EQ(OB_SUCCESS, datums2->resize(max_count));
datums2->reuse();
for (int64_t i = 0; i < size; i++) {

3
unittest/storage/blocksstable/encoding/test_encoding_util.cpp
View File

@ -60,8 +60,9 @@ TEST(ObMultiDimArray_T, timestamp_with_time_zone)
{
// for timestamp with time zone, we need use binary_equal to build encoding hash table.
ObEncodingHashTableBuilder hash_builder;
ObArenaAllocator local_arena;
ASSERT_EQ(OB_SUCCESS, hash_builder.create(8, 8));
ObColDatums time_arr;
ObColDatums time_arr(local_arena);
ObStorageDatum t1, t2;
ObTimeZoneInfo time_zone_info1, time_zone_info2;