database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Support boolean/date/datetime/decimal types in segment V2 (#1863)

Browse Source
This commit is contained in:
wubiao
2019-09-25 13:53:00 +08:00
committed by ZHAO Chun
parent c643cbd30c
commit eb840ecca8
3 changed files with 60 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/src/olap/rowset/segment_v2/bitshuffle_page.h
View File

@ -232,8 +232,10 @@ public:
switch (_size_of_element) {
case 1:
case 2:
case 3:
case 4:
case 8:
case 12:
case 16:
break;
default:

22
be/src/olap/rowset/segment_v2/encoding_info.cpp
View File

@ -19,6 +19,7 @@
#include "olap/olap_common.h"
#include "olap/rowset/segment_v2/bitshuffle_page.h"
#include "olap/rowset/segment_v2/rle_page.h"
namespace doris {
namespace segment_v2 {
@ -54,6 +55,18 @@ struct TypeEncodingTraits<type, BIT_SHUFFLE> {
}
};
template<FieldType type>
struct TypeEncodingTraits<type, RLE> {
static Status create_page_builder(const PageBuilderOptions& opts, PageBuilder** builder) {
*builder = new RlePageBuilder<type>(opts);
return Status::OK();
}
static Status create_page_decoder(const Slice& data, const PageDecoderOptions& opts, PageDecoder** decoder) {
*decoder = new RlePageDecoder<type>(data, opts);
return Status::OK();
}
};
template<FieldType Type, EncodingTypePB Encoding>
struct EncodingTraits : TypeEncodingTraits<Type, Encoding> {
static const FieldType type = Type;
@ -109,6 +122,15 @@ EncodingInfoResolver::EncodingInfoResolver() {
_add_map<OLAP_FIELD_TYPE_FLOAT, PLAIN_ENCODING>();
_add_map<OLAP_FIELD_TYPE_DOUBLE, BIT_SHUFFLE>();
_add_map<OLAP_FIELD_TYPE_DOUBLE, PLAIN_ENCODING>();
_add_map<OLAP_FIELD_TYPE_BOOL, RLE>();
_add_map<OLAP_FIELD_TYPE_BOOL, BIT_SHUFFLE>();
_add_map<OLAP_FIELD_TYPE_BOOL, PLAIN_ENCODING>();
_add_map<OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>();
_add_map<OLAP_FIELD_TYPE_DATE, PLAIN_ENCODING>();
_add_map<OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>();
_add_map<OLAP_FIELD_TYPE_DATETIME, PLAIN_ENCODING>();
_add_map<OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>();
_add_map<OLAP_FIELD_TYPE_DECIMAL, PLAIN_ENCODING>();
}
EncodingInfoResolver::~EncodingInfoResolver() {

36
be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
View File

@ -17,6 +17,7 @@
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/column_writer.h"
#include "olap/decimal12.h"
#include <gtest/gtest.h>
#include <iostream>
@ -201,6 +202,41 @@ TEST_F(ColumnReaderWriterTest, test_nullable) {
delete[] double_vals;
}
TEST_F(ColumnReaderWriterTest, test_types) {
size_t num_uint8_rows = 1024 * 1024;
uint8_t* is_null = new uint8_t[num_uint8_rows];
bool* bool_vals = new bool[num_uint8_rows];
uint24_t* date_vals = new uint24_t[num_uint8_rows];
uint64_t* datetime_vals = new uint64_t[num_uint8_rows];
decimal12_t* decimal_vals = new decimal12_t[num_uint8_rows];
for (int i = 0; i < num_uint8_rows; ++i) {
bool_vals[i] = i % 2;
date_vals[i] = i + 33;
datetime_vals[i] = i + 33;
decimal_vals[i] = decimal12_t(i, i); // 1.000000001
BitmapChange(is_null, i, (i % 4) == 0);
}
test_nullable_data<OLAP_FIELD_TYPE_BOOL, BIT_SHUFFLE>((uint8_t*)bool_vals, is_null, num_uint8_rows, "null_bool_bs");
test_nullable_data<OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>((uint8_t*)date_vals, is_null, num_uint8_rows / 3, "null_date_bs");
for (int i = 0; i < num_uint8_rows; ++i) {
BitmapChange(is_null, i, (i % 16) == 0);
}
test_nullable_data<OLAP_FIELD_TYPE_DATETIME, BIT_SHUFFLE>((uint8_t*)datetime_vals, is_null, num_uint8_rows / 8, "null_datetime_bs");
for (int i = 0; i< num_uint8_rows; ++i) {
BitmapChange(is_null, i, (i % 24) == 0);
}
test_nullable_data<OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>((uint8_t*)decimal_vals, is_null, num_uint8_rows / 12, "null_decimal_bs");
delete[] is_null;
delete[] bool_vals;
delete[] date_vals;
delete[] datetime_vals;
delete[] decimal_vals;
}
}
}