database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Make Segment v2 use string's real length(#1943) (#1944)

Browse Source
This commit is contained in:
wangbo
2019-10-13 13:23:43 +08:00
committed by ZHAO Chun
parent 8232261df1
commit 80e9b21fb0
14 changed files with 143 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
be/src/olap/field.h
View File

@ -55,9 +55,9 @@ public:
inline size_t field_size() const { return size() + 1; }
inline size_t index_size() const { return _index_size; }
inline void set_to_max(char* buf) const { return _type_info->set_to_max(buf); }
virtual inline void set_to_max(char* buf) const { return _type_info->set_to_max(buf); }
inline void set_to_min(char* buf) const { return _type_info->set_to_min(buf); }
inline char* allocate_value_from_arena(Arena* arena) const { return _type_info->allocate_value_from_arena(arena); }
virtual inline char* allocate_value_from_arena(Arena* arena) const { return arena->Allocate(_type_info->size()); }
inline void agg_update(RowCursorCell* dest, const RowCursorCell& src, MemPool* mem_pool = nullptr) const {
_agg_info->update(dest, src, mem_pool);
@ -200,10 +200,6 @@ public:
_type_info->deep_copy_with_arena(dest, src, arena);
}
inline void direct_copy_content(char* dest, const char* src) const {
_type_info->direct_copy(dest, src);
}
// Copy srouce content to destination in index format.
template<typename DstCellType, typename SrcCellType>
void to_index(DstCellType* dst, const SrcCellType& src) const;
@ -259,6 +255,14 @@ protected:
// 长度，单位为字节
// 除字符串外，其它类型都是确定的
uint32_t _length;
char* allocate_string_value_from_arena(Arena* arena) const {
char* type_value = arena->Allocate(sizeof(Slice));
auto slice = reinterpret_cast<Slice*>(type_value);
slice->size = _length;
slice->data = arena->Allocate(slice->size);
return type_value;
};
};
template<typename LhsCellType, typename RhsCellType>
@ -378,6 +382,16 @@ public:
CharField* clone() const override {
return new CharField(*this);
}
char* allocate_value_from_arena(Arena* arena) const override {
return Field::allocate_string_value_from_arena(arena);
}
void set_to_max(char* ch) const override {
auto slice = reinterpret_cast<Slice*>(ch);
slice->size = _length;
memset(slice->data, 0xFF, slice->size);
}
};
class VarcharField: public Field {
@ -389,6 +403,7 @@ public:
return _length - OLAP_STRING_MAX_BYTES;
}
// minus OLAP_STRING_MAX_BYTES here just for being compatible with old storage format
char* allocate_memory(char* cell_ptr, char* variable_ptr) const override {
auto slice = (Slice*)cell_ptr;
slice->data = variable_ptr;
@ -400,6 +415,16 @@ public:
VarcharField* clone() const override {
return new VarcharField(*this);
}
char* allocate_value_from_arena(Arena* arena) const override {
return Field::allocate_string_value_from_arena(arena);
}
void set_to_max(char* ch) const override {
auto slice = reinterpret_cast<Slice*>(ch);
slice->size = _length - OLAP_STRING_MAX_BYTES;
memset(slice->data, 0xFF, slice->size);
}
};
class BitmapAggField: public Field {

3
be/src/olap/olap_define.h
View File

@ -53,9 +53,6 @@ static constexpr uint32_t OLAP_COMPACTION_DEFAULT_CANDIDATE_SIZE = 10;
// the max length supported for varchar type
static const uint16_t OLAP_STRING_MAX_LENGTH = 65535;
//the max length supported for char type
static const uint16_t OLAP_CHAR_MAX_LENGTH = 255;
static const int32_t PREFERRED_SNAPSHOT_VERSION = 3;
// the max bytes for stored string length

16
be/src/olap/rowset/segment_v2/column_writer.cpp
View File

@ -72,13 +72,13 @@ private:
};
ColumnWriter::ColumnWriter(const ColumnWriterOptions& opts,
const TypeInfo* typeinfo,
std::unique_ptr<Field> field,
bool is_nullable,
WritableFile* output_file)
: _opts(opts),
_type_info(typeinfo),
_is_nullable(is_nullable),
_output_file(output_file) {
_output_file(output_file),
_field(std::move(field)) {
}
ColumnWriter::~ColumnWriter() {
@ -92,7 +92,7 @@ ColumnWriter::~ColumnWriter() {
}
Status ColumnWriter::init() {
RETURN_IF_ERROR(EncodingInfo::get(_type_info, _opts.encoding_type, &_encoding_info));
RETURN_IF_ERROR(EncodingInfo::get(_field->type_info(), _opts.encoding_type, &_encoding_info));
if (_opts.compression_type != NO_COMPRESSION) {
RETURN_IF_ERROR(get_block_compression_codec(_opts.compression_type, &_compress_codec));
}
@ -105,7 +105,7 @@ Status ColumnWriter::init() {
if (page_builder == nullptr) {
return Status::NotSupported(
Substitute("Failed to create page builder for type $0 and encoding $1",
_type_info->type(), _opts.encoding_type));
_field->type(), _opts.encoding_type));
}
_page_builder.reset(page_builder);
// create ordinal builder
@ -115,7 +115,7 @@ Status ColumnWriter::init() {
_null_bitmap_builder.reset(new NullBitmapBuilder());
}
if (_opts.need_zone_map) {
_column_zone_map_builder.reset(new ColumnZoneMapBuilder(_type_info));
_column_zone_map_builder.reset(new ColumnZoneMapBuilder(_field.get()));
}
return Status::OK();
}
@ -148,7 +148,7 @@ Status ColumnWriter::_append_data(const uint8_t** ptr, size_t num_rows) {
bool is_page_full = (num_written < remaining);
remaining -= num_written;
_next_rowid += num_written;
*ptr += _type_info->size() * num_written;
*ptr += _field->size() * num_written;
// we must write null bits after write data, because we don't
// know how many rows can be written into current page
if (_is_nullable) {
@ -240,7 +240,7 @@ Status ColumnWriter::write_zone_map() {
}
void ColumnWriter::write_meta(ColumnMetaPB* meta) {
meta->set_type(_type_info->type());
meta->set_type(_field->type());
meta->set_encoding(_opts.encoding_type);
meta->set_compression(_opts.compression_type);
meta->set_is_nullable(_is_nullable);

4
be/src/olap/rowset/segment_v2/column_writer.h
View File

@ -57,7 +57,7 @@ class PageBuilder;
class ColumnWriter {
public:
ColumnWriter(const ColumnWriterOptions& opts,
const TypeInfo* typeinfo,
std::unique_ptr<Field> field,
bool is_nullable,
WritableFile* output_file);
~ColumnWriter();
@ -138,7 +138,6 @@ private:
private:
ColumnWriterOptions _opts;
const TypeInfo* _type_info = nullptr;
bool _is_nullable;
WritableFile* _output_file = nullptr;
@ -154,6 +153,7 @@ private:
std::unique_ptr<NullBitmapBuilder> _null_bitmap_builder;
std::unique_ptr<OrdinalPageIndexBuilder> _ordinal_index_builder;
std::unique_ptr<ColumnZoneMapBuilder> _column_zone_map_builder;
std::unique_ptr<Field> _field;
PagePointer _ordinal_index_pp;
PagePointer _zone_map_pp;

9
be/src/olap/rowset/segment_v2/column_zone_map.cpp
View File

@ -23,11 +23,10 @@ namespace doris {
namespace segment_v2 {
ColumnZoneMapBuilder::ColumnZoneMapBuilder(const TypeInfo* type_info) : _type_info(type_info) {
ColumnZoneMapBuilder::ColumnZoneMapBuilder(Field* field) : _field(field) {
PageBuilderOptions options;
options.data_page_size = 0;
_page_builder.reset(new BinaryPlainPageBuilder(options));
_field.reset(FieldFactory::create_by_type(_type_info->type()));
_zone_map.min_value = _field->allocate_value_from_arena(&_arena);
_zone_map.max_value = _field->allocate_value_from_arena(&_arena);
@ -38,12 +37,12 @@ Status ColumnZoneMapBuilder::add(const uint8_t *vals, size_t count) {
if (vals != nullptr) {
for (int i = 0; i < count; ++i) {
if (_field->compare(_zone_map.min_value, (char *)vals) > 0) {
_field->direct_copy_content(_zone_map.min_value, (const char *)vals);
_field->type_info()->direct_copy(_zone_map.min_value, (const char *)vals);
}
if (_field->compare(_zone_map.max_value, (char *)vals) < 0) {
_field->direct_copy_content(_zone_map.max_value, (const char *)vals);
_field->type_info()->direct_copy(_zone_map.max_value, (const char *)vals);
}
vals += _type_info->size();
vals += _field->size();
if (!_zone_map.has_not_null) {
_zone_map.has_not_null = true;
}

5
be/src/olap/rowset/segment_v2/column_zone_map.h
View File

@ -50,7 +50,7 @@ struct ZoneMap {
// The binary is encoded by BinaryPlainPageBuilder
class ColumnZoneMapBuilder {
public:
ColumnZoneMapBuilder(const TypeInfo* type_info);
ColumnZoneMapBuilder(Field* field);
Status add(const uint8_t* vals, size_t count);
@ -68,9 +68,8 @@ private:
void _reset_zone_map();
private:
const TypeInfo* _type_info;
std::unique_ptr<BinaryPlainPageBuilder> _page_builder;
std::unique_ptr<Field> _field;
Field* _field;
// memory will be managed by arena
ZoneMap _zone_map;
Arena _arena;

10
be/src/olap/rowset/segment_v2/segment_writer.cpp
View File

@ -55,17 +55,17 @@ Status SegmentWriter::init(uint32_t write_mbytes_per_sec) {
bool is_nullable = column.is_nullable();
column_meta->set_is_nullable(is_nullable);
// TODO(zc): we can add type_info into TabletColumn?
const TypeInfo* type_info = get_type_info(column.type());
DCHECK(type_info != nullptr);
ColumnWriterOptions opts;
opts.compression_type = segment_v2::CompressionTypePB::LZ4F;
// now we create zone map for key columns
if (column.is_key()) {
opts.need_zone_map = true;
}
std::unique_ptr<ColumnWriter> writer(new ColumnWriter(opts, type_info, is_nullable, _output_file.get()));
std::unique_ptr<Field> field(FieldFactory::create(column));
DCHECK(field.get() != nullptr);
std::unique_ptr<ColumnWriter> writer(new ColumnWriter(opts, std::move(field), is_nullable, _output_file.get()));
RETURN_IF_ERROR(writer->init());
_column_writers.push_back(std::move(writer));
}

3
be/src/olap/types.cpp
View File

@ -19,6 +19,8 @@
namespace doris {
void (*FieldTypeTraits<OLAP_FIELD_TYPE_CHAR>::set_to_max)(void*) = nullptr;
template<typename TypeTraitsClass>
TypeInfo::TypeInfo(TypeTraitsClass t)
: _equal(TypeTraitsClass::equal),
@ -27,7 +29,6 @@ TypeInfo::TypeInfo(TypeTraitsClass t)
_deep_copy(TypeTraitsClass::deep_copy),
_deep_copy_with_arena(TypeTraitsClass::deep_copy_with_arena),
_direct_copy(TypeTraitsClass::direct_copy),
_allocate_value_from_arena(TypeTraitsClass::allocate_value_from_arena),
_from_string(TypeTraitsClass::from_string),
_to_string(TypeTraitsClass::to_string),
_set_to_max(TypeTraitsClass::set_to_max),

39
be/src/olap/types.h
View File

@ -64,10 +64,6 @@ public:
_direct_copy(dest, src);
}
inline char* allocate_value_from_arena(Arena* arena) const {
return _allocate_value_from_arena(arena);
}
OLAPStatus from_string(void* buf, const std::string& scan_key) const {
return _from_string(buf, scan_key);
}
@ -89,7 +85,6 @@ private:
void (*_deep_copy)(void* dest, const void* src, MemPool* mem_pool);
void (*_deep_copy_with_arena)(void* dest, const void* src, Arena* arena);
void (*_direct_copy)(void* dest, const void* src);
char* (*_allocate_value_from_arena)(Arena* arena);
OLAPStatus (*_from_string)(void* buf, const std::string& scan_key);
std::string (*_to_string)(const void* src);
@ -218,10 +213,6 @@ struct BaseFieldtypeTraits : public CppTypeTraits<field_type> {
return HashUtil::hash(data, sizeof(CppType), seed);
}
static inline char* allocate_value_from_arena(Arena* arena) {
return arena->Allocate(sizeof(CppType));
}
static std::string to_string(const void* src) {
std::stringstream stream;
stream << *reinterpret_cast<const CppType*>(src);
@ -563,12 +554,10 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_CHAR> : public BaseFieldtypeTraits<OLAP_F
memory_copy(l_slice->data, r_slice->data, r_slice->size);
l_slice->size = r_slice->size;
}
static void set_to_max(void* buf) {
// this function is used by scan key,
// the size may be greater than length in schema.
auto slice = reinterpret_cast<Slice*>(buf);
memset(slice->data, 0xff, slice->size);
}
// using field.set_to_max to set varchar/char,not here
static void (*set_to_max)(void*);
static void set_to_min(void* buf) {
auto slice = reinterpret_cast<Slice*>(buf);
memset(slice->data, 0, slice->size);
@ -577,13 +566,6 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_CHAR> : public BaseFieldtypeTraits<OLAP_F
auto slice = reinterpret_cast<const Slice*>(data);
return HashUtil::hash(slice->data, slice->size, seed);
}
static char* allocate_value_from_arena(Arena* arena) {
char* type_value = arena->Allocate(sizeof(Slice));
auto slice = reinterpret_cast<Slice*>(type_value);
slice->size = OLAP_CHAR_MAX_LENGTH;
slice->data = arena->Allocate(OLAP_CHAR_MAX_LENGTH);
return type_value;
}
};
template<>
@ -601,22 +583,11 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_VARCHAR> : public FieldTypeTraits<OLAP_FI
slice->size = value_len;
return OLAP_SUCCESS;
}
static void set_to_max(void* buf) {
auto slice = reinterpret_cast<Slice*>(buf);
slice->size = 1;
memset(slice->data, 0xFF, 1);
}
static void set_to_min(void* buf) {
auto slice = reinterpret_cast<Slice*>(buf);
slice->size = 0;
}
static char* allocate_value_from_arena(Arena* arena) {
char* type_value = arena->Allocate(sizeof(Slice));
auto slice = reinterpret_cast<Slice*>(type_value);
slice->size = OLAP_STRING_MAX_LENGTH;
slice->data = arena->Allocate(OLAP_STRING_MAX_LENGTH);
return type_value;
}
};
template<>

39
be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
View File

@ -17,6 +17,7 @@
#include "olap/rowset/segment_v2/column_reader.h"
#include "olap/rowset/segment_v2/column_writer.h"
#include "olap/tablet_schema_helper.h"
#include "olap/decimal12.h"
#include <gtest/gtest.h>
@ -62,7 +63,14 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s
writer_opts.compression_type = segment_v2::CompressionTypePB::LZ4F;
writer_opts.need_zone_map = true;
ColumnWriter writer(writer_opts, type_info, true, wfile.get());
TabletColumn column(OLAP_FIELD_AGGREGATION_NONE, type);
if (type == OLAP_FIELD_TYPE_VARCHAR) {
column = create_varchar_key(1);
} else if (type == OLAP_FIELD_TYPE_CHAR) {
column = create_char_key(1);
}
std::unique_ptr<Field> field(FieldFactory::create(column));
ColumnWriter writer(writer_opts, std::move(field), true, wfile.get());
st = writer.init();
ASSERT_TRUE(st.ok());
@ -113,6 +121,7 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s
Arena arena;
Type vals[1024];
Type* vals_ = vals;
uint8_t is_null[1024];
ColumnBlock col(type_info, (uint8_t*)vals, is_null, &arena);
@ -126,7 +135,13 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s
// << ", src[idx]=" << src[idx] << ", vals[j]=" << vals[j];
ASSERT_EQ(BitmapTest(src_is_null, idx), BitmapTest(is_null, j));
if (!BitmapTest(is_null, j)) {
ASSERT_EQ(src[idx], vals[j]);
if (type == OLAP_FIELD_TYPE_VARCHAR || type == OLAP_FIELD_TYPE_CHAR) {
Slice* src_slice = (Slice*)src_data;
Slice* dst_slice = (Slice*)vals_;
ASSERT_EQ(src_slice[idx].to_string(), dst_slice[j].to_string()) << "j:" << j;
} else {
ASSERT_EQ(src[idx], vals[j]);
}
}
idx++;
}
@ -155,7 +170,13 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, s
// << ", src[idx]=" << src[idx] << ", vals[j]=" << vals[j];
ASSERT_EQ(BitmapTest(src_is_null, idx), BitmapTest(is_null, j));
if (!BitmapTest(is_null, j)) {
ASSERT_EQ(src[idx], vals[j]);
if (type == OLAP_FIELD_TYPE_VARCHAR || type == OLAP_FIELD_TYPE_CHAR) {
Slice* src_slice = (Slice*)src_data;
Slice* dst_slice = (Slice*)vals;
ASSERT_EQ(src_slice[idx].to_string(), dst_slice[j].to_string());
} else {
ASSERT_EQ(src[idx], vals[j]);
}
}
idx++;
}
@ -203,6 +224,8 @@ TEST_F(ColumnReaderWriterTest, test_nullable) {
}
TEST_F(ColumnReaderWriterTest, test_types) {
Arena arena;
size_t num_uint8_rows = 1024 * 1024;
uint8_t* is_null = new uint8_t[num_uint8_rows];
@ -210,13 +233,21 @@ TEST_F(ColumnReaderWriterTest, test_types) {
uint24_t* date_vals = new uint24_t[num_uint8_rows];
uint64_t* datetime_vals = new uint64_t[num_uint8_rows];
decimal12_t* decimal_vals = new decimal12_t[num_uint8_rows];
Slice* varchar_vals = new Slice[num_uint8_rows];
Slice* char_vals = new Slice[num_uint8_rows];
for (int i = 0; i < num_uint8_rows; ++i) {
bool_vals[i] = i % 2;
date_vals[i] = i + 33;
datetime_vals[i] = i + 33;
decimal_vals[i] = decimal12_t(i, i); // 1.000000001
set_column_value_by_type(OLAP_FIELD_TYPE_VARCHAR, i, (char*)&varchar_vals[i], &arena);
set_column_value_by_type(OLAP_FIELD_TYPE_CHAR, i, (char*)&char_vals[i], &arena, 8);
BitmapChange(is_null, i, (i % 4) == 0);
}
test_nullable_data<OLAP_FIELD_TYPE_CHAR, DICT_ENCODING>((uint8_t*)char_vals, is_null, num_uint8_rows, "null_char_bs");
test_nullable_data<OLAP_FIELD_TYPE_VARCHAR, DICT_ENCODING>((uint8_t*)varchar_vals, is_null, num_uint8_rows, "null_varchar_bs");
test_nullable_data<OLAP_FIELD_TYPE_BOOL, BIT_SHUFFLE>((uint8_t*)bool_vals, is_null, num_uint8_rows, "null_bool_bs");
test_nullable_data<OLAP_FIELD_TYPE_DATE, BIT_SHUFFLE>((uint8_t*)date_vals, is_null, num_uint8_rows / 3, "null_date_bs");
@ -230,6 +261,8 @@ TEST_F(ColumnReaderWriterTest, test_types) {
}
test_nullable_data<OLAP_FIELD_TYPE_DECIMAL, BIT_SHUFFLE>((uint8_t*)decimal_vals, is_null, num_uint8_rows / 12, "null_decimal_bs");
delete[] char_vals;
delete[] varchar_vals;
delete[] is_null;
delete[] bool_vals;
delete[] date_vals;

20
be/test/olap/rowset/segment_v2/column_zone_map_test.cpp
View File

@ -19,15 +19,15 @@
#include <memory>
#include "olap/rowset/segment_v2/column_zone_map.h"
#include "olap/tablet_schema_helper.h"
namespace doris {
namespace segment_v2 {
class ColumnZoneMapTest : public testing::Test {
public:
void test_string(FieldType type) {
TypeInfo *type_info = get_type_info(OLAP_FIELD_TYPE_CHAR);
ColumnZoneMapBuilder builder(type_info);
void test_string(Field* field) {
ColumnZoneMapBuilder builder(field);
std::vector<std::string> values1 = {"aaaa", "bbbb", "cccc", "dddd", "eeee", "ffff"};
for (auto value : values1) {
builder.add((const uint8_t*)&value, 1);
@ -67,8 +67,10 @@ public:
// Test for int
TEST_F(ColumnZoneMapTest, NormalTestIntPage) {
TypeInfo* type_info = get_type_info(OLAP_FIELD_TYPE_INT);
ColumnZoneMapBuilder builder(type_info);
TabletColumn int_column = create_int_key(0);
Field* field = FieldFactory::create(int_column);
ColumnZoneMapBuilder builder(field);
std::vector<int> values1 = {1, 10, 11, 20, 21, 22};
for (auto value : values1) {
builder.add((const uint8_t*)&value, 1);
@ -108,12 +110,16 @@ TEST_F(ColumnZoneMapTest, NormalTestIntPage) {
// Test for string
TEST_F(ColumnZoneMapTest, NormalTestVarcharPage) {
test_string(OLAP_FIELD_TYPE_VARCHAR);
TabletColumn varchar_column = create_varchar_key(0);
Field* field = FieldFactory::create(varchar_column);
test_string(field);
}
// Test for string
TEST_F(ColumnZoneMapTest, NormalTestCharPage) {
test_string(OLAP_FIELD_TYPE_CHAR);
TabletColumn char_column = create_char_key(0);
Field* field = FieldFactory::create(char_column);
test_string(field);
}
}

23
be/test/olap/rowset/segment_v2/segment_test.cpp
View File

@ -606,29 +606,6 @@ TEST_F(SegmentReaderWriterTest, TestDefaultValueColumn) {
}
}
void set_column_value_by_type(FieldType fieldType, int src, char* target, Arena* _arena, size_t _length = 0) {
if (fieldType == OLAP_FIELD_TYPE_CHAR) {
char* src_value = &std::to_string(src)[0];
int src_len = strlen(src_value);
auto* dest_slice = (Slice*)target;
dest_slice->size = _length;
dest_slice->data = _arena->Allocate(dest_slice->size);
memcpy(dest_slice->data, src_value, src_len);
memset(dest_slice->data + src_len, 0, dest_slice->size - src_len);
} else if (fieldType == OLAP_FIELD_TYPE_VARCHAR) {
char* src_value = &std::to_string(src)[0];
int src_len = strlen(src_value);
auto* dest_slice = (Slice*)target;
dest_slice->size = src_len;
dest_slice->data = _arena->Allocate(src_len);
std::memcpy(dest_slice->data, src_value, src_len);
} else {
*(int*)target = src;
}
}
TEST_F(SegmentReaderWriterTest, TestStringDict) {
size_t num_rows_per_block = 10;
Arena _arena;

18
be/test/olap/storage_types_test.cpp
View File

@ -22,6 +22,7 @@
#include "runtime/mem_tracker.h"
#include "runtime/mem_pool.h"
#include "util/slice.h"
#include "olap/field.h"
namespace doris {
@ -70,10 +71,13 @@ void common_test(typename TypeTraits<field_type>::CppType src_val) {
}
}
void test_char(FieldType field_type, Slice src_val) {
TypeInfo* type = get_type_info(field_type);
template<FieldType fieldType>
void test_char(Slice src_val) {
Field* field = FieldFactory::create_by_type(fieldType);
field->_length = src_val.size;
const TypeInfo* type = field->type_info();
ASSERT_EQ(field_type, type->type());
ASSERT_EQ(field->type(), fieldType);
ASSERT_EQ(sizeof(src_val), type->size());
{
char buf[64];
@ -95,7 +99,7 @@ void test_char(FieldType field_type, Slice src_val) {
{
char buf[64];
Slice dst_val(buf, sizeof(buf));
type->set_to_min((char*)&dst_val);
field->set_to_min((char*)&dst_val);
ASSERT_FALSE(type->equal((char*)&src_val, (char*)&dst_val));
ASSERT_TRUE(type->cmp((char*)&src_val, (char*)&dst_val) > 0);
@ -104,7 +108,7 @@ void test_char(FieldType field_type, Slice src_val) {
{
char buf[64];
Slice dst_val(buf, sizeof(buf));
type->set_to_max((char*)&dst_val);
field->set_to_max((char*)&dst_val);
ASSERT_FALSE(type->equal((char*)&src_val, (char*)&dst_val));
ASSERT_TRUE(type->cmp((char*)&src_val, (char*)&dst_val) < 0);
@ -113,12 +117,12 @@ void test_char(FieldType field_type, Slice src_val) {
template<>
void common_test<OLAP_FIELD_TYPE_CHAR>(Slice src_val) {
test_char(OLAP_FIELD_TYPE_CHAR, src_val);
test_char<OLAP_FIELD_TYPE_VARCHAR>(src_val);
}
template<>
void common_test<OLAP_FIELD_TYPE_VARCHAR>(Slice src_val) {
test_char(OLAP_FIELD_TYPE_VARCHAR, src_val);
test_char<OLAP_FIELD_TYPE_VARCHAR>(src_val);
}
TEST(TypesTest, copy_and_equal) {

25
be/test/olap/tablet_schema_helper.h
View File

@ -74,9 +74,32 @@ TabletColumn create_varchar_key(int32_t id, bool is_nullable = true) {
column._type = OLAP_FIELD_TYPE_VARCHAR;
column._is_key = true;
column._is_nullable = is_nullable;
column._length = 4;
column._length = 8;
column._index_length = 4;
return column;
}
void set_column_value_by_type(FieldType fieldType, int src, char* target, Arena* _arena, size_t _length = 0) {
if (fieldType == OLAP_FIELD_TYPE_CHAR) {
char* src_value = &std::to_string(src)[0];
int src_len = strlen(src_value);
auto* dest_slice = (Slice*)target;
dest_slice->size = _length;
dest_slice->data = _arena->Allocate(dest_slice->size);
memcpy(dest_slice->data, src_value, src_len);
memset(dest_slice->data + src_len, 0, dest_slice->size - src_len);
} else if (fieldType == OLAP_FIELD_TYPE_VARCHAR) {
char* src_value = &std::to_string(src)[0];
int src_len = strlen(src_value);
auto* dest_slice = (Slice*)target;
dest_slice->size = src_len;
dest_slice->data = _arena->Allocate(src_len);
std::memcpy(dest_slice->data, src_value, src_len);
} else {
*(int*)target = src;
}
}
}