From dd36ccc3bf2f04d01cb6a4f5014fa605a412786a Mon Sep 17 00:00:00 2001 From: xinghuayu007 <1450306854@qq.com> Date: Thu, 2 Dec 2021 11:39:51 +0800 Subject: [PATCH] [feature](storage-format) Z-Order Implement (#7149) Support sort data by Z-Order: ``` CREATE TABLE table2 ( siteid int(11) NULL DEFAULT "10" COMMENT "", citycode int(11) NULL COMMENT "", username varchar(32) NULL DEFAULT "" COMMENT "", pv bigint(20) NULL DEFAULT "0" COMMENT "" ) ENGINE=OLAP DUPLICATE KEY(siteid, citycode) COMMENT "OLAP" DISTRIBUTED BY HASH(siteid) BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "data_sort.sort_type" = "ZORDER", "data_sort.col_num" = "2", "in_memory" = "false", "storage_format" = "V2" ); ``` --- be/src/olap/collect_iterator.cpp | 55 +- be/src/olap/collect_iterator.h | 35 +- be/src/olap/memtable.cpp | 13 +- be/src/olap/memtable.h | 10 +- .../olap/rowset/segment_v2/column_reader.cpp | 2 + .../rowset/segment_v2/segment_iterator.cpp | 5 +- be/src/olap/skiplist.h | 14 +- be/src/olap/tablet.h | 10 + be/src/olap/tablet_meta.cpp | 9 + be/src/olap/tablet_schema.cpp | 4 + be/src/olap/tablet_schema.h | 4 + be/src/util/CMakeLists.txt | 1 + be/src/util/bit_util.h | 116 ++ be/src/util/tuple_row_zorder_compare.cpp | 233 ++++ be/src/util/tuple_row_zorder_compare.h | 70 ++ be/test/olap/skiplist_test.cpp | 16 +- .../olap/test_data/header_without_inc_rs.txt | 4 +- be/test/util/CMakeLists.txt | 1 + .../util/tuple_row_zorder_compare_test.cpp | 1120 +++++++++++++++++ .../Create/CREATE-TABLE.md | 7 + .../Create/CREATE-TABLE.md | 6 + .../java/org/apache/doris/alter/Alter.java | 5 + .../apache/doris/analysis/DataSortInfo.java | 103 ++ .../org/apache/doris/catalog/Catalog.java | 46 +- .../org/apache/doris/catalog/OlapTable.java | 24 +- .../apache/doris/catalog/TableProperty.java | 26 +- .../doris/common/util/PropertyAnalyzer.java | 41 + .../apache/doris/task/CreateReplicaTask.java | 41 + .../apache/doris/catalog/CreateTableTest.java | 41 + gensrc/proto/olap_file.proto | 7 + gensrc/thrift/AgentService.thrift | 2 + gensrc/thrift/Types.thrift | 5 + 32 files changed, 2025 insertions(+), 51 deletions(-) create mode 100644 be/src/util/tuple_row_zorder_compare.cpp create mode 100644 be/src/util/tuple_row_zorder_compare.h create mode 100644 be/test/util/tuple_row_zorder_compare_test.cpp create mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/DataSortInfo.java diff --git a/be/src/olap/collect_iterator.cpp b/be/src/olap/collect_iterator.cpp index 3f4d8b5dc9..22446a03d1 100644 --- a/be/src/olap/collect_iterator.cpp +++ b/be/src/olap/collect_iterator.cpp @@ -56,6 +56,8 @@ OLAPStatus CollectIterator::add_child(RowsetReaderSharedPtr rs_reader) { void CollectIterator::build_heap(const std::vector& rs_readers) { DCHECK(rs_readers.size() == _children.size()); _reverse = _reader->_tablet->tablet_schema().keys_type() == KeysType::UNIQUE_KEYS; + SortType sort_type = _reader->_tablet->tablet_schema().sort_type(); + int sort_col_num = _reader->_tablet->tablet_schema().sort_col_num(); if (_children.empty()) { _inner_iter.reset(nullptr); return; @@ -86,18 +88,19 @@ void CollectIterator::build_heap(const std::vector& rs_re ++i; } Level1Iterator* cumu_iter = - new Level1Iterator(cumu_children, cumu_children.size() > 1, _reverse, _reader->_sequence_col_idx); + new Level1Iterator(cumu_children, cumu_children.size() > 1, _reverse, + _reader->_sequence_col_idx, sort_type, sort_col_num); cumu_iter->init(); _inner_iter.reset(new Level1Iterator(std::list{*base_reader_child, cumu_iter}, _merge, - _reverse, _reader->_sequence_col_idx)); + _reverse, _reader->_sequence_col_idx, sort_type, sort_col_num)); } else { // _children.size() == 1 _inner_iter.reset(new Level1Iterator(_children, _merge, - _reverse, _reader->_sequence_col_idx)); + _reverse, _reader->_sequence_col_idx, sort_type, sort_col_num)); } } else { _inner_iter.reset(new Level1Iterator(_children, _merge, - _reverse, _reader->_sequence_col_idx)); + _reverse, _reader->_sequence_col_idx, sort_type, sort_col_num)); } _inner_iter->init(); // Clear _children earlier to release any related references @@ -132,6 +135,34 @@ bool CollectIterator::LevelIteratorComparator::operator()(const LevelIterator* a return a->version() > b->version(); } +CollectIterator::BaseComparator::BaseComparator( + std::shared_ptr& cmp) { + _cmp = cmp; +} + +bool CollectIterator::BaseComparator::operator()(const LevelIterator* a, const LevelIterator* b) { + return _cmp->operator()(a, b); +} + +bool CollectIterator::LevelZorderIteratorComparator::operator()(const LevelIterator* a, + const LevelIterator* b) { + // First compare row cursor. + const RowCursor* first = a->current_row(); + const RowCursor* second = b->current_row(); + int cmp_res = _comparator.compare_row(*first, *second); + if (cmp_res != 0) { + return cmp_res > 0; + } + // if row cursors equal, compare data version. + // read data from higher version to lower version. + // for UNIQUE_KEYS just read the highest version and no need agg_update. + // for AGG_KEYS if a version is deleted, the lower version no need to agg_update + if (_reverse) { + return a->version() < b->version(); + } + return a->version() > b->version(); +} + const RowCursor* CollectIterator::current_row(bool* delete_flag) const { if (LIKELY(_inner_iter)) { return _inner_iter->current_row(delete_flag); @@ -233,8 +264,11 @@ OLAPStatus CollectIterator::Level0Iterator::next(const RowCursor** row, bool* de } CollectIterator::Level1Iterator::Level1Iterator( - std::list children, bool merge, bool reverse, int sequence_id_idx) - : _children(std::move(children)), _merge(merge), _reverse(reverse), _sequence_id_idx(sequence_id_idx) {} + const std::list& children, + bool merge, bool reverse, int sequence_id_idx, + SortType sort_type, int sort_col_num) + : _children(children), _merge(merge), _reverse(reverse), + _sort_type(sort_type), _sort_col_num(sort_col_num) {} CollectIterator::LevelIterator::~LevelIterator() = default; @@ -304,7 +338,14 @@ OLAPStatus CollectIterator::Level1Iterator::init() { // Only when there are multiple children that need to be merged if (_merge && _children.size() > 1) { - _heap.reset(new MergeHeap(LevelIteratorComparator(_reverse, _sequence_id_idx))); + std::shared_ptr cmp; + if (_sort_type == SortType::ZORDER) { + cmp = std::make_shared(_reverse, _sequence_id_idx, _sort_col_num); + } else { + cmp = std::make_shared(_reverse, _sequence_id_idx); + } + BaseComparator bcmp(cmp); + _heap.reset(new MergeHeap(bcmp)); for (auto child : _children) { DCHECK(child != nullptr); DCHECK(child->current_row() != nullptr); diff --git a/be/src/olap/collect_iterator.h b/be/src/olap/collect_iterator.h index 9965ae24d2..60b670efa9 100644 --- a/be/src/olap/collect_iterator.h +++ b/be/src/olap/collect_iterator.h @@ -20,6 +20,7 @@ #include "olap/olap_define.h" #include "olap/row_cursor.h" #include "olap/rowset/rowset_reader.h" +#include "util/tuple_row_zorder_compare.h" namespace doris { @@ -74,15 +75,39 @@ private: public: LevelIteratorComparator(const bool reverse = false, int sequence_id_idx = -1) : _reverse(reverse), _sequence_id_idx(sequence_id_idx) {} - bool operator()(const LevelIterator* a, const LevelIterator* b); + virtual bool operator()(const LevelIterator* a, const LevelIterator* b); private: bool _reverse; int _sequence_id_idx; }; + class LevelZorderIteratorComparator: public LevelIteratorComparator { + public: + LevelZorderIteratorComparator(const bool reverse = false, int sequence_id_idx = -1, const size_t sort_col_num = 0) : + _reverse(reverse), _sequence_id_idx(sequence_id_idx), _sort_col_num(sort_col_num) { + _comparator = TupleRowZOrderComparator(sort_col_num); + } + virtual bool operator()(const LevelIterator* a, const LevelIterator* b); + + private: + bool _reverse = false; + int _sequence_id_idx; + size_t _sort_col_num = 0; + TupleRowZOrderComparator _comparator; + }; + + class BaseComparator { + public: + BaseComparator(std::shared_ptr& cmp); + bool operator()(const LevelIterator* a, const LevelIterator* b); + + private: + std::shared_ptr _cmp; + }; + typedef std::priority_queue, - LevelIteratorComparator> + BaseComparator> MergeHeap; // Iterate from rowset reader. This Iterator usually like a leaf node class Level0Iterator : public LevelIterator { @@ -118,7 +143,9 @@ private: // Iterate from LevelIterators (maybe Level0Iterators or Level1Iterator or mixed) class Level1Iterator : public LevelIterator { public: - Level1Iterator(std::list, bool, bool, int); + + Level1Iterator(const std::list& children, bool merge, bool reverse, + int sequence_id_idx, SortType sort_type, int sort_col_num); OLAPStatus init() override; @@ -157,6 +184,8 @@ private: // used when `_merge == false` int _child_idx = 0; int _sequence_id_idx = -1; + SortType _sort_type; + int _sort_col_num; }; std::unique_ptr _inner_iter; diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 3ce5db1e49..2ec1ccbbc1 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -39,14 +39,19 @@ MemTable::MemTable(int64_t tablet_id, Schema* schema, const TabletSchema* tablet _tuple_desc(tuple_desc), _slot_descs(slot_descs), _keys_type(keys_type), - _row_comparator(_schema), _mem_tracker(MemTracker::CreateTracker(-1, "MemTable", parent_tracker)), _buffer_mem_pool(new MemPool(_mem_tracker.get())), _table_mem_pool(new MemPool(_mem_tracker.get())), _schema_size(_schema->schema_size()), - _skip_list(new Table(_row_comparator, _table_mem_pool.get(), - _keys_type == KeysType::DUP_KEYS)), - _rowset_writer(rowset_writer) {} + _rowset_writer(rowset_writer) { + if (tablet_schema->sort_type() == SortType::ZORDER) { + _row_comparator = std::make_shared(_schema, tablet_schema->sort_col_num()); + } else { + _row_comparator = std::make_shared(_schema); + } + _skip_list = new Table(_row_comparator.get(), _table_mem_pool.get(), _keys_type == KeysType::DUP_KEYS); + +} MemTable::~MemTable() { delete _skip_list; diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h index 02783d715b..fdb574cb1e 100644 --- a/be/src/olap/memtable.h +++ b/be/src/olap/memtable.h @@ -24,6 +24,7 @@ #include "olap/olap_define.h" #include "olap/skiplist.h" #include "runtime/mem_tracker.h" +#include "util/tuple_row_zorder_compare.h" namespace doris { @@ -53,16 +54,17 @@ public: int64_t flush_size() const { return _flush_size; } private: - class RowCursorComparator { + class RowCursorComparator: public RowComparator { public: RowCursorComparator(const Schema* schema); - int operator()(const char* left, const char* right) const; + virtual int operator()(const char* left, const char* right) const; private: const Schema* _schema; }; - typedef SkipList Table; +private: + typedef SkipList Table; typedef Table::key_type TableKey; public: @@ -95,7 +97,7 @@ private: const std::vector* _slot_descs; KeysType _keys_type; - RowCursorComparator _row_comparator; + std::shared_ptr _row_comparator; std::shared_ptr _mem_tracker; // This is a buffer, to hold the memory referenced by the rows that have not // been inserted into the SkipList diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 7ac50a7b34..3d78cebd20 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -239,6 +239,8 @@ Status ColumnReader::_get_filtered_pages(CondColumn* cond_column, CondColumn* de } } } + VLOG(1) << "total-pages: " << page_size << " not-filtered-pages: " << page_indexes->size() + << " filtered-percent:" << 1.0 - (page_indexes->size()*1.0)/(page_size*1.0); return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 24b7d74261..6406d95b46 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -128,7 +128,10 @@ Status SegmentIterator::_init() { _row_bitmap.addRange(0, _segment->num_rows()); RETURN_IF_ERROR(_init_return_column_iterators()); RETURN_IF_ERROR(_init_bitmap_index_iterators()); - RETURN_IF_ERROR(_get_row_ranges_by_keys()); + // z-order can not use prefix index + if (_segment->_tablet_schema->sort_type() != SortType::ZORDER) { + RETURN_IF_ERROR(_get_row_ranges_by_keys()); + } RETURN_IF_ERROR(_get_row_ranges_by_column_conditions()); _init_lazy_materialization(); _range_iter.reset(new BitmapRangeIterator(_row_bitmap)); diff --git a/be/src/olap/skiplist.h b/be/src/olap/skiplist.h index f2676de561..8d392b4905 100644 --- a/be/src/olap/skiplist.h +++ b/be/src/olap/skiplist.h @@ -67,7 +67,7 @@ public: // and will allocate memory using "*mem_pool". // NOTE: Objects allocated in the mem_pool must remain allocated for // the lifetime of the skiplist object. - explicit SkipList(Comparator cmp, MemPool* mem_pool, bool can_dup); + explicit SkipList(Comparator* cmp, MemPool* mem_pool, bool can_dup); // Insert key into the list. void Insert(const Key& key, bool* overwritten); @@ -121,7 +121,7 @@ public: private: // Immutable after construction - Comparator const compare_; + Comparator* const compare_; // When value is true, means indicates that duplicate values are allowed. bool _can_dup; MemPool* const _mem_pool; // MemPool used for allocations of nodes @@ -139,7 +139,7 @@ private: Node* NewNode(const Key& key, int height); int RandomHeight(); - bool Equal(const Key& a, const Key& b) const { return (compare_(a, b) == 0); } + bool Equal(const Key& a, const Key& b) const { return ((*compare_)(a, b) == 0); } // Return true if key is greater than the data stored in "n" bool KeyIsAfterNode(const Key& key, Node* n) const; @@ -277,7 +277,7 @@ int SkipList::RandomHeight() { template bool SkipList::KeyIsAfterNode(const Key& key, Node* n) const { // nullptr n is considered infinite - return (n != nullptr) && (compare_(n->key, key) < 0); + return (n != nullptr) && ((*compare_)(n->key, key) < 0); } template @@ -308,9 +308,9 @@ typename SkipList::Node* SkipList::FindLessTha Node* x = head_; int level = GetMaxHeight() - 1; while (true) { - DCHECK(x == head_ || compare_(x->key, key) < 0); + DCHECK(x == head_ || (*compare_)(x->key, key) < 0); Node* next = x->Next(level); - if (next == nullptr || compare_(next->key, key) >= 0) { + if (next == nullptr || (*compare_)(next->key, key) >= 0) { if (level == 0) { return x; } else { @@ -343,7 +343,7 @@ typename SkipList::Node* SkipList::FindLast() } template -SkipList::SkipList(Comparator cmp, MemPool* mem_pool, bool can_dup) +SkipList::SkipList(Comparator* cmp, MemPool* mem_pool, bool can_dup) : compare_(cmp), _can_dup(can_dup), _mem_pool(mem_pool), diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 437c13662c..c93c3b285e 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -82,6 +82,8 @@ public: // properties encapsulated in TabletSchema inline KeysType keys_type() const; + inline SortType sort_type() const; + inline size_t sort_col_num() const; inline size_t num_columns() const; inline size_t num_null_columns() const; inline size_t num_key_columns() const; @@ -394,6 +396,14 @@ inline KeysType Tablet::keys_type() const { return _schema.keys_type(); } +inline SortType Tablet::sort_type() const { + return _schema.sort_type(); +} + +inline size_t Tablet::sort_col_num() const { + return _schema.sort_col_num(); +} + inline size_t Tablet::num_columns() const { return _schema.num_columns(); } diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 71794edb11..8739486133 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -86,6 +86,15 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id break; } schema->set_compress_kind(COMPRESS_LZ4); + + switch(tablet_schema.sort_type) { + case TSortType::type::ZORDER: + schema->set_sort_type(SortType::ZORDER); + break; + default: + schema->set_sort_type(SortType::LEXICAL); + } + schema->set_sort_col_num(tablet_schema.sort_col_num); tablet_meta_pb.set_in_restore_mode(false); // set column information diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index dd29b73002..072a4155a1 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -422,6 +422,8 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema) { _is_in_memory = schema.is_in_memory(); _delete_sign_idx = schema.delete_sign_idx(); _sequence_col_idx = schema.sequence_col_idx(); + _sort_type = schema.sort_type(); + _sort_col_num = schema.sort_col_num(); } void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_meta_pb) { @@ -440,6 +442,8 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_meta_pb) { tablet_meta_pb->set_is_in_memory(_is_in_memory); tablet_meta_pb->set_delete_sign_idx(_delete_sign_idx); tablet_meta_pb->set_sequence_col_idx(_sequence_col_idx); + tablet_meta_pb->set_sort_type(_sort_type); + tablet_meta_pb->set_sort_col_num(_sort_col_num); } uint32_t TabletSchema::mem_size() const { diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index ee2f9207fc..0cb894fdfb 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -130,6 +130,8 @@ public: inline size_t num_short_key_columns() const { return _num_short_key_columns; } inline size_t num_rows_per_row_block() const { return _num_rows_per_row_block; } inline KeysType keys_type() const { return _keys_type; } + inline SortType sort_type() const { return _sort_type; } + inline size_t sort_col_num() const { return _sort_col_num; } inline CompressKind compress_kind() const { return _compress_kind; } inline size_t next_column_unique_id() const { return _next_column_unique_id; } inline double bloom_filter_fpp() const { return _bf_fpp; } @@ -149,6 +151,8 @@ private: private: KeysType _keys_type = DUP_KEYS; + SortType _sort_type = SortType::LEXICAL; + size_t _sort_col_num = 0; std::vector _cols; std::unordered_map _field_name_to_index; size_t _num_columns = 0; diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt index 1fe088add1..985b78f239 100644 --- a/be/src/util/CMakeLists.txt +++ b/be/src/util/CMakeLists.txt @@ -106,6 +106,7 @@ set(UTIL_FILES s3_storage_backend.cpp s3_util.cpp topn_counter.cpp + tuple_row_zorder_compare.cpp ) if (WITH_MYSQL) diff --git a/be/src/util/bit_util.h b/be/src/util/bit_util.h index a4bf2ef6eb..549ce608fa 100644 --- a/be/src/util/bit_util.h +++ b/be/src/util/bit_util.h @@ -24,6 +24,12 @@ #include "gutil/bits.h" #include "gutil/port.h" #include "util/cpu_info.h" +#ifdef __aarch64__ +#include "sse2neon.h" +#else +#include +#include +#endif namespace doris { @@ -328,6 +334,116 @@ public: if (PREDICT_FALSE(num_bits >= 64)) return 0; return v >> num_bits; } + + static void ByteSwapScalar(void *dest, const void *source, int len) { + uint8_t *dst = reinterpret_cast(dest); + const uint8_t *src = reinterpret_cast(source); + switch (len) { + case 1: + *reinterpret_cast(dst) = *reinterpret_cast(src); + return; + case 2: + *reinterpret_cast(dst) = + BitUtil::byte_swap(*reinterpret_cast(src)); + return; + case 3: + *reinterpret_cast(dst + 1) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst) = *reinterpret_cast(src + 2); + return; + case 4: + *reinterpret_cast(dst) = + BitUtil::byte_swap(*reinterpret_cast(src)); + return; + case 5: + *reinterpret_cast(dst + 1) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst) = *reinterpret_cast(src + 4); + return; + case 6: + *reinterpret_cast(dst + 2) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst) = + BitUtil::byte_swap(*reinterpret_cast(src + 4)); + return; + case 7: + *reinterpret_cast(dst + 3) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst + 1) = + BitUtil::byte_swap(*reinterpret_cast(src + 4)); + *reinterpret_cast(dst) = *reinterpret_cast(src + 6); + return; + case 8: + *reinterpret_cast(dst) = + BitUtil::byte_swap(*reinterpret_cast(src)); + return; + case 9: + *reinterpret_cast(dst + 1) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst) = *reinterpret_cast(src + 8); + return; + case 10: + *reinterpret_cast(dst + 2) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst) = + BitUtil::byte_swap(*reinterpret_cast(src + 8)); + return; + case 11: + *reinterpret_cast(dst + 3) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst + 1) = + BitUtil::byte_swap(*reinterpret_cast(src + 8)); + *reinterpret_cast(dst) = *reinterpret_cast(src + 10); + return; + case 12: + *reinterpret_cast(dst + 4) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst) = + BitUtil::byte_swap(*reinterpret_cast(src + 8)); + return; + case 13: + *reinterpret_cast(dst + 5) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst + 1) = + BitUtil::byte_swap(*reinterpret_cast(src + 8)); + *reinterpret_cast(dst) = *reinterpret_cast(src + 12); + return; + case 14: + *reinterpret_cast(dst + 6) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst + 2) = + BitUtil::byte_swap(*reinterpret_cast(src + 8)); + *reinterpret_cast(dst) = + BitUtil::byte_swap(*reinterpret_cast(src + 12)); + return; + case 15: + *reinterpret_cast(dst + 7) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst + 3) = + BitUtil::byte_swap(*reinterpret_cast(src + 8)); + *reinterpret_cast(dst + 1) = + BitUtil::byte_swap(*reinterpret_cast(src + 12)); + *reinterpret_cast(dst) = *reinterpret_cast(src + 14); + return; + case 16: + *reinterpret_cast(dst + 8) = + BitUtil::byte_swap(*reinterpret_cast(src)); + *reinterpret_cast(dst) = + BitUtil::byte_swap(*reinterpret_cast(src + 8)); + return; + default: + // Revert to slow loop-based swap. + ByteSwapScalarLoop(source, len, dest); + return; + } + } + + static void ByteSwapScalarLoop(const void *src, int len, void *dst) { + //TODO: improve the performance of following code further using BSWAP intrinsic + uint8_t *d = reinterpret_cast(dst); + const uint8_t *s = reinterpret_cast(src); + for (int i = 0; i < len; ++i) d[i] = s[len - i - 1]; + } }; } // namespace doris diff --git a/be/src/util/tuple_row_zorder_compare.cpp b/be/src/util/tuple_row_zorder_compare.cpp new file mode 100644 index 0000000000..58318e67f4 --- /dev/null +++ b/be/src/util/tuple_row_zorder_compare.cpp @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/tuple_row_zorder_compare.h" + +namespace doris { + + RowComparator::RowComparator(Schema* schema) { + + } + + int RowComparator::operator()(const char* left, const char* right) const { + return -1; + } + + TupleRowZOrderComparator::TupleRowZOrderComparator() { + _schema = nullptr; + _sort_col_num = 0; + } + + TupleRowZOrderComparator::TupleRowZOrderComparator(int sort_col_num) { + _schema = nullptr; + _sort_col_num = sort_col_num; + } + + TupleRowZOrderComparator::TupleRowZOrderComparator(Schema* schema, int sort_col_num) + :_schema(schema), _sort_col_num(sort_col_num) { + _max_col_size = get_type_byte_size(_schema->column(0)->type()); + for (size_t i = 1; i < _sort_col_num; ++i) { + if (_max_col_size < get_type_byte_size(_schema->column(i)->type())) { + _max_col_size = get_type_byte_size(_schema->column(i)->type()); + } + } + } + + int TupleRowZOrderComparator::compare(const char* lhs, const char* rhs) const { + ContiguousRow lhs_row(_schema, lhs); + ContiguousRow rhs_row(_schema, rhs); + if (_max_col_size <= 4) { + return compare_based_on_size(lhs_row, rhs_row); + } else if (_max_col_size <= 8) { + return compare_based_on_size(lhs_row, rhs_row); + } else { + return compare_based_on_size(lhs_row, rhs_row); + } + } + + void TupleRowZOrderComparator::max_col_size(const RowCursor& rc) { + _max_col_size = get_type_byte_size(rc.schema()->column(0)->type()); + for (size_t i = 1; i < _sort_col_num; ++i) { + if (_max_col_size < get_type_byte_size(rc.schema()->column(i)->type())) { + _max_col_size = get_type_byte_size(rc.schema()->column(i)->type()); + } + } + } + + int TupleRowZOrderComparator::compare_row(const RowCursor& lhs, const RowCursor& rhs) { + max_col_size(lhs); + if (_max_col_size <= 4) { + return compare_based_on_size(lhs, rhs); + } else if (_max_col_size <= 8) { + return compare_based_on_size(lhs, lhs); + } else { + return compare_based_on_size(lhs, lhs); + } + } + + template + int TupleRowZOrderComparator::compare_based_on_size(LhsRowType& lhs, LhsRowType& rhs) const { + auto less_msb = [](U x, U y) { return x < y && x < (x ^ y); }; + FieldType type = lhs.schema()->column(0)->type(); + U msd_lhs = get_shared_representation(lhs.cell(0).is_null() ? nullptr : lhs.cell(0).cell_ptr(), + type); + U msd_rhs = get_shared_representation(rhs.cell(0).is_null() ? nullptr : rhs.cell(0).cell_ptr(), + type); + for (int i = 1; i < _sort_col_num; ++i) { + type = lhs.schema()->column(i)->type(); + const void *lhs_v = lhs.cell(i).is_null() ? nullptr : lhs.cell(i).cell_ptr(); + const void *rhs_v = rhs.cell(i).is_null() ? nullptr : rhs.cell(i).cell_ptr(); + U lhsi = get_shared_representation(lhs_v, type); + U rhsi = get_shared_representation(rhs_v, type); + if (less_msb(msd_lhs ^ msd_rhs, lhsi ^ rhsi)) { + msd_lhs = lhsi; + msd_rhs = rhsi; + } + } + return msd_lhs < msd_rhs ? -1 : (msd_lhs > msd_rhs ? 1 : 0); + } + + template + U TupleRowZOrderComparator::get_shared_representation(const void *val, FieldType type) const { + // The mask used for setting the sign bit correctly. + if (val == NULL) return 0; + constexpr U mask = (U) 1 << (sizeof(U) * 8 - 1); + switch (type) { + case FieldType::OLAP_FIELD_TYPE_NONE: + return 0; + case FieldType::OLAP_FIELD_TYPE_BOOL: + return static_cast(*reinterpret_cast(val)) << (sizeof(U) * 8 - 1); + case FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT: + case FieldType::OLAP_FIELD_TYPE_TINYINT: + return get_shared_int_representation( + *reinterpret_cast(val), mask); + case FieldType::OLAP_FIELD_TYPE_SMALLINT: + case FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT: + return get_shared_int_representation( + *reinterpret_cast(val), mask); + case FieldType::OLAP_FIELD_TYPE_INT: + return get_shared_int_representation( + *reinterpret_cast(val), mask); + case FieldType::OLAP_FIELD_TYPE_DATETIME: + case FieldType::OLAP_FIELD_TYPE_DATE: + case FieldType::OLAP_FIELD_TYPE_BIGINT: + case FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT: + return get_shared_int_representation( + *reinterpret_cast(val), mask); + case FieldType::OLAP_FIELD_TYPE_LARGEINT: + return static_cast(*reinterpret_cast(val)) ^ mask; + case FieldType::OLAP_FIELD_TYPE_FLOAT: + return get_shared_float_representation(val, mask); + case FieldType::OLAP_FIELD_TYPE_DISCRETE_DOUBLE: + case FieldType::OLAP_FIELD_TYPE_DOUBLE: + return get_shared_float_representation(val, mask); + case FieldType::OLAP_FIELD_TYPE_CHAR: + case FieldType::OLAP_FIELD_TYPE_VARCHAR:{ + const StringValue *string_value = reinterpret_cast(val); + return get_shared_string_representation(string_value->ptr, string_value->len); + } + case FieldType::OLAP_FIELD_TYPE_DECIMAL: { + decimal12_t decimal_val = *reinterpret_cast(val); + int128_t value = decimal_val.integer*DecimalV2Value::ONE_BILLION + decimal_val.fraction; + return static_cast(value) ^ mask; + } + default: + return 0; + } + } + + template + U inline TupleRowZOrderComparator::get_shared_int_representation(const T val, U mask) const { + uint64_t shift_size = static_cast( + std::max(static_cast((sizeof(U) - sizeof(T)) * 8), (int64_t) 0)); + return (static_cast(val) << shift_size) ^ mask; + } + + template + U inline TupleRowZOrderComparator::get_shared_float_representation(const void *val, U mask) const { + int64_t tmp; + T floating_value = *reinterpret_cast(val); + memcpy(&tmp, &floating_value, sizeof(T)); + if (UNLIKELY(std::isnan(floating_value))) return 0; + if (floating_value < 0.0) { + // Flipping all bits for negative values. + return static_cast(~tmp) << std::max((sizeof(U) - sizeof(T)) * 8, (uint64_t) 0); + } else { + // Flipping only first bit. + return (static_cast(tmp) << std::max((sizeof(U) - sizeof(T)) * 8, (uint64_t) 0)) ^ + mask; + } + } + + template + U inline TupleRowZOrderComparator::get_shared_string_representation(const char *char_ptr, + int length) const { + int len = length < sizeof(U) ? length : sizeof(U); + if (len == 0) return 0; + U dst = 0; + // We copy the bytes from the string but swap the bytes because of integer endianness. + BitUtil::ByteSwapScalar(&dst, char_ptr, len); + return dst << ((sizeof(U) - len) * 8); + } + + int TupleRowZOrderComparator::operator()(const char* lhs, const char* rhs) const { + int result = compare(lhs, rhs); + return result; + } + + int TupleRowZOrderComparator::get_type_byte_size(FieldType type) const { + switch (type) { + case FieldType::OLAP_FIELD_TYPE_OBJECT: + case FieldType::OLAP_FIELD_TYPE_HLL: + case FieldType::OLAP_FIELD_TYPE_STRUCT: + case FieldType::OLAP_FIELD_TYPE_ARRAY: + case FieldType::OLAP_FIELD_TYPE_MAP: + case FieldType::OLAP_FIELD_TYPE_CHAR: + case FieldType::OLAP_FIELD_TYPE_VARCHAR: + return 0; + case FieldType::OLAP_FIELD_TYPE_NONE: + case FieldType::OLAP_FIELD_TYPE_BOOL: + case FieldType::OLAP_FIELD_TYPE_UNSIGNED_TINYINT: + case FieldType::OLAP_FIELD_TYPE_TINYINT: + return 1; + case FieldType::OLAP_FIELD_TYPE_SMALLINT: + case FieldType::OLAP_FIELD_TYPE_UNSIGNED_SMALLINT: + return 2; + case FieldType::OLAP_FIELD_TYPE_FLOAT: + case FieldType::OLAP_FIELD_TYPE_INT: + case FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT: + return 4; + case FieldType::OLAP_FIELD_TYPE_DISCRETE_DOUBLE: + case FieldType::OLAP_FIELD_TYPE_DOUBLE: + case FieldType::OLAP_FIELD_TYPE_BIGINT: + case FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT: + return 8; + case FieldType::OLAP_FIELD_TYPE_DECIMAL: + case FieldType::OLAP_FIELD_TYPE_LARGEINT: + case FieldType::OLAP_FIELD_TYPE_DATETIME: + case FieldType::OLAP_FIELD_TYPE_DATE: + return 16; + case FieldType::OLAP_FIELD_TYPE_UNKNOWN: + DCHECK(false); + break; + default: + DCHECK(false); + } + return -1; + } + +} diff --git a/be/src/util/tuple_row_zorder_compare.h b/be/src/util/tuple_row_zorder_compare.h new file mode 100644 index 0000000000..e44008c4f6 --- /dev/null +++ b/be/src/util/tuple_row_zorder_compare.h @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef DORIS_TUPLE_ROW_ZORDER_COMPARE_H +#define DORIS_TUPLE_ROW_ZORDER_COMPARE_H + +#include "exec/sort_exec_exprs.h" +#include "exprs/expr.h" +#include "exprs/expr_context.h" +#include "runtime/descriptors.h" +#include "runtime/raw_value.h" +#include "runtime/tuple.h" +#include "runtime/tuple_row.h" +#include "olap/schema.h" +#include "olap/row.h" +#include "olap/row_cursor.h" + +namespace doris { +class RowComparator { +public: + RowComparator()=default; + RowComparator(Schema* schema); + virtual int operator()(const char* left, const char* right) const; +}; + +class TupleRowZOrderComparator: public RowComparator { + +private: + typedef __uint128_t uint128_t; + int _max_col_size = 0; + const Schema* _schema; + int _sort_col_num = 0; + +public: + TupleRowZOrderComparator(); + TupleRowZOrderComparator(int sort_col_num); + TupleRowZOrderComparator(Schema* schema, int sort_col_num); + int compare(const char* lhs, const char* rhs) const; + void max_col_size(const RowCursor& rc); + int compare_row(const RowCursor& lhs, const RowCursor& rhs); + template + int compare_based_on_size(LhsRowType& lhs, LhsRowType& rhs) const; + template + U get_shared_representation(const void *val, FieldType type) const; + template + U inline get_shared_int_representation(const T val, U mask) const; + template + U inline get_shared_float_representation(const void *val, U mask) const; + template + U inline get_shared_string_representation(const char *char_ptr, int length) const; + virtual int operator()(const char* lhs, const char* rhs) const; + int get_type_byte_size(FieldType type) const; +}; +} + +#endif //DORIS_TUPLE_ROW_ZORDER_COMPARE_H \ No newline at end of file diff --git a/be/test/olap/skiplist_test.cpp b/be/test/olap/skiplist_test.cpp index eb5831a006..7c76ac5e3b 100644 --- a/be/test/olap/skiplist_test.cpp +++ b/be/test/olap/skiplist_test.cpp @@ -55,7 +55,7 @@ TEST_F(SkipTest, Empty) { std::shared_ptr tracker(new MemTracker(-1)); std::unique_ptr mem_pool(new MemPool(tracker.get())); - TestComparator cmp; + TestComparator* cmp = new TestComparator(); SkipList list(cmp, mem_pool.get(), false); ASSERT_TRUE(!list.Contains(10)); @@ -67,6 +67,7 @@ TEST_F(SkipTest, Empty) { ASSERT_TRUE(!iter.Valid()); iter.SeekToLast(); ASSERT_TRUE(!iter.Valid()); + delete cmp; } TEST_F(SkipTest, InsertAndLookup) { @@ -77,7 +78,7 @@ TEST_F(SkipTest, InsertAndLookup) { const int R = 5000; Random rnd(1000); std::set keys; - TestComparator cmp; + TestComparator* cmp = new TestComparator(); SkipList list(cmp, mem_pool.get(), false); for (int i = 0; i < N; i++) { Key key = rnd.Next() % R; @@ -147,6 +148,7 @@ TEST_F(SkipTest, InsertAndLookup) { } ASSERT_TRUE(!iter.Valid()); } + delete cmp; } // Only non-DUP model will use Find() and InsertWithHint(). @@ -158,7 +160,7 @@ TEST_F(SkipTest, InsertWithHintNoneDupModel) { const int R = 5000; Random rnd(1000); std::set keys; - TestComparator cmp; + TestComparator* cmp = new TestComparator(); SkipList list(cmp, mem_pool.get(), false); SkipList::Hint hint; for (int i = 0; i < N; i++) { @@ -179,6 +181,7 @@ TEST_F(SkipTest, InsertWithHintNoneDupModel) { ASSERT_EQ(keys.count(i), 0); } } + delete cmp; } // We want to make sure that with a single writer and multiple @@ -259,7 +262,7 @@ private: std::shared_ptr _mem_tracker; std::unique_ptr _mem_pool; - + std::shared_ptr _comparator; // SkipList is not protected by _mu. We just use a single writer // thread to modify it. SkipList _list; @@ -268,8 +271,9 @@ public: ConcurrentTest() : _mem_tracker(new MemTracker(-1)), _mem_pool(new MemPool(_mem_tracker.get())), - _list(TestComparator(), _mem_pool.get(), false) {} - + _comparator(new TestComparator()), + _list(_comparator.get(), _mem_pool.get(), false) {} + // REQUIRES: External synchronization void write_step(Random* rnd) { const uint32_t k = rnd->Next() % K; diff --git a/be/test/olap/test_data/header_without_inc_rs.txt b/be/test/olap/test_data/header_without_inc_rs.txt index edd7d03628..11139bbaa1 100644 --- a/be/test/olap/test_data/header_without_inc_rs.txt +++ b/be/test/olap/test_data/header_without_inc_rs.txt @@ -51,7 +51,9 @@ "next_column_unique_id": 3, "is_in_memory": false, "delete_sign_idx": -1, - "sequence_col_idx": -1 + "sequence_col_idx": -1, + "sort_type": "LEXICAL", + "sort_col_num": 0 }, "rs_metas": [ { diff --git a/be/test/util/CMakeLists.txt b/be/test/util/CMakeLists.txt index 73decdfd9c..ec88f8c12a 100644 --- a/be/test/util/CMakeLists.txt +++ b/be/test/util/CMakeLists.txt @@ -74,5 +74,6 @@ ADD_BE_TEST(broker_storage_backend_test) ADD_BE_TEST(sort_heap_test) ADD_BE_TEST(counts_test) ADD_BE_TEST(date_func_test) +ADD_BE_TEST(tuple_row_zorder_compare_test) target_link_libraries(Test_util Common Util Gutil ${Boost_LIBRARIES} glog gflags fmt protobuf) diff --git a/be/test/util/tuple_row_zorder_compare_test.cpp b/be/test/util/tuple_row_zorder_compare_test.cpp new file mode 100644 index 0000000000..b83b53cccc --- /dev/null +++ b/be/test/util/tuple_row_zorder_compare_test.cpp @@ -0,0 +1,1120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/json_util.h" +#include +#include "common/logging.h" +#include "exec/sort_exec_exprs.h" +#include "exprs/expr.h" +#include "exprs/expr_context.h" +#include "runtime/descriptors.h" +#include "runtime/raw_value.h" +#include "runtime/tuple.h" +#include "runtime/tuple_row.h" +#include "olap/schema.h" +#include "olap/row.h" +#include "util/tuple_row_zorder_compare.h" +#include "runtime/large_int_value.h" +#include "olap/memtable.h" + +namespace doris { + class TupleRowZOrderCompareTest : public testing::Test { + public: + TupleRowZOrderComparator _comparator; + ObjectPool _agg_buffer_pool; + std::unique_ptr _buffer_mem_pool; + MemTracker* _mem_tracker; + Schema* _schema; + std::vector* _slot_descs; + + public: + TupleRowZOrderCompareTest() { + _mem_tracker = new MemTracker(-1); + _buffer_mem_pool.reset(new MemPool(_mem_tracker)); + } + + template + int CompareInt8Test(T lval1, T lval2, T rval1, T rval2) { + int col_num = 2; + std::vector slot_descs; + for (int i = 1; i <= col_num; i++) { + TSlotDescriptor slot_desc; + slot_desc.id = i; + slot_desc.parent = 0; + TTypeDesc type; + { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::type::TINYINT); + node.__set_scalar_type(scalar_type); + type.types.push_back(node); + } + slot_desc.slotType = type; + slot_desc.columnPos = i; + slot_desc.byteOffset = (i-1)*sizeof(int8_t) + 1*i; + + slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(int8_t) + 1*(i-1); + slot_desc.nullIndicatorBit = 1; + std::ostringstream ss; + ss << "col_" << i; + slot_desc.colName = ss.str(); + slot_desc.slotIdx = i; + slot_desc.isMaterialized = true; + SlotDescriptor* slot = new SlotDescriptor(slot_desc); + slot_descs.push_back(slot); + } + _slot_descs = &slot_descs; + + std::vector col_schemas; + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_TINYINT, true); + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_TINYINT, true); + Schema schema(col_schemas, col_num); + _schema = &schema; + + TupleRowZOrderComparator comparator(&schema, 2); + _comparator = comparator; + + Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int8_t), _buffer_mem_pool.get()); + if (IS_FIRST_SLOT_NULL) { + lhs_tuple->set_null(NullIndicatorOffset(0, 1)); + } + FillMem(lhs_tuple, 1, lval1, lval2); + uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int8_t)); + ContiguousRow lhs_row(_schema, lhs_tuple_buf); + tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + + Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int8_t), _buffer_mem_pool.get()); + FillMem(rhs_tuple, 1, rval1, rval2); + uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int8_t)); + ContiguousRow rhs_row(_schema, rhs_tuple_buf); + tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + reinterpret_cast(rhs_row.row_ptr())); + return result; + } + + template + int CompareInt16Test(T lval1, T lval2, T rval1, T rval2) { + int col_num = 2; + std::vector slot_descs; + for (int i = 1; i <= col_num; i++) { + TSlotDescriptor slot_desc; + slot_desc.id = i; + slot_desc.parent = 0; + TTypeDesc type; + { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::type::SMALLINT); + node.__set_scalar_type(scalar_type); + type.types.push_back(node); + } + slot_desc.slotType = type; + slot_desc.columnPos = i; + slot_desc.byteOffset = (i-1)*sizeof(int16_t) + 1*i; + + slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(int16_t) + 1*(i-1); + slot_desc.nullIndicatorBit = 1; + std::ostringstream ss; + ss << "col_" << i; + slot_desc.colName = ss.str(); + slot_desc.slotIdx = i; + slot_desc.isMaterialized = true; + SlotDescriptor* slot = new SlotDescriptor(slot_desc); + slot_descs.push_back(slot); + } + _slot_descs = &slot_descs; + + std::vector col_schemas; + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_SMALLINT, true); + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_SMALLINT, true); + Schema schema(col_schemas, col_num); + _schema = &schema; + + TupleRowZOrderComparator comparator(&schema, 2); + _comparator = comparator; + + Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int16_t), _buffer_mem_pool.get()); + if (IS_FIRST_SLOT_NULL) { + lhs_tuple->set_null(NullIndicatorOffset(0, 1)); + } + FillMem(lhs_tuple, 1, lval1, lval2); + uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int16_t)); + ContiguousRow lhs_row(_schema, lhs_tuple_buf); + tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + + Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int16_t), _buffer_mem_pool.get()); + FillMem(rhs_tuple, 1, rval1, rval2); + uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int16_t)); + ContiguousRow rhs_row(_schema, rhs_tuple_buf); + tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + reinterpret_cast(rhs_row.row_ptr())); + return result; + } + + template + int CompareIntTest(T lval1, T lval2, T rval1, T rval2) { + int col_num = 2; + std::vector slot_descs; + for (int i = 1; i <= col_num; i++) { + TSlotDescriptor slot_desc; + slot_desc.id = i; + slot_desc.parent = 0; + TTypeDesc type; + { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::type::INT); + node.__set_scalar_type(scalar_type); + type.types.push_back(node); + } + slot_desc.slotType = type; + slot_desc.columnPos = i; + slot_desc.byteOffset = (i-1)*sizeof(int32_t) + 1*i; + + slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(int32_t) + 1*(i-1); + slot_desc.nullIndicatorBit = 1; + std::ostringstream ss; + ss << "col_" << i; + slot_desc.colName = ss.str(); + slot_desc.slotIdx = i; + slot_desc.isMaterialized = true; + SlotDescriptor* slot = new SlotDescriptor(slot_desc); + slot_descs.push_back(slot); + } + _slot_descs = &slot_descs; + + std::vector col_schemas; + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_INT, true); + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_INT, true); + Schema schema(col_schemas, col_num); + _schema = &schema; + + TupleRowZOrderComparator comparator(&schema, 2); + _comparator = comparator; + + Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int32_t), _buffer_mem_pool.get()); + if (IS_FIRST_SLOT_NULL) { + lhs_tuple->set_null(NullIndicatorOffset(0, 1)); + } + FillMem(lhs_tuple, 1, lval1, lval2); + uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int32_t)); + ContiguousRow lhs_row(_schema, lhs_tuple_buf); + tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + + Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int32_t), _buffer_mem_pool.get()); + FillMem(rhs_tuple, 1, rval1, rval2); + uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int32_t)); + ContiguousRow rhs_row(_schema, rhs_tuple_buf); + tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + reinterpret_cast(rhs_row.row_ptr())); + return result; + } + + template + int CompareInt64Test(T lval1, T lval2, T rval1, T rval2) { + int col_num = 2; + std::vector slot_descs; + for (int i = 1; i <= col_num; i++) { + TSlotDescriptor slot_desc; + slot_desc.id = i; + slot_desc.parent = 0; + TTypeDesc type; + { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::type::BIGINT); + node.__set_scalar_type(scalar_type); + type.types.push_back(node); + } + slot_desc.slotType = type; + slot_desc.columnPos = i; + slot_desc.byteOffset = (i-1)*sizeof(int64_t) + 1*i; + + slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(int64_t) + 1*(i-1); + slot_desc.nullIndicatorBit = 1; + std::ostringstream ss; + ss << "col_" << i; + slot_desc.colName = ss.str(); + slot_desc.slotIdx = i; + slot_desc.isMaterialized = true; + SlotDescriptor* slot = new SlotDescriptor(slot_desc); + slot_descs.push_back(slot); + } + _slot_descs = &slot_descs; + + std::vector col_schemas; + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_BIGINT, true); + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_BIGINT, true); + Schema schema(col_schemas, col_num); + _schema = &schema; + + TupleRowZOrderComparator comparator(&schema, 2); + _comparator = comparator; + + Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int64_t), _buffer_mem_pool.get()); + if (IS_FIRST_SLOT_NULL) { + lhs_tuple->set_null(NullIndicatorOffset(0, 1)); + } + FillMem(lhs_tuple, 1, lval1, lval2); + uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int64_t)); + ContiguousRow lhs_row(_schema, lhs_tuple_buf); + tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + + Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int64_t), _buffer_mem_pool.get()); + FillMem(rhs_tuple, 1, rval1, rval2); + uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int64_t)); + ContiguousRow rhs_row(_schema, rhs_tuple_buf); + tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + reinterpret_cast(rhs_row.row_ptr())); + return result; + } + + template + int CompareInt128Test(T lval1, T lval2, T rval1, T rval2) { + int col_num = 2; + std::vector slot_descs; + for (int i = 1; i <= col_num; i++) { + TSlotDescriptor slot_desc; + slot_desc.id = i; + slot_desc.parent = 0; + TTypeDesc type; + { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::type::LARGEINT); + node.__set_scalar_type(scalar_type); + type.types.push_back(node); + } + slot_desc.slotType = type; + slot_desc.columnPos = i; + slot_desc.byteOffset = (i-1)*sizeof(int128_t) + 1*i; + + slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(int128_t) + 1*(i-1); + slot_desc.nullIndicatorBit = 1; + std::ostringstream ss; + ss << "col_" << i; + slot_desc.colName = ss.str(); + slot_desc.slotIdx = i; + slot_desc.isMaterialized = true; + SlotDescriptor* slot = new SlotDescriptor(slot_desc); + slot_descs.push_back(slot); + } + _slot_descs = &slot_descs; + + std::vector col_schemas; + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_LARGEINT, true); + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_LARGEINT, true); + Schema schema(col_schemas, col_num); + _schema = &schema; + + TupleRowZOrderComparator comparator(&schema, 2); + _comparator = comparator; + MemTable::RowCursorComparator row_comparator(&schema); + + Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int128_t), _buffer_mem_pool.get()); + if (IS_FIRST_SLOT_NULL) { + lhs_tuple->set_null(NullIndicatorOffset(0, 1)); + } + FillMem(lhs_tuple, 1, lval1, lval2); + uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int128_t)); + ContiguousRow lhs_row(_schema, lhs_tuple_buf); + tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + + Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int128_t), _buffer_mem_pool.get()); + FillMem(rhs_tuple, 1, rval1, rval2); + uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int128_t)); + ContiguousRow rhs_row(_schema, rhs_tuple_buf); + tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + reinterpret_cast(rhs_row.row_ptr())); + return result; + } + + template + int CompareFloatTest(T lval1, T lval2, T rval1, T rval2) { + int col_num = 2; + std::vector slot_descs; + for (int i = 1; i <= col_num; i++) { + TSlotDescriptor slot_desc; + slot_desc.id = i; + slot_desc.parent = 0; + TTypeDesc type; + { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::type::FLOAT); + node.__set_scalar_type(scalar_type); + type.types.push_back(node); + } + slot_desc.slotType = type; + slot_desc.columnPos = i; + slot_desc.byteOffset = (i-1)*sizeof(float) + 1*i; + + slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(float) + 1*(i-1); + slot_desc.nullIndicatorBit = 1; + std::ostringstream ss; + ss << "col_" << i; + slot_desc.colName = ss.str(); + slot_desc.slotIdx = i; + slot_desc.isMaterialized = true; + SlotDescriptor* slot = new SlotDescriptor(slot_desc); + slot_descs.push_back(slot); + } + _slot_descs = &slot_descs; + + std::vector col_schemas; + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_FLOAT, true); + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_FLOAT, true); + Schema schema(col_schemas, col_num); + _schema = &schema; + + TupleRowZOrderComparator comparator(&schema, 2); + _comparator = comparator; + + Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(float), _buffer_mem_pool.get()); + if (IS_FIRST_SLOT_NULL) { + lhs_tuple->set_null(NullIndicatorOffset(0, 1)); + } + FillMem(lhs_tuple, 1, lval1, lval2); + uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(float)); + ContiguousRow lhs_row(_schema, lhs_tuple_buf); + tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + + Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(float), _buffer_mem_pool.get()); + FillMem(rhs_tuple, 1, rval1, rval2); + uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(float)); + ContiguousRow rhs_row(_schema, rhs_tuple_buf); + tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + reinterpret_cast(rhs_row.row_ptr())); + return result; + } + + template + int CompareDoubleTest(T lval1, T lval2, T rval1, T rval2) { + int col_num = 2; + std::vector slot_descs; + for (int i = 1; i <= col_num; i++) { + TSlotDescriptor slot_desc; + slot_desc.id = i; + slot_desc.parent = 0; + TTypeDesc type; + { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::type::DOUBLE); + node.__set_scalar_type(scalar_type); + type.types.push_back(node); + } + slot_desc.slotType = type; + slot_desc.columnPos = i; + slot_desc.byteOffset = (i-1)*sizeof(double) + 1*i; + + slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(double) + 1*(i-1); + slot_desc.nullIndicatorBit = 1; + std::ostringstream ss; + ss << "col_" << i; + slot_desc.colName = ss.str(); + slot_desc.slotIdx = i; + slot_desc.isMaterialized = true; + SlotDescriptor* slot = new SlotDescriptor(slot_desc); + slot_descs.push_back(slot); + } + _slot_descs = &slot_descs; + + std::vector col_schemas; + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DOUBLE, true); + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DOUBLE, true); + Schema schema(col_schemas, col_num); + _schema = &schema; + + TupleRowZOrderComparator comparator(&schema, 2); + _comparator = comparator; + + Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(double), _buffer_mem_pool.get()); + if (IS_FIRST_SLOT_NULL) { + lhs_tuple->set_null(NullIndicatorOffset(0, 1)); + } + FillMem(lhs_tuple, 1, lval1, lval2); + uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(double)); + ContiguousRow lhs_row(_schema, lhs_tuple_buf); + tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + + Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(double), _buffer_mem_pool.get()); + FillMem(rhs_tuple, 1, rval1, rval2); + uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(double)); + ContiguousRow rhs_row(_schema, rhs_tuple_buf); + tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + reinterpret_cast(rhs_row.row_ptr())); + return result; + } + + template + int CompareBoolTest(T lval1, T lval2, T rval1, T rval2) { + int col_num = 2; + std::vector slot_descs; + for (int i = 1; i <= col_num; i++) { + TSlotDescriptor slot_desc; + slot_desc.id = i; + slot_desc.parent = 0; + TTypeDesc type; + { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::type::BOOLEAN); + node.__set_scalar_type(scalar_type); + type.types.push_back(node); + } + slot_desc.slotType = type; + slot_desc.columnPos = i; + slot_desc.byteOffset = (i-1)*sizeof(bool) + 1*i; + slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(bool) + 1*(i-1); + slot_desc.nullIndicatorBit = 1; + std::ostringstream ss; + ss << "col_" << i; + slot_desc.colName = ss.str(); + slot_desc.slotIdx = i; + slot_desc.isMaterialized = true; + SlotDescriptor* slot = new SlotDescriptor(slot_desc); + slot_descs.push_back(slot); + } + _slot_descs = &slot_descs; + + std::vector col_schemas; + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_BOOL, true); + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_BOOL, true); + Schema schema(col_schemas, col_num); + _schema = &schema; + + TupleRowZOrderComparator comparator(&schema, 2); + _comparator = comparator; + + Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(bool), _buffer_mem_pool.get()); + if (IS_FIRST_SLOT_NULL) { + lhs_tuple->set_null(NullIndicatorOffset(0, 1)); + } + FillMem(lhs_tuple, 1, lval1, lval2); + uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(bool)); + ContiguousRow lhs_row(_schema, lhs_tuple_buf); + tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + + Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(bool), _buffer_mem_pool.get()); + FillMem(rhs_tuple, 1, rval1, rval2); + uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(bool)); + ContiguousRow rhs_row(_schema, rhs_tuple_buf); + tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + reinterpret_cast(rhs_row.row_ptr())); + return result; + } + + template + int CompareCharTest(T lval1, T lval2, T rval1, T rval2) { + int col_num = 2; + std::vector slot_descs; + for (int i = 1; i <= col_num; i++) { + TSlotDescriptor slot_desc; + slot_desc.id = i; + slot_desc.parent = 0; + TTypeDesc type; + { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::type::VARCHAR); + scalar_type.__set_len(65535); + node.__set_scalar_type(scalar_type); + type.types.push_back(node); + } + slot_desc.slotType = type; + slot_desc.columnPos = i; + slot_desc.byteOffset = (i-1)*sizeof(StringValue) + 1*i; + slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(StringValue) + 1*(i-1); + slot_desc.nullIndicatorBit = 1; + std::ostringstream ss; + ss << "col_" << i; + slot_desc.colName = ss.str(); + slot_desc.slotIdx = i; + slot_desc.isMaterialized = true; + SlotDescriptor* slot = new SlotDescriptor(slot_desc); + slot_descs.push_back(slot); + } + _slot_descs = &slot_descs; + + std::vector col_schemas; + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_VARCHAR, true); + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_VARCHAR, true); + Schema schema(col_schemas, col_num); + _schema = &schema; + + TupleRowZOrderComparator comparator(&schema, 2); + _comparator = comparator; + + Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(StringValue), _buffer_mem_pool.get()); + if (IS_FIRST_SLOT_NULL) { + lhs_tuple->set_null(NullIndicatorOffset(0, 1)); + } + FillMem(lhs_tuple, 1, lval1, lval2); + uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(StringValue)); + ContiguousRow lhs_row(_schema, lhs_tuple_buf); + tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + + Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(StringValue), _buffer_mem_pool.get()); + FillMem(rhs_tuple, 1, rval1, rval2); + uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(StringValue)); + ContiguousRow rhs_row(_schema, rhs_tuple_buf); + tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + reinterpret_cast(rhs_row.row_ptr())); + return result; + } + + template + int CompareDateTest(T lval1, T lval2, T rval1, T rval2) { + int col_num = 2; + std::vector slot_descs; + for (int i = 1; i <= col_num; i++) { + TSlotDescriptor slot_desc; + slot_desc.id = i; + slot_desc.parent = 0; + TTypeDesc type; + { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::type::DATETIME); + node.__set_scalar_type(scalar_type); + type.types.push_back(node); + } + slot_desc.slotType = type; + slot_desc.columnPos = i; + slot_desc.byteOffset = (i-1)*sizeof(DateTimeValue) + 1*i; + slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(DateTimeValue) + 1*(i-1); + slot_desc.nullIndicatorBit = 1; + std::ostringstream ss; + ss << "col_" << i; + slot_desc.colName = ss.str(); + slot_desc.slotIdx = i; + slot_desc.isMaterialized = true; + SlotDescriptor* slot = new SlotDescriptor(slot_desc); + slot_descs.push_back(slot); + } + _slot_descs = &slot_descs; + + std::vector col_schemas; + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DATETIME, true); + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DATETIME, true); + Schema schema(col_schemas, col_num); + _schema = &schema; + + TupleRowZOrderComparator comparator(&schema, 2); + _comparator = comparator; + + Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(DateTimeValue), _buffer_mem_pool.get()); + if (IS_FIRST_SLOT_NULL) { + lhs_tuple->set_null(NullIndicatorOffset(0, 1)); + } + FillMem(lhs_tuple, 1, lval1, lval2); + uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(DateTimeValue)); + ContiguousRow lhs_row(_schema, lhs_tuple_buf); + tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + + Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(DateTimeValue), _buffer_mem_pool.get()); + FillMem(rhs_tuple, 1, rval1, rval2); + uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(DateTimeValue)); + ContiguousRow rhs_row(_schema, rhs_tuple_buf); + tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + reinterpret_cast(rhs_row.row_ptr())); + return result; + } + + template + int CompareDecimalTest(T lval1, T lval2, T rval1, T rval2) { + int col_num = 2; + std::vector slot_descs; + for (int i = 1; i <= col_num; i++) { + TSlotDescriptor slot_desc; + slot_desc.id = i; + slot_desc.parent = 0; + TTypeDesc type; + { + TTypeNode node; + node.__set_type(TTypeNodeType::SCALAR); + TScalarType scalar_type; + scalar_type.__set_type(TPrimitiveType::type::DECIMALV2); + scalar_type.__isset.precision = true; + scalar_type.__isset.scale = true; + scalar_type.__set_precision(-1); + scalar_type.__set_scale(-1); + node.__set_scalar_type(scalar_type); + type.types.push_back(node); + } + slot_desc.slotType = type; + slot_desc.columnPos = i; + slot_desc.byteOffset = (i-1)*sizeof(int128_t) + 1*i; + slot_desc.nullIndicatorByte = 0+(i-1)*sizeof(int128_t) + 1*(i-1); + slot_desc.nullIndicatorBit = 1; + std::ostringstream ss; + ss << "col_" << i; + slot_desc.colName = ss.str(); + slot_desc.slotIdx = i; + slot_desc.isMaterialized = true; + SlotDescriptor* slot = new SlotDescriptor(slot_desc); + slot_descs.push_back(slot); + } + _slot_descs = &slot_descs; + + std::vector col_schemas; + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DECIMAL, true); + col_schemas.emplace_back(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_DECIMAL, true); + Schema schema(col_schemas, col_num); + _schema = &schema; + + TupleRowZOrderComparator comparator(&schema, 2); + _comparator = comparator; + + Tuple* lhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int128_t), _buffer_mem_pool.get()); + if (IS_FIRST_SLOT_NULL) { + lhs_tuple->set_null(NullIndicatorOffset(0, 1)); + } + FillMem(lhs_tuple, 1, lval1.value(), lval2.value()); + uint8_t* lhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int128_t)); + ContiguousRow lhs_row(_schema, lhs_tuple_buf); + tuple_to_row(lhs_tuple, &lhs_row, _buffer_mem_pool.get()); + + Tuple* rhs_tuple = Tuple::create(col_num*sizeof(char) + col_num*sizeof(int128_t), _buffer_mem_pool.get()); + + FillMem(rhs_tuple, 1, rval1.value(), rval2.value()); + uint8_t* rhs_tuple_buf = _buffer_mem_pool->allocate(col_num*sizeof(char) + col_num*sizeof(int128_t)); + ContiguousRow rhs_row(_schema, rhs_tuple_buf); + tuple_to_row(rhs_tuple, &rhs_row, _buffer_mem_pool.get()); + int result = _comparator.compare(reinterpret_cast(lhs_row.row_ptr()), + reinterpret_cast(rhs_row.row_ptr())); + return result; + } + + void tuple_to_row(const Tuple* tuple, ContiguousRow* row, MemPool* mem_pool) { + for (size_t i = 0; i < _slot_descs->size(); ++i) { + auto cell = row->cell(i); + const SlotDescriptor* slot = (*_slot_descs)[i]; + bool is_null = tuple->is_null(slot->null_indicator_offset()); + const void* value = tuple->get_slot(slot->tuple_offset()); + _schema->column(i)->consume(&cell, (const char*)value, is_null, mem_pool, + &_agg_buffer_pool); + } + } + + template + void FillMem(Tuple* tuple_mem, int idx, T val) { + memcpy(tuple_mem->get_slot(idx), &val, sizeof(T)); + } + + template + void FillMem(Tuple* tuple_mem, int idx, T val, Args... args) { + // Use memcpy to avoid gcc generating unaligned instructions like movaps + // for int128_t. They will raise SegmentFault when addresses are not + // aligned to 16 bytes. + memcpy(tuple_mem->get_slot(idx), &val, sizeof(T)); + FillMem(tuple_mem, idx + sizeof(T)+1, args...); + } + + template + int Int8Int8Test(int8_t lval1, int8_t lval2, int8_t rval1, int8_t rval2) { + return CompareInt8Test(lval1, lval2, rval1, rval2); + } + + template + int Int16Int16Test(int16_t lval1, int16_t lval2, int16_t rval1, int16_t rval2) { + return CompareInt16Test(lval1, lval2, rval1, rval2); + } + + template + int IntIntTest(int32_t lval1, int32_t lval2, int32_t rval1, int32_t rval2) { + return CompareIntTest(lval1, lval2, rval1, rval2); + } + + template + int Int64Int64Test(int64_t lval1, int64_t lval2, int64_t rval1, int64_t rval2) { + return CompareInt64Test(lval1, lval2, rval1, rval2); + } + + template + int Int128Int128Test(int128_t lval1, int128_t lval2, int128_t rval1, int128_t rval2) { + return CompareInt128Test(lval1, lval2, rval1, rval2); + } + + template + int FloatFloatTest(float lval1, float lval2, float rval1, float rval2) { + return CompareFloatTest(lval1, lval2, rval1, rval2); + } + + template + int DoubleDoubleTest(double lval1, double lval2, double rval1, double rval2) { + return CompareDoubleTest(lval1, lval2, rval1, rval2); + } + + template + int BoolBoolTest(bool lval1, bool lval2, bool rval1, bool rval2) { + return CompareBoolTest(lval1, lval2, rval1, rval2); + } + + template + int CharCharTest(StringValue lval1, StringValue lval2, StringValue rval1, StringValue rval2) { + return CompareCharTest(lval1, lval2, rval1, rval2); + } + + template + int DateDateTest(DateTimeValue lval1, DateTimeValue lval2, DateTimeValue rval1, DateTimeValue rval2) { + return CompareDateTest(lval1, lval2, rval1, rval2); + } + + template + int DecimalDecimalTest(DecimalV2Value lval1, DecimalV2Value lval2, + DecimalV2Value rval1, DecimalV2Value rval2) { + return CompareDecimalTest(lval1, lval2, rval1, rval2); + } + }; + +TEST_F(TupleRowZOrderCompareTest, DecimalTest) { + std::string str1 = "1.00"; + std::string str2 = "1.00"; + std::string str3 = "1.00"; + std::string str4 = "1.00"; + DecimalV2Value val1(str1); + DecimalV2Value val2(str2); + DecimalV2Value val3(str3); + DecimalV2Value val4(str4); + EXPECT_EQ(DecimalDecimalTest(val1, val2, val3, val4), 0); + str1 = "-5.0"; + str2 = "3.0"; + str3 = "-5.0"; + str4 = "3.0"; + DecimalV2Value val5(str1); + DecimalV2Value val6(str2); + DecimalV2Value val7(str3); + DecimalV2Value val8(str4); + EXPECT_EQ(DecimalDecimalTest(val5, val6, val7, val8), 0); + str1 = "1.0"; + str2 = "0.0"; + str3 = "0.0"; + str4 = "1.0"; + DecimalV2Value val9(str1); + DecimalV2Value val10(str2); + DecimalV2Value val11(str3); + DecimalV2Value val12(str4); + EXPECT_EQ(DecimalDecimalTest(val9, val10, val11, val12), 1); + str1 = "0"; + str2 = "1"; + str3 = "1"; + str4 = "0"; + DecimalV2Value val13(str1); + DecimalV2Value val14(str2); + DecimalV2Value val15(str3); + DecimalV2Value val16(str4); + EXPECT_EQ(DecimalDecimalTest(val13, val14, val15, val16), -1); + str1 = "256"; + str2 = "10"; + str3 = "255"; + str4 = "100"; + DecimalV2Value val17(str1); + DecimalV2Value val18(str2); + DecimalV2Value val19(str3); + DecimalV2Value val20(str4); + EXPECT_EQ(DecimalDecimalTest(val17, val18, val19, val20), -1); + str1 = "3"; + str2 = "1024"; + str3 = "128"; + str4 = "1023"; + DecimalV2Value val21(str1); + DecimalV2Value val22(str2); + DecimalV2Value val23(str3); + DecimalV2Value val24(str4); + EXPECT_EQ(DecimalDecimalTest(val21, val22, val23, val24), -1); + str1 = "1024"; + str2 = "511"; + str3 = "1023"; + str4 = "0"; + DecimalV2Value val25(str1); + DecimalV2Value val26(str2); + DecimalV2Value val27(str3); + DecimalV2Value val28(str4); + EXPECT_EQ(DecimalDecimalTest(val25, val26, val27, val28), 1); + str1 = "5550"; + str2 = "0"; + str3 = "5000"; + str4 = "4097"; + DecimalV2Value val29(str1); + DecimalV2Value val30(str2); + DecimalV2Value val31(str3); + DecimalV2Value val32(str4); + EXPECT_EQ(DecimalDecimalTest(val29, val30, val31, val32), -1); +} + +TEST_F(TupleRowZOrderCompareTest, DateDateTest) { + DateTimeValue val1; + DateTimeValue val2; + DateTimeValue val3; + DateTimeValue val4; + std::string str1 = "2015-04-09 14:07:46"; + std::string str2 = "2015-04-09 14:07:46"; + std::string str3 = "2015-04-09 14:07:46"; + std::string str4 = "2015-04-09 14:07:46"; + val1.from_date_str(str1.c_str(), str1.size()); + val2.from_date_str(str2.c_str(), str2.size()); + val3.from_date_str(str3.c_str(), str3.size()); + val4.from_date_str(str4.c_str(), str4.size()); + EXPECT_EQ(DateDateTest(val1, val2, val3, val4), 0); + str1 = "1415-12-09 10:07:44"; + str2 = "2015-04-09 14:07:46"; + str3 = "1415-12-09 10:07:44"; + str4 = "2015-04-09 14:07:46"; + val1.from_date_str(str1.c_str(), str1.size()); + val2.from_date_str(str2.c_str(), str2.size()); + val3.from_date_str(str3.c_str(), str3.size()); + val4.from_date_str(str4.c_str(), str4.size()); + EXPECT_EQ(DateDateTest(val1, val2, val3, val4), 0); + str1 = "1400-01-01 00:00:00"; + str2 = "9999-12-31 14:07:46"; + str3 = "8000-12-09 10:07:44"; + str4 = "2015-04-09 14:07:46"; + val1.from_date_str(str1.c_str(), str1.size()); + val2.from_date_str(str2.c_str(), str2.size()); + val3.from_date_str(str3.c_str(), str3.size()); + val4.from_date_str(str4.c_str(), str4.size()); + EXPECT_EQ(DateDateTest(val1, val2, val3, val4), -1); + str1 = "1400-01-01 00:00:01"; + str2 = "1400-01-01 00:00:00"; + str3 = "1400-01-01 00:00:00"; + str4 = "1400-01-01 00:00:01"; + val1.from_date_str(str1.c_str(), str1.size()); + val2.from_date_str(str2.c_str(), str2.size()); + val3.from_date_str(str3.c_str(), str3.size()); + val4.from_date_str(str4.c_str(), str4.size()); + EXPECT_EQ(DateDateTest(val1, val2, val3, val4), 1); + str1 = "1400-01-01 00:00:03"; + str2 = "1400-01-01 00:00:07"; + str3 = "1400-01-01 00:00:04"; + str4 = "1400-01-01 00:00:00"; + val1.from_date_str(str1.c_str(), str1.size()); + val2.from_date_str(str2.c_str(), str2.size()); + val3.from_date_str(str3.c_str(), str3.size()); + val4.from_date_str(str4.c_str(), str4.size()); + EXPECT_EQ(DateDateTest(val1, val2, val3, val4), -1); + str1 = "1400-01-01 00:00:06"; + str2 = "1400-01-01 00:00:04"; + str3 = "1400-01-01 00:00:05"; + str4 = "1400-01-01 00:00:07"; + val1.from_date_str(str1.c_str(), str1.size()); + val2.from_date_str(str2.c_str(), str2.size()); + val3.from_date_str(str3.c_str(), str3.size()); + val4.from_date_str(str4.c_str(), str4.size()); + EXPECT_EQ(DateDateTest(val1, val2, val3, val4), 1); + str1 = "1400-01-01 00:00:05"; + str2 = "1400-01-01 00:00:05"; + str3 = "1400-01-01 00:00:06"; + str4 = "1400-01-01 00:00:04"; + val1.from_date_str(str1.c_str(), str1.size()); + val2.from_date_str(str2.c_str(), str2.size()); + val3.from_date_str(str3.c_str(), str3.size()); + val4.from_date_str(str4.c_str(), str4.size()); + EXPECT_EQ(DateDateTest(val1, val2, val3, val4), -1); + str1 = "1400-01-01 23:59:59"; + str2 = "1400-01-01 00:00:00"; + str3 = "1400-01-02 00:00:00"; + str4 = "1400-01-01 00:00:00"; + val1.from_date_str(str1.c_str(), str1.size()); + val2.from_date_str(str2.c_str(), str2.size()); + val3.from_date_str(str3.c_str(), str3.size()); + val4.from_date_str(str4.c_str(), str4.size()); + EXPECT_EQ(DateDateTest(val1, val2, val3, val4), -1); + str1 = "3541-11-03 23:59:59"; + str2 = "3541-11-03 00:00:00"; + str3 = "3541-11-04 00:00:00"; + str4 = "3541-11-03 00:00:00"; + val1.from_date_str(str1.c_str(), str1.size()); + val2.from_date_str(str2.c_str(), str2.size()); + val3.from_date_str(str3.c_str(), str3.size()); + val4.from_date_str(str4.c_str(), str4.size()); + EXPECT_EQ(DateDateTest(val1, val2, val3, val4), -1); +} +TEST_F(TupleRowZOrderCompareTest, CharTest) { + EXPECT_EQ(CharCharTest(StringValue("a"), StringValue("b"), StringValue("a"), StringValue("b")), 0); + EXPECT_EQ(CharCharTest(StringValue("a"), StringValue("b"), StringValue("a"), StringValue("b")), 0); + EXPECT_EQ(CharCharTest(StringValue("h"), StringValue("0"), StringValue("h"), StringValue("0")), 0); + EXPECT_EQ(CharCharTest(StringValue("h"), StringValue("z"), StringValue("z"), StringValue("h")), -1); + EXPECT_EQ(CharCharTest(StringValue("a"), StringValue("0"), StringValue("h"), StringValue("0")), -1); + EXPECT_EQ(CharCharTest(StringValue("!"), StringValue("{"), StringValue("0"), StringValue("K")), 1); + EXPECT_EQ(CharCharTest(StringValue("A"), StringValue("~"), StringValue("B"), StringValue("Z")), 1); + EXPECT_EQ(CharCharTest(StringValue("aaa"), StringValue("bbb"), StringValue("aaa"), StringValue("bbb")), 0); + EXPECT_EQ(CharCharTest(StringValue("abc"), StringValue("bbc"), StringValue("abc"), StringValue("bbc")), 0); + EXPECT_EQ(CharCharTest(StringValue("aah"), StringValue("aa0"), StringValue("aah"), StringValue("aa0")), 0); + EXPECT_EQ(CharCharTest(StringValue("aaa"), StringValue("aa0"), StringValue("aah"), StringValue("aa0")), -1); + EXPECT_EQ(CharCharTest(StringValue("aah"), StringValue("aaz"), StringValue("aaz"), StringValue("aah")), -1); + EXPECT_EQ(CharCharTest(StringValue("aa!"), StringValue("aa{"), StringValue("aa0"), StringValue("aaK")), 1); + EXPECT_EQ(CharCharTest(StringValue("aaA"), StringValue("aa~"), StringValue("aaB"), StringValue("aaZ")), 1); +} +TEST_F(TupleRowZOrderCompareTest, BoolTest) { + EXPECT_EQ(BoolBoolTest(true, false, true, false), 0); + EXPECT_EQ(BoolBoolTest(false, true, false, true), 0); + EXPECT_EQ(BoolBoolTest(true, true, true, false), 1); + EXPECT_EQ(BoolBoolTest(false, true, true, true), -1); + EXPECT_EQ(BoolBoolTest(false, true, false, false), 1); + EXPECT_EQ(BoolBoolTest(false, false, false, true), -1); + EXPECT_EQ(BoolBoolTest(true, false, false, false), 1); +} +TEST_F(TupleRowZOrderCompareTest, DoubleTest) { + EXPECT_EQ(DoubleDoubleTest(1.0, 0.0, 0.0, 1.0f), 1); + EXPECT_EQ(DoubleDoubleTest(0.0, 1.0, 1.0, 0.0f), -1); + EXPECT_EQ(DoubleDoubleTest(4.0, 3.0, 3.0, 4.0), 1); + EXPECT_EQ(DoubleDoubleTest(5.0, 7.0, 4.0, 10.0), -1); + EXPECT_EQ(DoubleDoubleTest(6.0, 10.0, 7.0, 3.0), 1); + EXPECT_EQ(DoubleDoubleTest(9.0, 7.0, 8.0, 10.0), -1); + EXPECT_EQ(DoubleDoubleTest(8.0, 8.0, 9.0, 7.0), 1); + EXPECT_EQ(DoubleDoubleTest(9.0, 4.0, 6.0, 10.0), 1); + EXPECT_EQ(DoubleDoubleTest(-4.0, -3.0, -3.0, -4.0), -1); + EXPECT_EQ(DoubleDoubleTest(-5.0, -7.0, -4.0, -10.0), 1); + EXPECT_EQ(DoubleDoubleTest(-6.0, -10.0, -7.0, -3.0), -1); + EXPECT_EQ(DoubleDoubleTest(-9.0, -7.0, -8.0, -10.0), 1); + EXPECT_EQ(DoubleDoubleTest(-8.0, -8.0, -9.0, -7.0), -1); + EXPECT_EQ(DoubleDoubleTest(-9.0, -4.0, -6.0, -10.0), -1); + EXPECT_EQ(DoubleDoubleTest(DBL_MAX / 2.0 + 2.0, 1.0, 1.0, DBL_MAX), 1); +} +TEST_F(TupleRowZOrderCompareTest, FloatTest) { + EXPECT_EQ(FloatFloatTest(1.0f, 0.0f, 0.0f, 1.0f), 1); + EXPECT_EQ(FloatFloatTest(0.0f, 1.0f, 1.0f, 0.0f), -1); + EXPECT_EQ(FloatFloatTest(4.0f, 3.0f, 3.0f, 4.0f), 1); + EXPECT_EQ(FloatFloatTest(5.0f, 7.0f, 4.0f, 10.0f), -1); + EXPECT_EQ(FloatFloatTest(6.0f, 10.0f, 7.0f, 3.0f), 1); + EXPECT_EQ(FloatFloatTest(9.0f, 7.0f, 8.0f, 10.0f), -1); + EXPECT_EQ(FloatFloatTest(8.0f , 8.0f, 9.0f, 7.0f), 1); + EXPECT_EQ(FloatFloatTest(9.0f, 4.0f, 6.0f, 10.0f), 1); + EXPECT_EQ(FloatFloatTest(-4.0f, -3.0f, -3.0f, -4.0f), -1); + EXPECT_EQ(FloatFloatTest(-5.0f, -7.0f, -4.0f, -10.0f), 1); + EXPECT_EQ(FloatFloatTest(-6.0f, -10.0f, -7.0f, -3.0f), -1); + EXPECT_EQ(FloatFloatTest(-9.0f, -7.0f, -8.0f, -10.0f), 1); + EXPECT_EQ(FloatFloatTest(-8.0f, -8.0f, -9.0f, -7.0f), -1); + EXPECT_EQ(FloatFloatTest(-9.0f, -4.0f, -6.0f, -10.0f), -1); + EXPECT_EQ(FloatFloatTest(FLT_MAX / 2.0f + 2.0f, 1.0f, 1.0f, FLT_MAX), 1); +} +TEST_F(TupleRowZOrderCompareTest, Int8Test) { + EXPECT_EQ(Int8Int8Test(0, 0, 0, 0), 0); + EXPECT_EQ(Int8Int8Test(-5, 3, -5, 3), 0); + EXPECT_EQ(Int8Int8Test(1, 0, 0, 1), 1); + EXPECT_EQ(Int8Int8Test(0, 1, 1, 0), -1); + EXPECT_EQ(Int8Int8Test(1, 0, 0, 1), 1); + EXPECT_EQ(Int8Int8Test(2, 4, 1, 7), 1); + EXPECT_EQ(Int8Int8Test(3, 7, 4, 0), -1); + EXPECT_EQ(Int8Int8Test(6, 4, 5, 7), 1); + EXPECT_EQ(Int8Int8Test(5, 5, 6, 4), -1); + EXPECT_EQ(Int8Int8Test(6, 1, 3, 7), 1); + EXPECT_EQ(Int8Int8Test(INT8_MAX / 2 + 2, 1, 1, INT8_MAX), 1); + EXPECT_EQ(Int8Int8Test(INT8_MAX / 2, 1, 1, INT8_MAX), -1); +} +TEST_F(TupleRowZOrderCompareTest, Int16Test) { + EXPECT_EQ(Int16Int16Test(0, 0, 0, 0), 0); + EXPECT_EQ(Int16Int16Test(-5, 3, -5, 3), 0); + EXPECT_EQ(Int16Int16Test(1, 0, 0, 1), 1); + EXPECT_EQ(Int16Int16Test(0, 1, 1, 0), -1); + EXPECT_EQ(Int16Int16Test(1, 0, 0, 1), 1); + EXPECT_EQ(Int16Int16Test(2, 4, 1, 7), 1); + EXPECT_EQ(Int16Int16Test(3, 7, 4, 0), -1); + EXPECT_EQ(Int16Int16Test(6, 4, 5, 7), 1); + EXPECT_EQ(Int16Int16Test(5, 5, 6, 4), -1); + EXPECT_EQ(Int16Int16Test(6, 1, 3, 7), 1); + EXPECT_EQ(Int16Int16Test(INT16_MAX / 2 + 2, 1, 1, INT16_MAX), 1); + EXPECT_EQ(Int16Int16Test(INT16_MAX / 2, 1, 1, INT16_MAX), -1); +} + +TEST_F(TupleRowZOrderCompareTest, Int32Test) { + EXPECT_EQ(IntIntTest(0, 2, 1, 1), 1); + EXPECT_EQ(IntIntTest(-5, 3, -5, 3), 0); + + EXPECT_EQ(IntIntTest(0, 4, 1, 2), 1); + EXPECT_EQ(IntIntTest(0, 1, 1, 0), -1); + + EXPECT_EQ(IntIntTest(1, 0, 0, 1), 1); + EXPECT_EQ(IntIntTest(2, 4, 1, 7), 1); + EXPECT_EQ(IntIntTest(3, 7, 4, 0), -1); + EXPECT_EQ(IntIntTest(6, 4, 5, 7), 1); + EXPECT_EQ(IntIntTest(5, 5, 6, 4), -1); + EXPECT_EQ(IntIntTest(6, 1, 3, 7), 1); + + EXPECT_EQ(IntIntTest(INT32_MAX / 2 + 2, 1, 1, INT32_MAX), 1); + EXPECT_EQ(IntIntTest(INT32_MAX / 2, 1, 1, INT32_MAX), -1); + + EXPECT_EQ(IntIntTest(1, 1, 1, 1), -1); + EXPECT_EQ(IntIntTest(4242, 1, 1, 1), -1); + EXPECT_EQ(IntIntTest(1, 0, 0, 1), -1); + EXPECT_EQ(IntIntTest(1, 0, INT32_MIN, 0), 0); +} + +TEST_F(TupleRowZOrderCompareTest, Int64Test) { + EXPECT_EQ(Int64Int64Test(0, 0, 0, 0), 0); + EXPECT_EQ(Int64Int64Test(-5, 3, -5, 3), 0); + EXPECT_EQ(Int64Int64Test(1, 0, 0, 1), 1); + EXPECT_EQ(Int64Int64Test(0, 1, 1, 0), -1); + EXPECT_EQ(Int64Int64Test(1, 0, 0, 1), 1); + EXPECT_EQ(Int64Int64Test(2, 4, 1, 7), 1); + EXPECT_EQ(Int64Int64Test(3, 7, 4, 0), -1); + EXPECT_EQ(Int64Int64Test(6, 4, 5, 7), 1); + EXPECT_EQ(Int64Int64Test(5, 5, 6, 4), -1); + EXPECT_EQ(Int64Int64Test(6, 1, 3, 7), 1); + EXPECT_EQ(Int64Int64Test(INT64_MAX / 2 + 2, 1, 1, INT64_MAX), 1); + EXPECT_EQ(Int64Int64Test(INT64_MAX / 2, 1, 1, INT64_MAX), -1); +} +TEST_F(TupleRowZOrderCompareTest, LargeIntTest) { + EXPECT_EQ(Int128Int128Test(0, 0, 0, 0), 0); + EXPECT_EQ(Int128Int128Test(-5, 3, -5, 3), 0); + EXPECT_EQ(Int128Int128Test(1, 0, 0, 1), 1); + EXPECT_EQ(Int128Int128Test(0, 1, 1, 0), -1); + EXPECT_EQ(Int128Int128Test(1, 0, 0, 1), 1); + EXPECT_EQ(Int128Int128Test(2, 4, 1, 7), 1); + EXPECT_EQ(Int128Int128Test(3, 7, 4, 0), -1); + EXPECT_EQ(Int128Int128Test(6, 4, 5, 7), 1); + EXPECT_EQ(Int128Int128Test(5, 5, 6, 4), -1); + EXPECT_EQ(Int128Int128Test(6, 1, 3, 7), 1); + EXPECT_EQ(Int128Int128Test(MAX_INT128 / 2 + 2, 1, 1, MAX_INT128), 1); + Int128Int128Test(MAX_INT128 / 2, 1, 1, MAX_INT128); +} + +} // namespace doris + +int main(int argc, char** argv) { + std::string conffile = std::string(getenv("DORIS_HOME")) + "/conf/be.conf"; + if (!doris::config::init(conffile.c_str(), false)) { + fprintf(stderr, "error read config file. \n"); + return -1; + } + doris::init_glog("be-test"); + ::testing::InitGoogleTest(&argc, argv); + doris::CpuInfo::init(); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md index 156fc5184d..72916ef337 100644 --- a/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md +++ b/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md @@ -297,6 +297,13 @@ distribution_info * `dynamic_partition.history_partition_num`: Specify the number of historical partitions to be created. * `dynamic_partition.reserved_history_periods`: Used to specify the range of reserved history periods. + * Data Sort Info + + The relevant parameters of data sort info are as follows: + + * `data_sort.sort_type`: the method of data sorting, options: z-order/lexical, default is lexical + * `data_sort.col_num`: the first few columns to sort, col_num muster less than total key counts + ### Example 1. Create a detailed model table diff --git a/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md index b240f0ab86..1b0229c024 100644 --- a/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md +++ b/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md @@ -298,6 +298,12 @@ distribution_info * `dynamic_partition.history_partition_num`: 指定创建历史分区的数量。 * `dynamic_partition.reserved_history_periods`: 用于指定保留的历史分区的时间段。 + * 数据排序相关 + + 数据排序相关参数如下: + + * `data_sort.sort_type`: 数据排序使用的方法，目前支持两种：lexical/z-order，默认是lexical + * `data_sort.col_num`: 数据排序使用的列数，取最前面几列，不能超过总的key 列数 ### Example 1. 创建一个明细模型的表 diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index ee8a049fe0..85eaaaf45b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -66,6 +66,7 @@ import org.apache.doris.persist.ModifyTableEngineOperationLog; import org.apache.doris.persist.ReplaceTableOperationLog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TOdbcTableType; +import org.apache.doris.thrift.TSortType; import org.apache.doris.thrift.TTabletType; import com.google.common.base.Preconditions; @@ -126,6 +127,10 @@ public class Alter { private boolean processAlterOlapTable(AlterTableStmt stmt, OlapTable olapTable, List alterClauses, final String clusterName, Database db) throws UserException { + if (olapTable.getDataSortInfo() != null + && olapTable.getDataSortInfo().getSortType() == TSortType.ZORDER) { + throw new UserException("z-order table can not support schema change!"); + } stmt.rewriteAlterClause(olapTable); // check conflict alter ops first diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataSortInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataSortInfo.java new file mode 100644 index 0000000000..5b33d25cf1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataSortInfo.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import com.google.gson.annotations.SerializedName; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.thrift.TSortType; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.Map; + +public class DataSortInfo implements Writable { + public static final String DATA_SORT_PROPERTY_PREFIX = "data_sort"; + public static final String DATA_SORT_TYPE = "data_sort.sort_type"; + public static final String DATA_SORT_COL_NUM = "data_sort.col_num"; + + @SerializedName(value = "sort_type") + private TSortType sortType; + @SerializedName(value = "col_num") + private int colNum; + + public DataSortInfo() { + + } + + public DataSortInfo(Map properties) { + if (properties != null && !properties.isEmpty()) { + if (properties.get(DATA_SORT_TYPE).equalsIgnoreCase("ZORDER")) { + this.sortType = TSortType.ZORDER; + } else { + this.sortType = TSortType.LEXICAL; + } + this.colNum = Integer.parseInt(properties.get(DATA_SORT_COL_NUM)); + } + } + + public DataSortInfo (TSortType sortType, int colNum) { + this.sortType = sortType; + this.colNum = colNum; + } + + public TSortType getSortType() { + return sortType; + } + + public void setSortType(TSortType sortType) { + this.sortType = sortType; + } + + public int getColNum() { + return colNum; + } + + public void setColNum(int colNum) { + this.colNum = colNum; + } + + @Override + public void write(DataOutput out) throws IOException { + String json = GsonUtils.GSON.toJson(this); + Text.writeString(out, json); + } + + public static DataSortInfo read(DataInput in) throws IOException { + String json = Text.readString(in); + return GsonUtils.GSON.fromJson(json, DataSortInfo.class); + } + + public boolean equals(DataSortInfo dataSortInfo) { + if (this.sortType != dataSortInfo.sortType) { + return false; + } + if (this.colNum != dataSortInfo.colNum) { + return false; + } + return true; + } + + public String toSql() { + String res = ",\n\"" + DATA_SORT_TYPE + "\" = \"" + this.sortType + "\"" + + ",\n\"" + DATA_SORT_COL_NUM + "\" = \"" + this.colNum + "\""; + return res; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index b56393bddd..b2cf59de88 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -93,6 +93,7 @@ import org.apache.doris.analysis.TypeDef; import org.apache.doris.analysis.UninstallPluginStmt; import org.apache.doris.analysis.UserDesc; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.analysis.DataSortInfo; import org.apache.doris.backup.BackupHandler; import org.apache.doris.blockrule.SqlBlockRuleMgr; import org.apache.doris.catalog.ColocateTableIndex.GroupId; @@ -3254,7 +3255,6 @@ public class Catalog { // create partition outside db lock DataProperty dataProperty = singlePartitionDesc.getPartitionDataProperty(); Preconditions.checkNotNull(dataProperty); - // check replica quota if this operation done long indexNum = indexIdToMeta.size(); long bucketNum = distributionInfo.getBucketNum(); @@ -3281,7 +3281,8 @@ public class Catalog { tabletIdSet, olapTable.getCopiedIndexes(), singlePartitionDesc.isInMemory(), olapTable.getStorageFormat(), - singlePartitionDesc.getTabletType() + singlePartitionDesc.getTabletType(), + olapTable.getDataSortInfo() ); // check again @@ -3512,7 +3513,8 @@ public class Catalog { List indexes, boolean isInMemory, TStorageFormat storageFormat, - TTabletType tabletType) throws DdlException { + TTabletType tabletType, + DataSortInfo dataSortInfo)throws DdlException { // create base index first. Preconditions.checkArgument(baseIndexId != -1); MaterializedIndex baseIndex = new MaterializedIndex(baseIndexId, IndexState.NORMAL); @@ -3579,7 +3581,8 @@ public class Catalog { countDownLatch, indexes, isInMemory, - tabletType); + tabletType, + dataSortInfo); task.setStorageFormat(storageFormat); batchTask.addTask(task); // add to AgentTaskQueue for handling finish report. @@ -3688,6 +3691,20 @@ public class Catalog { // this should be done before create partition. Map properties = stmt.getProperties(); + // get storage format + TStorageFormat storageFormat = TStorageFormat.V2; // default is segment v2 + try { + storageFormat = PropertyAnalyzer.analyzeStorageFormat(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + olapTable.setStorageFormat(storageFormat); + + // check data sort properties + DataSortInfo dataSortInfo = PropertyAnalyzer.analyzeDataSortInfo(properties, keysType, + keysDesc.keysColumnSize(), storageFormat); + olapTable.setDataSortInfo(dataSortInfo); + // analyze bloom filter columns Set bfColumns = null; double bfFpp = 0; @@ -3828,15 +3845,6 @@ public class Catalog { } Preconditions.checkNotNull(versionInfo); - // get storage format - TStorageFormat storageFormat = TStorageFormat.V2; // default is segment v2 - try { - storageFormat = PropertyAnalyzer.analyzeStorageFormat(properties); - } catch (AnalysisException e) { - throw new DdlException(e.getMessage()); - } - olapTable.setStorageFormat(storageFormat); - // a set to record every new tablet created when create table // if failed in any step, use this set to do clear things Set tabletIdSet = new HashSet<>(); @@ -3868,7 +3876,7 @@ public class Catalog { partitionInfo.getReplicaAllocation(partitionId), versionInfo, bfColumns, bfFpp, tabletIdSet, olapTable.getCopiedIndexes(), - isInMemory, storageFormat, tabletType); + isInMemory, storageFormat, tabletType, olapTable.getDataSortInfo()); olapTable.addPartition(partition); } else if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { try { @@ -3918,7 +3926,7 @@ public class Catalog { versionInfo, bfColumns, bfFpp, tabletIdSet, olapTable.getCopiedIndexes(), isInMemory, storageFormat, - partitionInfo.getTabletType(entry.getValue())); + partitionInfo.getTabletType(entry.getValue()), olapTable.getDataSortInfo()); olapTable.addPartition(partition); } } else { @@ -4235,6 +4243,11 @@ public class Catalog { sb.append(olapTable.getTableProperty().getDynamicPartitionProperty().getProperties(replicaAlloc)); } + // only display z-order sort info + if (olapTable.isZOrderSort()) { + sb.append(olapTable.getDataSortInfo().toSql()); + } + // in memory sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_INMEMORY).append("\" = \""); sb.append(olapTable.isInMemory()).append("\""); @@ -6761,7 +6774,8 @@ public class Catalog { copiedTbl.getCopiedIndexes(), copiedTbl.isInMemory(), copiedTbl.getStorageFormat(), - copiedTbl.getPartitionInfo().getTabletType(oldPartitionId)); + copiedTbl.getPartitionInfo().getTabletType(oldPartitionId), + copiedTbl.getDataSortInfo()); newPartitions.add(newPartition); } } catch (DdlException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 5fde029fd6..73fa24f3f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.DataSortInfo; import org.apache.doris.backup.Status; import org.apache.doris.backup.Status.ErrCode; import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; @@ -54,6 +55,7 @@ import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; +import org.apache.doris.thrift.TSortType; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -197,6 +199,12 @@ public class OlapTable extends Table { && tableProperty.getDynamicPartitionProperty().isExist(); } + public boolean isZOrderSort() { + return tableProperty != null + && tableProperty.getDataSortInfo() != null + && tableProperty.getDataSortInfo().getSortType() == TSortType.ZORDER; + } + public void setBaseIndexId(long baseIndexId) { this.baseIndexId = baseIndexId; } @@ -1263,7 +1271,6 @@ public class OlapTable extends Table { tempPartitions.unsetPartitionInfo(); } } - // In the present, the fullSchema could be rebuilt by schema change while the properties is changed by MV. // After that, some properties of fullSchema and nameToColumn may be not same as properties of base columns. // So, here we need to rebuild the fullSchema to ensure the correctness of the properties. @@ -1569,6 +1576,14 @@ public class OlapTable extends Table { tableProperty.buildInMemory(); } + public void setDataSortInfo(DataSortInfo dataSortInfo) { + if (tableProperty == null) { + tableProperty = new TableProperty(new HashMap<>()); + } + tableProperty.modifyDataSortInfoProperties(dataSortInfo); + tableProperty.buildDataSortInfo(); + } + // return true if partition with given name already exist, both in partitions and temp partitions. // return false otherwise public boolean checkPartitionNameExist(String partitionName) { @@ -1714,6 +1729,13 @@ public class OlapTable extends Table { return tableProperty.getStorageFormat(); } + public DataSortInfo getDataSortInfo() { + if (tableProperty == null) { + return new DataSortInfo(TSortType.LEXICAL, this.getKeysNum()); + } + return tableProperty.getDataSortInfo(); + } + // For non partitioned table: // The table's distribute hash columns need to be a subset of the aggregate columns. // diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index dcbaeba8e6..7d1c4cab36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -17,6 +17,7 @@ package org.apache.doris.catalog; +import org.apache.doris.analysis.DataSortInfo; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -69,6 +70,8 @@ public class TableProperty implements Writable { */ private TStorageFormat storageFormat = TStorageFormat.DEFAULT; + private DataSortInfo dataSortInfo = new DataSortInfo(); + public TableProperty(Map properties) { this.properties = properties; } @@ -127,6 +130,17 @@ public class TableProperty implements Writable { return this; } + public TableProperty buildDataSortInfo() { + HashMap dataSortInfoProperties = new HashMap<>(); + for (Map.Entry entry : properties.entrySet()) { + if (entry.getKey().startsWith(DataSortInfo.DATA_SORT_PROPERTY_PREFIX)) { + dataSortInfoProperties.put(entry.getKey(), entry.getValue()); + } + } + dataSortInfo = new DataSortInfo(dataSortInfoProperties); + return this; + } + public TableProperty buildStorageFormat() { storageFormat = TStorageFormat.valueOf(properties.getOrDefault(PropertyAnalyzer.PROPERTIES_STORAGE_FORMAT, TStorageFormat.DEFAULT.name())); @@ -137,6 +151,11 @@ public class TableProperty implements Writable { properties.putAll(modifyProperties); } + public void modifyDataSortInfoProperties(DataSortInfo dataSortInfo) { + properties.put(DataSortInfo.DATA_SORT_TYPE, String.valueOf(dataSortInfo.getSortType())); + properties.put(DataSortInfo.DATA_SORT_COL_NUM, String.valueOf(dataSortInfo.getColNum())); + } + public void setReplicaAlloc(ReplicaAllocation replicaAlloc) { this.replicaAlloc = replicaAlloc; // set it to "properties" so that this info can be persisted @@ -178,6 +197,10 @@ public class TableProperty implements Writable { return storageFormat; } + public DataSortInfo getDataSortInfo() { + return dataSortInfo; + } + public void buildReplicaAllocation() { try { // Must copy the properties because "analyzeReplicaAllocation" with remove the property @@ -200,7 +223,8 @@ public class TableProperty implements Writable { TableProperty tableProperty = GsonUtils.GSON.fromJson(Text.readString(in), TableProperty.class) .executeBuildDynamicProperty() .buildInMemory() - .buildStorageFormat(); + .buildStorageFormat() + .buildDataSortInfo(); if (Catalog.getCurrentCatalogJournalVersion() < FeMetaVersion.VERSION_105) { // get replica num from property map and create replica allocation String repNum = tableProperty.properties.remove(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 29e7c7f716..798b4d33f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -17,6 +17,7 @@ package org.apache.doris.common.util; +import org.apache.doris.analysis.DataSortInfo; import org.apache.doris.analysis.DateLiteral; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DataProperty; @@ -34,6 +35,7 @@ import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; import org.apache.doris.thrift.TTabletType; +import org.apache.doris.thrift.TSortType; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -534,4 +536,43 @@ public class PropertyAnalyzer { } return replicaAlloc; } + + public static DataSortInfo analyzeDataSortInfo(Map properties, KeysType keyType, + int keyCount, TStorageFormat storageFormat) throws AnalysisException { + if (properties == null || properties.isEmpty()) { + return new DataSortInfo(TSortType.LEXICAL, keyCount); + } + String sortMethod = TSortType.LEXICAL.name(); + if (properties.containsKey(DataSortInfo.DATA_SORT_TYPE)) { + sortMethod = properties.remove(DataSortInfo.DATA_SORT_TYPE); + } + TSortType sortType = TSortType.LEXICAL; + if (sortMethod.equalsIgnoreCase(TSortType.ZORDER.name())) { + sortType = TSortType.ZORDER; + } else if (sortMethod.equalsIgnoreCase(TSortType.LEXICAL.name())) { + sortType = TSortType.LEXICAL; + } else { + throw new AnalysisException("only support zorder/lexical method!"); + } + if (keyType != KeysType.DUP_KEYS && sortType == TSortType.ZORDER) { + throw new AnalysisException("only duplicate key supports zorder method!"); + } + if (storageFormat != TStorageFormat.V2 && sortType == TSortType.ZORDER) { + throw new AnalysisException("only V2 storage format supports zorder method!"); + } + + int colNum = keyCount; + if (properties.containsKey(DataSortInfo.DATA_SORT_COL_NUM)) { + try { + colNum = Integer.valueOf(properties.remove(DataSortInfo.DATA_SORT_COL_NUM)); + } catch (Exception e) { + throw new AnalysisException("param " + DataSortInfo.DATA_SORT_COL_NUM + " error"); + } + } + if (sortType == TSortType.ZORDER && (colNum <= 1 || colNum > keyCount)) { + throw new AnalysisException("z-order needs 2 columns at least, " + keyCount + " columns at most!"); + } + DataSortInfo dataSortInfo = new DataSortInfo(sortType, colNum); + return dataSortInfo; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index 04055d7d84..858f7ddf8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -18,6 +18,7 @@ package org.apache.doris.task; import org.apache.doris.alter.SchemaChangeHandler; +import org.apache.doris.analysis.DataSortInfo; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Index; import org.apache.doris.catalog.KeysType; @@ -82,6 +83,8 @@ public class CreateReplicaTask extends AgentTask { // true if this task is created by recover request(See comment of Config.recover_with_empty_tablet) private boolean isRecoverTask = false; + private DataSortInfo dataSortInfo; + public CreateReplicaTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, short shortKeyColumnCount, int schemaHash, long version, long versionHash, KeysType keysType, TStorageType storageType, @@ -114,6 +117,40 @@ public class CreateReplicaTask extends AgentTask { this.tabletType = tabletType; } + public CreateReplicaTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, + short shortKeyColumnCount, int schemaHash, long version, long versionHash, + KeysType keysType, TStorageType storageType, + TStorageMedium storageMedium, List columns, + Set bfColumns, double bfFpp, MarkedCountDownLatch latch, + List indexes, + boolean isInMemory, + TTabletType tabletType, + DataSortInfo dataSortInfo) { + super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId); + + this.shortKeyColumnCount = shortKeyColumnCount; + this.schemaHash = schemaHash; + + this.version = version; + this.versionHash = versionHash; + + this.keysType = keysType; + this.storageType = storageType; + this.storageMedium = storageMedium; + + this.columns = columns; + + this.bfColumns = bfColumns; + this.indexes = indexes; + this.bfFpp = bfFpp; + + this.latch = latch; + + this.isInMemory = isInMemory; + this.tabletType = tabletType; + this.dataSortInfo = dataSortInfo; + } + public void setIsRecoverTask(boolean isRecoverTask) { this.isRecoverTask = isRecoverTask; } @@ -165,6 +202,10 @@ public class CreateReplicaTask extends AgentTask { tSchema.setSchemaHash(schemaHash); tSchema.setKeysType(keysType.toThrift()); tSchema.setStorageType(storageType); + if (dataSortInfo != null) { + tSchema.setSortType(dataSortInfo.getSortType()); + tSchema.setSortColNum(dataSortInfo.getColNum()); + } int deleteSign = -1; int sequenceCol = -1; List tColumns = new ArrayList(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java index 4db3053096..24b3c89bbc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java @@ -478,4 +478,45 @@ public class CreateTableTest { ");")); } + + @Test + public void testZOrderTable() { + // create lexically sort table + ExceptionChecker.expectThrowsNoException(() -> createTable( + "create table test.zorder_tbl1\n" + "(k1 varchar(40), k2 int, k3 int)\n" + "duplicate key(k1, k2, k3)\n" + + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1'," + + " 'data_sort.sort_type' = 'lexical');")); + + // create z-order sort table, default col_num + ExceptionChecker.expectThrowsNoException(() -> createTable( + "create table test.zorder_tbl2\n" + "(k1 varchar(40), k2 int, k3 int)\n" + "duplicate key(k1, k2, k3)\n" + + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1'," + + " 'data_sort.sort_type' = 'zorder');")); + + // create z-order sort table, define sort_col_num + ExceptionChecker.expectThrowsNoException(() -> createTable( + "create table test.zorder_tbl3\n" + "(k1 varchar(40), k2 int, k3 int)\n" + "duplicate key(k1, k2, k3)\n" + + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1'," + + " 'data_sort.sort_type' = 'zorder'," + + " 'data_sort.col_num' = '2');")); + // create z-order sort table, only 1 sort column + ExceptionChecker + .expectThrowsWithMsg(AnalysisException.class, "z-order needs 2 columns at least, 3 columns at most", + () -> createTable("create table test.zorder_tbl4\n" + "(k1 varchar(40), k2 int, k3 int)\n" + "duplicate key(k1, k2, k3)\n" + + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1'," + + " 'data_sort.sort_type' = 'zorder'," + + " 'data_sort.col_num' = '1');")); + // create z-order sort table, sort column is empty + ExceptionChecker + .expectThrowsWithMsg(AnalysisException.class, "param data_sort.col_num error", + () -> createTable("create table test.zorder_tbl4\n" + "(k1 varchar(40), k2 int, k3 int)\n" + "duplicate key(k1, k2, k3)\n" + + "partition by range(k2)\n" + "(partition p1 values less than(\"10\"))\n" + + "distributed by hash(k1) buckets 1\n" + "properties('replication_num' = '1'," + + " 'data_sort.sort_type' = 'zorder'," + + " 'data_sort.col_num' = '');")); + } } diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 622db02fb3..96e4c2e582 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -262,6 +262,11 @@ message ColumnPB { repeated string children_column_names = 18; } +enum SortType { + LEXICAL = 0; + ZORDER = 1; +} + message TabletSchemaPB { optional KeysType keys_type = 1; // OLAPHeaderMessage.keys_type repeated ColumnPB column = 2; // OLAPHeaderMessage.column @@ -273,6 +278,8 @@ message TabletSchemaPB { optional bool is_in_memory = 8 [default=false]; optional int32 delete_sign_idx = 9 [default = -1]; optional int32 sequence_col_idx = 10 [default= -1]; + optional SortType sort_type = 11; + optional int32 sort_col_num = 12; } enum TabletStatePB { diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index a18d6fd21e..3a98214400 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -49,6 +49,8 @@ struct TTabletSchema { 8: optional bool is_in_memory 9: optional i32 delete_sign_idx = -1 10: optional i32 sequence_col_idx = -1 + 11: optional Types.TSortType sort_type + 12: optional i32 sort_col_num } // this enum stands for different storage format in src_backends diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index efa657a559..506a97ea1d 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -424,6 +424,11 @@ enum TMergeType { DELETE } +enum TSortType { + LEXICAL, + ZORDER, +} + // represent a user identity struct TUserIdentity { 1: optional string username