diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index b57e6dcdc5..47d607518e 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -660,7 +660,8 @@ Status Compaction::modify_rowsets(const Merger::Statistics* stats) { output_rowsets.push_back(_output_rowset); if (_tablet->keys_type() == KeysType::UNIQUE_KEYS && - _tablet->enable_unique_key_merge_on_write()) { + _tablet->enable_unique_key_merge_on_write() && + _tablet->tablet_schema()->cluster_key_idxes().empty()) { Version version = _tablet->max_version(); DeleteBitmap output_rowset_delete_bitmap(_tablet->tablet_id()); std::set missed_rows; diff --git a/be/src/olap/delete_bitmap_calculator.cpp b/be/src/olap/delete_bitmap_calculator.cpp index 86d82783e4..6f6e0ec888 100644 --- a/be/src/olap/delete_bitmap_calculator.cpp +++ b/be/src/olap/delete_bitmap_calculator.cpp @@ -92,27 +92,31 @@ bool MergeIndexDeleteBitmapCalculatorContext::Comparator::operator()( Slice key1, key2; RETURN_IF_ERROR(lhs->get_current_key(key1)); RETURN_IF_ERROR(rhs->get_current_key(key2)); - if (_sequence_length == 0) { + if (_sequence_length == 0 && _rowid_length == 0) { auto cmp_result = key1.compare(key2); // when key1 is the same as key2, // we want the one with greater segment id to be popped first return cmp_result ? (cmp_result > 0) : (lhs->segment_id() < rhs->segment_id()); } // smaller key popped first - auto key1_without_seq = Slice(key1.get_data(), key1.get_size() - _sequence_length); - auto key2_without_seq = Slice(key2.get_data(), key2.get_size() - _sequence_length); + auto key1_without_seq = + Slice(key1.get_data(), key1.get_size() - _sequence_length - _rowid_length); + auto key2_without_seq = + Slice(key2.get_data(), key2.get_size() - _sequence_length - _rowid_length); auto cmp_result = key1_without_seq.compare(key2_without_seq); if (cmp_result != 0) { return cmp_result > 0; } - // greater sequence value popped first - auto key1_sequence_val = - Slice(key1.get_data() + key1.get_size() - _sequence_length, _sequence_length); - auto key2_sequence_val = - Slice(key2.get_data() + key2.get_size() - _sequence_length, _sequence_length); - cmp_result = key1_sequence_val.compare(key2_sequence_val); - if (cmp_result != 0) { - return cmp_result < 0; + if (_sequence_length > 0) { + // greater sequence value popped first + auto key1_sequence_val = + Slice(key1.get_data() + key1_without_seq.get_size() + 1, _sequence_length - 1); + auto key2_sequence_val = + Slice(key2.get_data() + key2_without_seq.get_size() + 1, _sequence_length - 1); + cmp_result = key1_sequence_val.compare(key2_sequence_val); + if (cmp_result != 0) { + return cmp_result < 0; + } } // greater segment id popped first return lhs->segment_id() < rhs->segment_id(); @@ -120,19 +124,22 @@ bool MergeIndexDeleteBitmapCalculatorContext::Comparator::operator()( bool MergeIndexDeleteBitmapCalculatorContext::Comparator::is_key_same(Slice const& lhs, Slice const& rhs) const { - DCHECK(lhs.get_size() >= _sequence_length); - DCHECK(rhs.get_size() >= _sequence_length); - auto lhs_without_seq = Slice(lhs.get_data(), lhs.get_size() - _sequence_length); - auto rhs_without_seq = Slice(rhs.get_data(), rhs.get_size() - _sequence_length); + DCHECK(lhs.get_size() >= _sequence_length + _rowid_length); + DCHECK(rhs.get_size() >= _sequence_length + _rowid_length); + auto lhs_without_seq = Slice(lhs.get_data(), lhs.get_size() - _sequence_length - _rowid_length); + auto rhs_without_seq = Slice(rhs.get_data(), rhs.get_size() - _sequence_length - _rowid_length); return lhs_without_seq.compare(rhs_without_seq) == 0; } Status MergeIndexDeleteBitmapCalculator::init(RowsetId rowset_id, std::vector const& segments, - size_t seq_col_length, size_t max_batch_size) { + size_t seq_col_length, size_t rowdid_length, + size_t max_batch_size) { _rowset_id = rowset_id; _seq_col_length = seq_col_length; - _comparator = MergeIndexDeleteBitmapCalculatorContext::Comparator(seq_col_length); + _rowid_length = rowdid_length; + _comparator = + MergeIndexDeleteBitmapCalculatorContext::Comparator(seq_col_length, _rowid_length); _contexts.reserve(segments.size()); _heap = std::make_unique(_comparator); @@ -146,6 +153,10 @@ Status MergeIndexDeleteBitmapCalculator::init(RowsetId rowset_id, _contexts.emplace_back(std::move(index), index_type, segment->id(), pk_idx->num_rows()); _heap->push(&_contexts.back()); } + if (_rowid_length > 0) { + _rowid_coder = get_key_coder( + get_scalar_type_info()->type()); + } return Status::OK(); } @@ -159,6 +170,15 @@ Status MergeIndexDeleteBitmapCalculator::calculate_one(RowLocation& loc) { if (!_last_key.empty() && _comparator.is_key_same(cur_key, _last_key)) { loc.segment_id = cur_ctx->segment_id(); loc.row_id = cur_ctx->row_id(); + if (_rowid_length > 0) { + Slice key_without_seq = Slice(cur_key.get_data(), + cur_key.get_size() - _seq_col_length - _rowid_length); + Slice rowid_slice = + Slice(cur_key.get_data() + key_without_seq.get_size() + _seq_col_length + 1, + _rowid_length - 1); + RETURN_IF_ERROR(_rowid_coder->decode_ascending(&rowid_slice, _rowid_length, + (uint8_t*)&loc.row_id)); + } auto st = cur_ctx->advance(); if (st.ok()) { _heap->push(cur_ctx); @@ -176,8 +196,9 @@ Status MergeIndexDeleteBitmapCalculator::calculate_one(RowLocation& loc) { RETURN_IF_ERROR(nxt_ctx->get_current_key(nxt_key)); Status st = _comparator.is_key_same(cur_key, nxt_key) ? cur_ctx->advance() - : cur_ctx->seek_at_or_after(Slice( - nxt_key.get_data(), nxt_key.get_size() - _seq_col_length)); + : cur_ctx->seek_at_or_after( + Slice(nxt_key.get_data(), + nxt_key.get_size() - _seq_col_length - _rowid_length)); if (st.is()) { continue; } diff --git a/be/src/olap/delete_bitmap_calculator.h b/be/src/olap/delete_bitmap_calculator.h index dd17fe7b68..4242197279 100644 --- a/be/src/olap/delete_bitmap_calculator.h +++ b/be/src/olap/delete_bitmap_calculator.h @@ -29,6 +29,7 @@ #include "olap/base_tablet.h" #include "olap/binlog_config.h" #include "olap/data_dir.h" +#include "olap/key_coder.h" #include "olap/olap_common.h" #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_meta.h" @@ -47,13 +48,15 @@ class MergeIndexDeleteBitmapCalculatorContext { public: class Comparator { public: - Comparator(size_t sequence_length) : _sequence_length(sequence_length) {} + Comparator(size_t sequence_length, size_t rowid_length) + : _sequence_length(sequence_length), _rowid_length(rowid_length) {} bool operator()(MergeIndexDeleteBitmapCalculatorContext* lhs, MergeIndexDeleteBitmapCalculatorContext* rhs) const; bool is_key_same(Slice const& lhs, Slice const& rhs) const; private: size_t _sequence_length; + size_t _rowid_length; }; MergeIndexDeleteBitmapCalculatorContext(std::unique_ptr iter, @@ -90,7 +93,7 @@ public: MergeIndexDeleteBitmapCalculator() = default; Status init(RowsetId rowset_id, std::vector const& segments, - size_t seq_col_length = 0, size_t max_batch_size = 1024); + size_t seq_col_length = 0, size_t rowid_length = 0, size_t max_batch_size = 1024); Status calculate_one(RowLocation& loc); @@ -101,11 +104,13 @@ private: std::vector, MergeIndexDeleteBitmapCalculatorContext::Comparator>; std::vector _contexts; - MergeIndexDeleteBitmapCalculatorContext::Comparator _comparator {0}; + MergeIndexDeleteBitmapCalculatorContext::Comparator _comparator {0, 0}; RowsetId _rowset_id; std::unique_ptr _heap; std::string _last_key; size_t _seq_col_length; + size_t _rowid_length; + const KeyCoder* _rowid_coder = nullptr; }; } // namespace doris diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index d163abd26a..093c241573 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -290,6 +290,56 @@ size_t MemTable::_sort() { return same_keys_num; } +void MemTable::_sort_by_cluster_keys() { + SCOPED_RAW_TIMER(&_stat.sort_ns); + _stat.sort_times++; + // sort all rows + vectorized::Block in_block = _output_mutable_block.to_block(); + vectorized::MutableBlock mutable_block = + vectorized::MutableBlock::build_mutable_block(&in_block); + auto clone_block = in_block.clone_without_columns(); + _output_mutable_block = vectorized::MutableBlock::build_mutable_block(&clone_block); + + std::vector row_in_blocks; + std::unique_ptr> row_in_blocks_deleter((int*)0x01, [&](int*) { + std::for_each(row_in_blocks.begin(), row_in_blocks.end(), + std::default_delete()); + }); + row_in_blocks.reserve(mutable_block.rows()); + for (size_t i = 0; i < mutable_block.rows(); i++) { + row_in_blocks.emplace_back(new RowInBlock {i}); + } + Tie tie = Tie(0, mutable_block.rows()); + + for (auto i : _tablet_schema->cluster_key_idxes()) { + auto cmp = [&](const RowInBlock* lhs, const RowInBlock* rhs) -> int { + return mutable_block.compare_one_column(lhs->_row_pos, rhs->_row_pos, i, -1); + }; + _sort_one_column(row_in_blocks, tie, cmp); + } + + // sort extra round by _row_pos to make the sort stable + auto iter = tie.iter(); + while (iter.next()) { + pdqsort(std::next(row_in_blocks.begin(), iter.left()), + std::next(row_in_blocks.begin(), iter.right()), + [](const RowInBlock* lhs, const RowInBlock* rhs) -> bool { + return lhs->_row_pos < rhs->_row_pos; + }); + } + + in_block = mutable_block.to_block(); + SCOPED_RAW_TIMER(&_stat.put_into_output_ns); + std::vector row_pos_vec; + DCHECK(in_block.rows() <= std::numeric_limits::max()); + row_pos_vec.reserve(in_block.rows()); + for (int i = 0; i < row_in_blocks.size(); i++) { + row_pos_vec.emplace_back(row_in_blocks[i]->_row_pos); + } + _output_mutable_block.add_rows(&in_block, row_pos_vec.data(), + row_pos_vec.data() + in_block.rows()); +} + void MemTable::_sort_one_column(std::vector& row_in_blocks, Tie& tie, std::function cmp) { auto iter = tie.iter(); @@ -448,6 +498,10 @@ std::unique_ptr MemTable::to_block() { } else { _aggregate(); } + if (_keys_type == KeysType::UNIQUE_KEYS && _enable_unique_key_mow && + !_tablet_schema->cluster_key_idxes().empty()) { + _sort_by_cluster_keys(); + } return vectorized::Block::create_unique(_output_mutable_block.to_block()); } diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h index cade509aac..a5289dbaf6 100644 --- a/be/src/olap/memtable.h +++ b/be/src/olap/memtable.h @@ -244,6 +244,7 @@ private: //return number of same keys size_t _sort(); + void _sort_by_cluster_keys(); void _sort_one_column(std::vector& row_in_blocks, Tie& tie, std::function cmp); template diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index 82f1662d01..9a57bcd387 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -79,6 +79,9 @@ Status Merger::vmerge_rowsets(TabletSharedPtr tablet, ReaderType reader_type, merge_tablet_schema->merge_dropped_columns(*del_pred_rs->tablet_schema()); } reader_params.tablet_schema = merge_tablet_schema; + if (!tablet->tablet_schema()->cluster_key_idxes().empty()) { + reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap(); + } if (stats_output && stats_output->rowid_conversion) { reader_params.record_rowids = true; @@ -164,6 +167,13 @@ void Merger::vertical_split_columns(TabletSchemaSPtr tablet_schema, if (delete_sign_idx != -1) { key_columns.emplace_back(delete_sign_idx); } + if (!tablet_schema->cluster_key_idxes().empty()) { + for (const auto& cid : tablet_schema->cluster_key_idxes()) { + if (cid >= num_key_cols) { + key_columns.emplace_back(cid); + } + } + } } VLOG_NOTICE << "sequence_col_idx=" << sequence_col_idx << ", delete_sign_idx=" << delete_sign_idx; @@ -171,9 +181,9 @@ void Merger::vertical_split_columns(TabletSchemaSPtr tablet_schema, if (!key_columns.empty()) { column_groups->emplace_back(std::move(key_columns)); } - std::vector value_columns; for (auto i = num_key_cols; i < total_cols; ++i) { - if (i == sequence_col_idx || i == delete_sign_idx) { + if (i == sequence_col_idx || i == delete_sign_idx || + key_columns.end() != std::find(key_columns.begin(), key_columns.end(), i)) { continue; } if ((i - num_key_cols) % config::vertical_compaction_num_columns_per_group == 0) { @@ -187,12 +197,14 @@ Status Merger::vertical_compact_one_group( TabletSharedPtr tablet, ReaderType reader_type, TabletSchemaSPtr tablet_schema, bool is_key, const std::vector& column_group, vectorized::RowSourcesBuffer* row_source_buf, const std::vector& src_rowset_readers, - RowsetWriter* dst_rowset_writer, int64_t max_rows_per_segment, Statistics* stats_output) { + RowsetWriter* dst_rowset_writer, int64_t max_rows_per_segment, Statistics* stats_output, + std::vector key_group_cluster_key_idxes) { // build tablet reader VLOG_NOTICE << "vertical compact one group, max_rows_per_segment=" << max_rows_per_segment; vectorized::VerticalBlockReader reader(row_source_buf); TabletReader::ReaderParams reader_params; reader_params.is_key_column_group = is_key; + reader_params.key_group_cluster_key_idxes = key_group_cluster_key_idxes; reader_params.tablet = tablet; reader_params.reader_type = reader_type; @@ -214,6 +226,9 @@ Status Merger::vertical_compact_one_group( } reader_params.tablet_schema = merge_tablet_schema; + if (!tablet->tablet_schema()->cluster_key_idxes().empty()) { + reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap(); + } if (is_key && stats_output && stats_output->rowid_conversion) { reader_params.record_rowids = true; @@ -341,6 +356,10 @@ Status Merger::vertical_merge_rowsets(TabletSharedPtr tablet, ReaderType reader_ std::vector> column_groups; vertical_split_columns(tablet_schema, &column_groups); + std::vector key_group_cluster_key_idxes; + _generate_key_group_cluster_key_idxes(tablet_schema, column_groups, + key_group_cluster_key_idxes); + vectorized::RowSourcesBuffer row_sources_buf(tablet->tablet_id(), tablet->tablet_path(), reader_type); // compact group one by one @@ -349,7 +368,8 @@ Status Merger::vertical_merge_rowsets(TabletSharedPtr tablet, ReaderType reader_ bool is_key = (i == 0); RETURN_IF_ERROR(vertical_compact_one_group( tablet, reader_type, tablet_schema, is_key, column_groups[i], &row_sources_buf, - src_rowset_readers, dst_rowset_writer, max_rows_per_segment, stats_output)); + src_rowset_readers, dst_rowset_writer, max_rows_per_segment, stats_output, + key_group_cluster_key_idxes)); if (is_key) { RETURN_IF_ERROR(row_sources_buf.flush()); } @@ -363,4 +383,21 @@ Status Merger::vertical_merge_rowsets(TabletSharedPtr tablet, ReaderType reader_ return Status::OK(); } +void Merger::_generate_key_group_cluster_key_idxes( + TabletSchemaSPtr tablet_schema, std::vector>& column_groups, + std::vector& key_group_cluster_key_idxes) { + if (column_groups.size() > 0 && !tablet_schema->cluster_key_idxes().empty()) { + auto& key_column_group = column_groups[0]; + for (const auto& index_in_tablet_schema : tablet_schema->cluster_key_idxes()) { + for (auto j = 0; j < key_column_group.size(); ++j) { + auto cid = key_column_group[j]; + if (cid == index_in_tablet_schema) { + key_group_cluster_key_idxes.emplace_back(j); + break; + } + } + } + } +} + } // namespace doris diff --git a/be/src/olap/merger.h b/be/src/olap/merger.h index 37291c548c..ab948f55ed 100644 --- a/be/src/olap/merger.h +++ b/be/src/olap/merger.h @@ -74,8 +74,8 @@ public: bool is_key, const std::vector& column_group, vectorized::RowSourcesBuffer* row_source_buf, const std::vector& src_rowset_readers, - RowsetWriter* dst_rowset_writer, int64_t max_rows_per_segment, - Statistics* stats_output); + RowsetWriter* dst_rowset_writer, int64_t max_rows_per_segment, Statistics* stats_output, + std::vector key_group_cluster_key_idxes); // for segcompaction static Status vertical_compact_one_group(TabletSharedPtr tablet, ReaderType reader_type, @@ -86,6 +86,12 @@ public: segment_v2::SegmentWriter& dst_segment_writer, int64_t max_rows_per_segment, Statistics* stats_output, uint64_t* index_size, KeyBoundsPB& key_bounds); + + // for mow with cluster key table, the key group also contains cluster key columns. + // the `key_group_cluster_key_idxes` marks the positions of cluster key columns in key group. + static void _generate_key_group_cluster_key_idxes( + TabletSchemaSPtr tablet_schema, std::vector>& column_groups, + std::vector& key_group_cluster_key_idxes); }; } // namespace doris diff --git a/be/src/olap/primary_key_index.cpp b/be/src/olap/primary_key_index.cpp index cc39441220..b807b249a7 100644 --- a/be/src/olap/primary_key_index.cpp +++ b/be/src/olap/primary_key_index.cpp @@ -54,7 +54,7 @@ Status PrimaryKeyIndexBuilder::init() { Status PrimaryKeyIndexBuilder::add_item(const Slice& key) { RETURN_IF_ERROR(_primary_key_index_builder->add(&key)); - Slice key_without_seq = Slice(key.get_data(), key.get_size() - _seq_col_length); + Slice key_without_seq = Slice(key.get_data(), key.get_size() - _seq_col_length - _rowid_length); _bloom_filter_index_builder->add_values(&key_without_seq, 1); // the key is already sorted, so the first key is min_key, and // the last key is max_key. diff --git a/be/src/olap/primary_key_index.h b/be/src/olap/primary_key_index.h index 59b88c2f72..3fda3a763e 100644 --- a/be/src/olap/primary_key_index.h +++ b/be/src/olap/primary_key_index.h @@ -50,12 +50,13 @@ class PrimaryKeyIndexMetaPB; // NOTE: for now, it's only used when unique key merge-on-write property enabled. class PrimaryKeyIndexBuilder { public: - PrimaryKeyIndexBuilder(io::FileWriter* file_writer, size_t seq_col_length) + PrimaryKeyIndexBuilder(io::FileWriter* file_writer, size_t seq_col_length, size_t rowid_length) : _file_writer(file_writer), _num_rows(0), _size(0), _disk_size(0), - _seq_col_length(seq_col_length) {} + _seq_col_length(seq_col_length), + _rowid_length(rowid_length) {} Status init(); @@ -70,8 +71,12 @@ public: // used for be ut uint32_t data_page_num() const { return _primary_key_index_builder->data_page_num(); } - Slice min_key() { return Slice(_min_key.data(), _min_key.size() - _seq_col_length); } - Slice max_key() { return Slice(_max_key.data(), _max_key.size() - _seq_col_length); } + Slice min_key() { + return Slice(_min_key.data(), _min_key.size() - _seq_col_length - _rowid_length); + } + Slice max_key() { + return Slice(_max_key.data(), _max_key.size() - _seq_col_length - _rowid_length); + } Status finalize(segment_v2::PrimaryKeyIndexMetaPB* meta); @@ -81,6 +86,7 @@ private: uint64_t _size; uint64_t _disk_size; size_t _seq_col_length; + size_t _rowid_length; faststring _min_key; faststring _max_key; @@ -129,6 +135,8 @@ public: return _index_reader->get_memory_size(); } + static constexpr size_t ROW_ID_LENGTH = sizeof(uint32_t) + 1; + private: bool _index_parsed; bool _bf_parsed; diff --git a/be/src/olap/reader.h b/be/src/olap/reader.h index b0d5ed1fa2..044eff7808 100644 --- a/be/src/olap/reader.h +++ b/be/src/olap/reader.h @@ -172,6 +172,7 @@ public: // for vertical compaction bool is_key_column_group = false; + std::vector key_group_cluster_key_idxes; bool is_segcompaction = false; diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index f3db8689a3..62c32c9a57 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -360,7 +360,10 @@ bool BetaRowsetWriter::_check_and_set_is_doing_segcompaction() { Status BetaRowsetWriter::_segcompaction_if_necessary() { Status status = Status::OK(); + // leave _check_and_set_is_doing_segcompaction as the last condition + // otherwise _segcompacting_cond will never get notified if (!config::enable_segcompaction || !_context.enable_segcompaction || + !_context.tablet_schema->cluster_key_idxes().empty() || !_check_and_set_is_doing_segcompaction()) { return status; } diff --git a/be/src/olap/rowset/segment_creator.cpp b/be/src/olap/rowset/segment_creator.cpp index 043fcff876..eb8173609c 100644 --- a/be/src/olap/rowset/segment_creator.cpp +++ b/be/src/olap/rowset/segment_creator.cpp @@ -51,7 +51,8 @@ Status SegmentFlusher::flush_single_block(const vectorized::Block* block, int32_ return Status::OK(); } bool no_compression = block->bytes() <= config::segment_compression_threshold_kb * 1024; - if (config::enable_vertical_segment_writer) { + if (config::enable_vertical_segment_writer && + _context.tablet_schema->cluster_key_idxes().empty()) { std::unique_ptr writer; RETURN_IF_ERROR(_create_segment_writer(writer, segment_id, no_compression, flush_schema)); RETURN_IF_ERROR(_add_rows(writer, block, 0, block->rows())); diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index bf41f3a12e..a8c2c095e1 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -401,16 +401,20 @@ Status Segment::new_inverted_index_iterator(const TabletColumn& tablet_column, return Status::OK(); } -Status Segment::lookup_row_key(const Slice& key, bool with_seq_col, RowLocation* row_location) { +Status Segment::lookup_row_key(const Slice& key, bool with_seq_col, bool with_rowid, + RowLocation* row_location) { RETURN_IF_ERROR(load_pk_index_and_bf()); bool has_seq_col = _tablet_schema->has_sequence_col(); + bool has_rowid = !_tablet_schema->cluster_key_idxes().empty(); size_t seq_col_length = 0; if (has_seq_col) { seq_col_length = _tablet_schema->column(_tablet_schema->sequence_col_idx()).length() + 1; } + size_t rowid_length = has_rowid ? PrimaryKeyIndexReader::ROW_ID_LENGTH : 0; Slice key_without_seq = - Slice(key.get_data(), key.get_size() - (with_seq_col ? seq_col_length : 0)); + Slice(key.get_data(), key.get_size() - (with_seq_col ? seq_col_length : 0) - + (with_rowid ? rowid_length : 0)); DCHECK(_pk_index_reader != nullptr); if (!_pk_index_reader->check_present(key_without_seq)) { @@ -423,46 +427,64 @@ Status Segment::lookup_row_key(const Slice& key, bool with_seq_col, RowLocation* if (!st.ok() && !st.is()) { return st; } - if (st.is() || (!has_seq_col && !exact_match)) { + if (st.is() || (!has_seq_col && !has_rowid && !exact_match)) { return Status::Error("Can't find key in the segment"); } row_location->row_id = index_iterator->get_current_ordinal(); row_location->segment_id = _segment_id; row_location->rowset_id = _rowset_id; - if (has_seq_col) { - size_t num_to_read = 1; - auto index_type = vectorized::DataTypeFactory::instance().create_data_type( - _pk_index_reader->type_info()->type(), 1, 0); - auto index_column = index_type->create_column(); - size_t num_read = num_to_read; - RETURN_IF_ERROR(index_iterator->next_batch(&num_read, index_column)); - DCHECK(num_to_read == num_read); + size_t num_to_read = 1; + auto index_type = vectorized::DataTypeFactory::instance().create_data_type( + _pk_index_reader->type_info()->type(), 1, 0); + auto index_column = index_type->create_column(); + size_t num_read = num_to_read; + RETURN_IF_ERROR(index_iterator->next_batch(&num_read, index_column)); + DCHECK(num_to_read == num_read); - Slice sought_key = - Slice(index_column->get_data_at(0).data, index_column->get_data_at(0).size); + Slice sought_key = Slice(index_column->get_data_at(0).data, index_column->get_data_at(0).size); + + if (has_seq_col) { Slice sought_key_without_seq = - Slice(sought_key.get_data(), sought_key.get_size() - seq_col_length); + Slice(sought_key.get_data(), sought_key.get_size() - seq_col_length - rowid_length); // compare key if (key_without_seq.compare(sought_key_without_seq) != 0) { return Status::Error("Can't find key in the segment"); } - if (!with_seq_col) { - return Status::OK(); + if (with_seq_col) { + // compare sequence id + Slice sequence_id = + Slice(key.get_data() + key_without_seq.get_size() + 1, seq_col_length - 1); + Slice previous_sequence_id = + Slice(sought_key.get_data() + sought_key_without_seq.get_size() + 1, + seq_col_length - 1); + if (sequence_id.compare(previous_sequence_id) < 0) { + return Status::Error( + "key with higher sequence id exists"); + } } - - // compare sequence id - Slice sequence_id = - Slice(key.get_data() + key_without_seq.get_size() + 1, seq_col_length - 1); - Slice previous_sequence_id = Slice( - sought_key.get_data() + sought_key_without_seq.get_size() + 1, seq_col_length - 1); - if (sequence_id.compare(previous_sequence_id) < 0) { - return Status::Error( - "key with higher sequence id exists"); + } else if (has_rowid) { + Slice sought_key_without_rowid = + Slice(sought_key.get_data(), sought_key.get_size() - rowid_length); + // compare key + if (key_without_seq.compare(sought_key_without_rowid) != 0) { + return Status::Error("Can't find key in the segment"); } } + // found the key, use rowid in pk index if necessary. + if (has_rowid) { + Slice sought_key_without_seq = + Slice(sought_key.get_data(), sought_key.get_size() - seq_col_length - rowid_length); + Slice rowid_slice = Slice( + sought_key.get_data() + sought_key_without_seq.get_size() + seq_col_length + 1, + rowid_length - 1); + const auto* type_info = get_scalar_type_info(); + auto rowid_coder = get_key_coder(type_info->type()); + RETURN_IF_ERROR(rowid_coder->decode_ascending(&rowid_slice, rowid_length, + (uint8_t*)&row_location->row_id)); + } return Status::OK(); } @@ -479,7 +501,17 @@ Status Segment::read_key_by_rowid(uint32_t row_id, std::string* key) { size_t num_read = 1; RETURN_IF_ERROR(iter->next_batch(&num_read, index_column)); CHECK(num_read == 1); - *key = index_column->get_data_at(0).to_string(); + // trim row id + if (_tablet_schema->cluster_key_idxes().empty()) { + *key = index_column->get_data_at(0).to_string(); + } else { + Slice sought_key = + Slice(index_column->get_data_at(0).data, index_column->get_data_at(0).size); + Slice sought_key_without_rowid = + Slice(sought_key.get_data(), + sought_key.get_size() - PrimaryKeyIndexReader::ROW_ID_LENGTH); + *key = sought_key_without_rowid.to_string(); + } return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h index ea8ab43bd2..d24381a8fb 100644 --- a/be/src/olap/rowset/segment_v2/segment.h +++ b/be/src/olap/rowset/segment_v2/segment.h @@ -104,7 +104,8 @@ public: return _pk_index_reader.get(); } - Status lookup_row_key(const Slice& key, bool with_seq_col, RowLocation* row_location); + Status lookup_row_key(const Slice& key, bool with_seq_col, bool with_rowid, + RowLocation* row_location); Status read_key_by_rowid(uint32_t row_id, std::string* key); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 42772a21ae..d0ffcfdd6d 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -332,7 +332,8 @@ Status SegmentIterator::_lazy_init() { DorisMetrics::instance()->segment_read_total->increment(1); _row_bitmap.addRange(0, _segment->num_rows()); // z-order can not use prefix index - if (_segment->_tablet_schema->sort_type() != SortType::ZORDER) { + if (_segment->_tablet_schema->sort_type() != SortType::ZORDER && + _segment->_tablet_schema->cluster_key_idxes().empty()) { RETURN_IF_ERROR(_get_row_ranges_by_keys()); } RETURN_IF_ERROR(_get_row_ranges_by_column_conditions()); @@ -1324,6 +1325,10 @@ Status SegmentIterator::_lookup_ordinal_from_pk_index(const RowCursor& key, bool // The sequence column needs to be removed from primary key index when comparing key bool has_seq_col = _segment->_tablet_schema->has_sequence_col(); + // Used to get key range from primary key index, + // for mow with cluster key table, we should get key range from short key index. + DCHECK(_segment->_tablet_schema->cluster_key_idxes().empty()); + if (has_seq_col) { size_t seq_col_length = _segment->_tablet_schema->column(_segment->_tablet_schema->sequence_col_idx()) diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h index 352929678b..32d1336915 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.h +++ b/be/src/olap/rowset/segment_v2/segment_iterator.h @@ -163,6 +163,7 @@ private: [[nodiscard]] Status _lookup_ordinal(const RowCursor& key, bool is_include, rowid_t upper_bound, rowid_t* rowid); // lookup the ordinal of given key from short key index + // the returned rowid is rowid in primary index, not the rowid encoded in primary key [[nodiscard]] Status _lookup_ordinal_from_sk_index(const RowCursor& key, bool is_include, rowid_t upper_bound, rowid_t* rowid); // lookup the ordinal of given key from primary key index diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 329a7fa1e9..90663090f0 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -87,17 +87,40 @@ SegmentWriter::SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id, CHECK_NOTNULL(file_writer); _num_key_columns = _tablet_schema->num_key_columns(); _num_short_key_columns = _tablet_schema->num_short_key_columns(); - DCHECK(_num_key_columns >= _num_short_key_columns); + if (_tablet_schema->cluster_key_idxes().empty()) { + DCHECK(_num_key_columns >= _num_short_key_columns) + << ", table_id=" << _tablet_schema->table_id() + << ", num_key_columns=" << _num_key_columns + << ", num_short_key_columns=" << _num_short_key_columns + << ", cluster_key_columns=" << _tablet_schema->cluster_key_idxes().size(); + } for (size_t cid = 0; cid < _num_key_columns; ++cid) { const auto& column = _tablet_schema->column(cid); _key_coders.push_back(get_key_coder(column.type())); _key_index_size.push_back(column.index_length()); } - // encode the sequence id into the primary key index - if (_tablet_schema->has_sequence_col() && _tablet_schema->keys_type() == UNIQUE_KEYS && - _opts.enable_unique_key_merge_on_write) { - const auto& column = _tablet_schema->column(_tablet_schema->sequence_col_idx()); - _seq_coder = get_key_coder(column.type()); + if (_tablet_schema->keys_type() == UNIQUE_KEYS && _opts.enable_unique_key_merge_on_write) { + // encode the sequence id into the primary key index + if (_tablet_schema->has_sequence_col()) { + const auto& column = _tablet_schema->column(_tablet_schema->sequence_col_idx()); + _seq_coder = get_key_coder(column.type()); + } + // encode the rowid into the primary key index + if (!_tablet_schema->cluster_key_idxes().empty()) { + const auto* type_info = get_scalar_type_info(); + _rowid_coder = get_key_coder(type_info->type()); + // primary keys + _primary_key_coders.swap(_key_coders); + // cluster keys + _key_coders.clear(); + _key_index_size.clear(); + _num_key_columns = _tablet_schema->cluster_key_idxes().size(); + for (auto cid : _tablet_schema->cluster_key_idxes()) { + const auto& column = _tablet_schema->column(cid); + _key_coders.push_back(get_key_coder(column.type())); + _key_index_size.push_back(column.index_length()); + } + } } } @@ -227,8 +250,14 @@ Status SegmentWriter::init(const std::vector& col_ids, bool has_key) { seq_col_length = _tablet_schema->column(_tablet_schema->sequence_col_idx()).length() + 1; } + size_t rowid_length = 0; + if (!_tablet_schema->cluster_key_idxes().empty()) { + rowid_length = PrimaryKeyIndexReader::ROW_ID_LENGTH; + _short_key_index_builder.reset( + new ShortKeyIndexBuilder(_segment_id, _opts.num_rows_per_block)); + } _primary_key_index_builder.reset( - new PrimaryKeyIndexBuilder(_file_writer, seq_col_length)); + new PrimaryKeyIndexBuilder(_file_writer, seq_col_length, rowid_length)); RETURN_IF_ERROR(_primary_key_index_builder->init()); } else { _short_key_index_builder.reset( @@ -681,7 +710,7 @@ Status SegmentWriter::append_block(const vectorized::Block* block, size_t row_po return converted_result.first; } auto cid = _column_ids[id]; - if (_has_key && cid < _num_key_columns) { + if (_has_key && cid < _tablet_schema->num_key_columns()) { key_columns.push_back(converted_result.second); } else if (_has_key && _tablet_schema->has_sequence_col() && cid == _tablet_schema->sequence_col_idx()) { @@ -691,31 +720,38 @@ Status SegmentWriter::append_block(const vectorized::Block* block, size_t row_po converted_result.second->get_data(), num_rows)); } if (_has_key) { - if (_tablet_schema->keys_type() == UNIQUE_KEYS && _opts.enable_unique_key_merge_on_write) { - // create primary indexes - std::string last_key; - for (size_t pos = 0; pos < num_rows; pos++) { - std::string key = _full_encode_keys(key_columns, pos); - _maybe_invalid_row_cache(key); - if (_tablet_schema->has_sequence_col()) { - _encode_seq_column(seq_column, pos, &key); + // for now we don't need to query short key index for CLUSTER BY feature, + // but we still write the index for future usage. + bool need_primary_key_indexes = (_tablet_schema->keys_type() == UNIQUE_KEYS && + _opts.enable_unique_key_merge_on_write); + bool need_short_key_indexes = + !need_primary_key_indexes || + (need_primary_key_indexes && _tablet_schema->cluster_key_idxes().size() > 0); + if (need_primary_key_indexes && !need_short_key_indexes) { // mow table without cluster keys + RETURN_IF_ERROR(_generate_primary_key_index(_key_coders, key_columns, seq_column, + num_rows, false)); + } else if (!need_primary_key_indexes && need_short_key_indexes) { // other tables + RETURN_IF_ERROR(_generate_short_key_index(key_columns, num_rows, short_key_pos)); + } else if (need_primary_key_indexes && need_short_key_indexes) { // mow with cluster keys + // 1. generate primary key index, the key_columns is primary_key_columns + RETURN_IF_ERROR(_generate_primary_key_index(_primary_key_coders, key_columns, + seq_column, num_rows, true)); + // 2. generate short key index (use cluster key) + key_columns.clear(); + for (const auto& cid : _tablet_schema->cluster_key_idxes()) { + for (size_t id = 0; id < _column_writers.size(); ++id) { + // olap data convertor always start from id = 0 + if (cid == _column_ids[id]) { + auto converted_result = _olap_data_convertor->convert_column_data(id); + if (!converted_result.first.ok()) { + return converted_result.first; + } + key_columns.push_back(converted_result.second); + break; + } } - DCHECK(key.compare(last_key) > 0) - << "found duplicate key or key is not sorted! current key: " << key - << ", last key" << last_key; - RETURN_IF_ERROR(_primary_key_index_builder->add_item(key)); - last_key = std::move(key); - } - } else { - // create short key indexes' - // for min_max key - set_min_key(_full_encode_keys(key_columns, 0)); - set_max_key(_full_encode_keys(key_columns, num_rows - 1)); - - key_columns.resize(_num_short_key_columns); - for (const auto pos : short_key_pos) { - RETURN_IF_ERROR(_short_key_index_builder->add_item(_encode_keys(key_columns, pos))); } + RETURN_IF_ERROR(_generate_short_key_index(key_columns, num_rows, short_key_pos)); } } @@ -737,21 +773,35 @@ int64_t SegmentWriter::max_row_to_add(size_t row_avg_size_in_bytes) { } std::string SegmentWriter::_full_encode_keys( - const std::vector& key_columns, size_t pos) { + const std::vector& key_columns, size_t pos, + bool null_first) { assert(_key_index_size.size() == _num_key_columns); assert(key_columns.size() == _num_key_columns && _key_coders.size() == _num_key_columns); + return _full_encode_keys(_key_coders, key_columns, pos, null_first); +} + +std::string SegmentWriter::_full_encode_keys( + const std::vector& key_coders, + const std::vector& key_columns, size_t pos, + bool null_first) { + assert(key_columns.size() == key_coders.size()); std::string encoded_keys; size_t cid = 0; for (const auto& column : key_columns) { auto field = column->get_data_at(pos); if (UNLIKELY(!field)) { - encoded_keys.push_back(KEY_NULL_FIRST_MARKER); + if (null_first) { + encoded_keys.push_back(KEY_NULL_FIRST_MARKER); + } else { + encoded_keys.push_back(KEY_NORMAL_MARKER); + } ++cid; continue; } encoded_keys.push_back(KEY_NORMAL_MARKER); - _key_coders[cid]->full_encode_ascending(field, &encoded_keys); + DCHECK(key_coders[cid] != nullptr); + key_coders[cid]->full_encode_ascending(field, &encoded_keys); ++cid; } return encoded_keys; @@ -773,6 +823,11 @@ void SegmentWriter::_encode_seq_column(const vectorized::IOlapColumnDataAccessor _seq_coder->full_encode_ascending(field, encoded_keys); } +void SegmentWriter::_encode_rowid(const uint32_t rowid, string* encoded_keys) { + encoded_keys->push_back(KEY_NORMAL_MARKER); + _rowid_coder->full_encode_ascending(&rowid, encoded_keys); +} + std::string SegmentWriter::_encode_keys( const std::vector& key_columns, size_t pos) { assert(key_columns.size() == _num_short_key_columns); @@ -887,12 +942,19 @@ Status SegmentWriter::finalize_columns_index(uint64_t* index_size) { *index_size = _file_writer->bytes_appended() - index_start; if (_has_key) { + bool write_short_key_index = _tablet_schema->keys_type() != UNIQUE_KEYS || + (_tablet_schema->keys_type() == UNIQUE_KEYS && + !_opts.enable_unique_key_merge_on_write) || + (_tablet_schema->keys_type() == UNIQUE_KEYS && + _opts.enable_unique_key_merge_on_write && + !_tablet_schema->cluster_key_idxes().empty()); if (_tablet_schema->keys_type() == UNIQUE_KEYS && _opts.enable_unique_key_merge_on_write) { RETURN_IF_ERROR(_write_primary_key_index()); // IndexedColumnWriter write data pages mixed with segment data, we should use // the stat from primary key index builder. *index_size += _primary_key_index_builder->disk_size(); - } else { + } + if (write_short_key_index) { RETURN_IF_ERROR(_write_short_key_index()); *index_size = _file_writer->bytes_appended() - index_start; } @@ -1035,12 +1097,14 @@ Status SegmentWriter::_write_raw_data(const std::vector& slices) { } Slice SegmentWriter::min_encoded_key() { - return (_primary_key_index_builder == nullptr) ? Slice(_min_key.data(), _min_key.size()) - : _primary_key_index_builder->min_key(); + return (_primary_key_index_builder == nullptr || !_tablet_schema->cluster_key_idxes().empty()) + ? Slice(_min_key.data(), _min_key.size()) + : _primary_key_index_builder->min_key(); } Slice SegmentWriter::max_encoded_key() { - return (_primary_key_index_builder == nullptr) ? Slice(_max_key.data(), _max_key.size()) - : _primary_key_index_builder->max_key(); + return (_primary_key_index_builder == nullptr || !_tablet_schema->cluster_key_idxes().empty()) + ? Slice(_max_key.data(), _max_key.size()) + : _primary_key_index_builder->max_key(); } void SegmentWriter::set_min_max_key(const Slice& key) { @@ -1070,5 +1134,64 @@ void SegmentWriter::set_mow_context(std::shared_ptr mow_context) { _mow_context = mow_context; } +Status SegmentWriter::_generate_primary_key_index( + const std::vector& primary_key_coders, + const std::vector& primary_key_columns, + vectorized::IOlapColumnDataAccessor* seq_column, size_t num_rows, bool need_sort) { + if (!need_sort) { // mow table without cluster key + std::string last_key; + for (size_t pos = 0; pos < num_rows; pos++) { + // use _key_coders + std::string key = _full_encode_keys(primary_key_columns, pos); + _maybe_invalid_row_cache(key); + if (_tablet_schema->has_sequence_col()) { + _encode_seq_column(seq_column, pos, &key); + } + DCHECK(key.compare(last_key) > 0) + << "found duplicate key or key is not sorted! current key: " << key + << ", last key" << last_key; + RETURN_IF_ERROR(_primary_key_index_builder->add_item(key)); + last_key = std::move(key); + } + } else { // mow table with cluster key + // 1. generate primary keys in memory + std::vector primary_keys; + for (uint32_t pos = 0; pos < num_rows; pos++) { + std::string key = _full_encode_keys(primary_key_coders, primary_key_columns, pos); + _maybe_invalid_row_cache(key); + if (_tablet_schema->has_sequence_col()) { + _encode_seq_column(seq_column, pos, &key); + } + _encode_rowid(pos, &key); + primary_keys.emplace_back(std::move(key)); + } + // 2. sort primary keys + std::sort(primary_keys.begin(), primary_keys.end()); + // 3. write primary keys index + std::string last_key; + for (const auto& key : primary_keys) { + DCHECK(key.compare(last_key) > 0) + << "found duplicate key or key is not sorted! current key: " << key + << ", last key" << last_key; + RETURN_IF_ERROR(_primary_key_index_builder->add_item(key)); + } + } + return Status::OK(); +} + +Status SegmentWriter::_generate_short_key_index( + std::vector& key_columns, size_t num_rows, + const std::vector& short_key_pos) { + // use _key_coders + set_min_key(_full_encode_keys(key_columns, 0)); + set_max_key(_full_encode_keys(key_columns, num_rows - 1)); + + key_columns.resize(_num_short_key_columns); + for (const auto pos : short_key_pos) { + RETURN_IF_ERROR(_short_key_index_builder->add_item(_encode_keys(key_columns, pos))); + } + return Status::OK(); +} + } // namespace segment_v2 } // namespace doris diff --git a/be/src/olap/rowset/segment_v2/segment_writer.h b/be/src/olap/rowset/segment_v2/segment_writer.h index 3b50d0a4aa..37011fdf0f 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.h +++ b/be/src/olap/rowset/segment_v2/segment_writer.h @@ -149,15 +149,29 @@ private: size_t pos); // used for unique-key with merge on write and segment min_max key std::string _full_encode_keys( - const std::vector& key_columns, size_t pos); + const std::vector& key_columns, size_t pos, + bool null_first = true); + + std::string _full_encode_keys( + const std::vector& key_coders, + const std::vector& key_columns, size_t pos, + bool null_first = true); + // used for unique-key with merge on write void _encode_seq_column(const vectorized::IOlapColumnDataAccessor* seq_column, size_t pos, string* encoded_keys); + void _encode_rowid(const uint32_t rowid, string* encoded_keys); void set_min_max_key(const Slice& key); void set_min_key(const Slice& key); void set_max_key(const Slice& key); bool _should_create_writers_with_dynamic_block(size_t num_columns_in_block); void _serialize_block_to_row_column(vectorized::Block& block); + Status _generate_primary_key_index( + const std::vector& primary_key_coders, + const std::vector& primary_key_columns, + vectorized::IOlapColumnDataAccessor* seq_column, size_t num_rows, bool need_sort); + Status _generate_short_key_index(std::vector& key_columns, + size_t num_rows, const std::vector& short_key_pos); private: uint32_t _segment_id; @@ -181,8 +195,12 @@ private: std::unique_ptr _olap_data_convertor; // used for building short key index or primary key index during vectorized write. + // for mow table with cluster keys, this is cluster keys std::vector _key_coders; + // for mow table with cluster keys, this is primary keys + std::vector _primary_key_coders; const KeyCoder* _seq_coder = nullptr; + const KeyCoder* _rowid_coder = nullptr; std::vector _key_index_size; size_t _short_key_row_pos = 0; diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 23ae7cfd42..4b0df03d35 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -231,7 +231,14 @@ Status VerticalSegmentWriter::init() { seq_col_length = _tablet_schema->column(_tablet_schema->sequence_col_idx()).length() + 1; } - _primary_key_index_builder.reset(new PrimaryKeyIndexBuilder(_file_writer, seq_col_length)); + size_t rowid_length = 0; + if (!_tablet_schema->cluster_key_idxes().empty()) { + rowid_length = PrimaryKeyIndexReader::ROW_ID_LENGTH; + _short_key_index_builder.reset( + new ShortKeyIndexBuilder(_segment_id, _opts.num_rows_per_block)); + } + _primary_key_index_builder.reset( + new PrimaryKeyIndexBuilder(_file_writer, seq_col_length, rowid_length)); RETURN_IF_ERROR(_primary_key_index_builder->init()); } else { _short_key_index_builder.reset( diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 8de046d634..e2ed1bb88d 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2786,7 +2786,7 @@ Status Tablet::lookup_row_key(const Slice& encoded_key, bool with_seq_col, const std::vector& specified_rowsets, RowLocation* row_location, uint32_t version, std::vector>& segment_caches, - RowsetSharedPtr* rowset) { + RowsetSharedPtr* rowset, bool with_rowid) { SCOPED_BVAR_LATENCY(g_tablet_lookup_rowkey_latency); size_t seq_col_length = 0; if (_tablet_meta->tablet_schema()->has_sequence_col() && with_seq_col) { @@ -2795,7 +2795,12 @@ Status Tablet::lookup_row_key(const Slice& encoded_key, bool with_seq_col, .length() + 1; } - Slice key_without_seq = Slice(encoded_key.get_data(), encoded_key.get_size() - seq_col_length); + size_t rowid_length = 0; + if (with_rowid && !_tablet_meta->tablet_schema()->cluster_key_idxes().empty()) { + rowid_length = PrimaryKeyIndexReader::ROW_ID_LENGTH; + } + Slice key_without_seq = + Slice(encoded_key.get_data(), encoded_key.get_size() - seq_col_length - rowid_length); RowLocation loc; for (size_t i = 0; i < specified_rowsets.size(); i++) { @@ -2805,9 +2810,13 @@ Status Tablet::lookup_row_key(const Slice& encoded_key, bool with_seq_col, DCHECK_EQ(segments_key_bounds.size(), num_segments); std::vector picked_segments; for (int i = num_segments - 1; i >= 0; i--) { - if (key_without_seq.compare(segments_key_bounds[i].max_key()) > 0 || - key_without_seq.compare(segments_key_bounds[i].min_key()) < 0) { - continue; + // If mow table has cluster keys, the key bounds is short keys, not primary keys + // use PrimaryKeyIndexMetaPB in primary key index? + if (_tablet_meta->tablet_schema()->cluster_key_idxes().empty()) { + if (key_without_seq.compare(segments_key_bounds[i].max_key()) > 0 || + key_without_seq.compare(segments_key_bounds[i].min_key()) < 0) { + continue; + } } picked_segments.emplace_back(i); } @@ -2824,7 +2833,7 @@ Status Tablet::lookup_row_key(const Slice& encoded_key, bool with_seq_col, DCHECK_EQ(segments.size(), num_segments); for (auto id : picked_segments) { - Status s = segments[id]->lookup_row_key(encoded_key, with_seq_col, &loc); + Status s = segments[id]->lookup_row_key(encoded_key, with_seq_col, with_rowid, &loc); if (s.is()) { continue; } @@ -2969,6 +2978,28 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, for (size_t i = 0; i < num_read; i++, row_id++) { Slice key = Slice(index_column->get_data_at(i).data, index_column->get_data_at(i).size); RowLocation loc; + // calculate row id + if (!_tablet_meta->tablet_schema()->cluster_key_idxes().empty()) { + size_t seq_col_length = 0; + if (_tablet_meta->tablet_schema()->has_sequence_col()) { + seq_col_length = + _tablet_meta->tablet_schema() + ->column(_tablet_meta->tablet_schema()->sequence_col_idx()) + .length() + + 1; + } + size_t rowid_length = PrimaryKeyIndexReader::ROW_ID_LENGTH; + Slice key_without_seq = + Slice(key.get_data(), key.get_size() - seq_col_length - rowid_length); + Slice rowid_slice = + Slice(key.get_data() + key_without_seq.get_size() + seq_col_length + 1, + rowid_length - 1); + const auto* type_info = + get_scalar_type_info(); + auto rowid_coder = get_key_coder(type_info->type()); + RETURN_IF_ERROR(rowid_coder->decode_ascending(&rowid_slice, rowid_length, + (uint8_t*)&row_id)); + } // same row in segments should be filtered if (delete_bitmap->contains({rowset_id, seg->id(), DeleteBitmap::TEMP_VERSION_COMMON}, row_id)) { @@ -3037,7 +3068,7 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, } remaining -= num_read; } - DCHECK_EQ(total, row_id) << "segment total rows: " << total << " row_id:" << row_id; + // DCHECK_EQ(total, row_id) << "segment total rows: " << total << " row_id:" << row_id; if (config::enable_merge_on_write_correctness_check) { RowsetIdUnorderedSet rowsetids; @@ -3726,11 +3757,15 @@ Status Tablet::calc_delete_bitmap_between_segments( size_t seq_col_length = 0; if (_tablet_meta->tablet_schema()->has_sequence_col()) { auto seq_col_idx = _tablet_meta->tablet_schema()->sequence_col_idx(); - seq_col_length = _tablet_meta->tablet_schema()->column(seq_col_idx).length(); + seq_col_length = _tablet_meta->tablet_schema()->column(seq_col_idx).length() + 1; + } + size_t rowid_length = 0; + if (!_tablet_meta->tablet_schema()->cluster_key_idxes().empty()) { + rowid_length = PrimaryKeyIndexReader::ROW_ID_LENGTH; } MergeIndexDeleteBitmapCalculator calculator; - RETURN_IF_ERROR(calculator.init(rowset_id, segments, seq_col_length)); + RETURN_IF_ERROR(calculator.init(rowset_id, segments, seq_col_length, rowid_length)); RETURN_IF_ERROR(calculator.calculate_all(delete_bitmap)); diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 00773c12ac..222435ab99 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -406,7 +406,7 @@ public: const std::vector& specified_rowsets, RowLocation* row_location, uint32_t version, std::vector>& segment_caches, - RowsetSharedPtr* rowset = nullptr); + RowsetSharedPtr* rowset = nullptr, bool with_rowid = true); // Lookup a row with TupleDescriptor and fill Block Status lookup_row_data(const Slice& encoded_key, const RowLocation& row_location, diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 76cf14a75b..237a1259cb 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -168,6 +168,9 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id schema->set_sort_type(SortType::LEXICAL); } schema->set_sort_col_num(tablet_schema.sort_col_num); + for (const auto& i : tablet_schema.cluster_key_idxes) { + schema->add_cluster_key_idxes(i); + } tablet_meta_pb.set_in_restore_mode(false); // set column information diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index d2b16a907a..af163275b3 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -730,6 +730,10 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema) { _indexes.clear(); _field_name_to_index.clear(); _field_id_to_index.clear(); + _cluster_key_idxes.clear(); + for (const auto& i : schema.cluster_key_idxes()) { + _cluster_key_idxes.push_back(i); + } for (auto& column_pb : schema.column()) { TabletColumn column; column.init_from_pb(column_pb); @@ -823,7 +827,10 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version _delete_sign_idx = -1; _sequence_col_idx = -1; _version_col_idx = -1; - + _cluster_key_idxes.clear(); + for (const auto& i : ori_tablet_schema._cluster_key_idxes) { + _cluster_key_idxes.push_back(i); + } for (auto& column : index->columns) { if (column->is_key()) { _num_key_columns++; @@ -892,6 +899,9 @@ bool TabletSchema::is_dropped_column(const TabletColumn& col) const { } void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { + for (const auto& i : _cluster_key_idxes) { + tablet_schema_pb->add_cluster_key_idxes(i); + } tablet_schema_pb->set_keys_type(_keys_type); for (auto& col : _cols) { ColumnPB* column = tablet_schema_pb->add_column(); diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 2fe6ea4558..794c9b5b5b 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -252,6 +252,7 @@ public: std::vector& mutable_columns(); size_t num_columns() const { return _num_columns; } size_t num_key_columns() const { return _num_key_columns; } + std::vector cluster_key_idxes() const { return _cluster_key_idxes; } size_t num_null_columns() const { return _num_null_columns; } size_t num_short_key_columns() const { return _num_short_key_columns; } size_t num_rows_per_row_block() const { return _num_rows_per_row_block; } @@ -373,6 +374,7 @@ private: size_t _num_columns = 0; size_t _num_variant_columns = 0; size_t _num_key_columns = 0; + std::vector _cluster_key_idxes; size_t _num_null_columns = 0; size_t _num_short_key_columns = 0; size_t _num_rows_per_row_block = 0; diff --git a/be/src/service/point_query_executor.cpp b/be/src/service/point_query_executor.cpp index 12e1f9f677..4974bcc40b 100644 --- a/be/src/service/point_query_executor.cpp +++ b/be/src/service/point_query_executor.cpp @@ -286,7 +286,7 @@ Status PointQueryExecutor::_lookup_row_key() { auto rowset_ptr = std::make_unique(); st = (_tablet->lookup_row_key(_row_read_ctxs[i]._primary_key, false, specified_rowsets, &location, INT32_MAX /*rethink?*/, segment_caches, - rowset_ptr.get())); + rowset_ptr.get(), false)); if (st.is()) { continue; } diff --git a/be/src/vec/olap/vertical_block_reader.cpp b/be/src/vec/olap/vertical_block_reader.cpp index 1612094326..52d35283b5 100644 --- a/be/src/vec/olap/vertical_block_reader.cpp +++ b/be/src/vec/olap/vertical_block_reader.cpp @@ -146,7 +146,7 @@ Status VerticalBlockReader::_init_collect_iter(const ReaderParams& read_params) _vcollect_iter = new_vertical_heap_merge_iterator( std::move(*segment_iters_ptr), iterator_init_flag, rowset_ids, ori_return_col_size, read_params.tablet->keys_type(), seq_col_idx, - _row_sources_buffer); + _row_sources_buffer, read_params.key_group_cluster_key_idxes); } } else { _vcollect_iter = new_vertical_mask_merge_iterator(std::move(*segment_iters_ptr), @@ -224,9 +224,13 @@ Status VerticalBlockReader::init(const ReaderParams& read_params) { _next_block_func = &VerticalBlockReader::_direct_next_block; break; case KeysType::UNIQUE_KEYS: - _next_block_func = &VerticalBlockReader::_unique_key_next_block; - if (_filter_delete) { - _delete_filter_column = ColumnUInt8::create(); + if (tablet()->tablet_meta()->tablet_schema()->cluster_key_idxes().empty()) { + _next_block_func = &VerticalBlockReader::_unique_key_next_block; + if (_filter_delete) { + _delete_filter_column = ColumnUInt8::create(); + } + } else { + _next_block_func = &VerticalBlockReader::_direct_next_block; } break; case KeysType::AGG_KEYS: diff --git a/be/src/vec/olap/vertical_merge_iterator.cpp b/be/src/vec/olap/vertical_merge_iterator.cpp index f912ee7dff..fe521c2705 100644 --- a/be/src/vec/olap/vertical_merge_iterator.cpp +++ b/be/src/vec/olap/vertical_merge_iterator.cpp @@ -259,8 +259,14 @@ Status VerticalMergeIteratorContext::block_reset(const std::shared_ptr& b } bool VerticalMergeIteratorContext::compare(const VerticalMergeIteratorContext& rhs) const { - int cmp_res = _block->compare_at(_index_in_block, rhs._index_in_block, _num_key_columns, + int cmp_res; + if (_key_group_cluster_key_idxes.empty()) { + cmp_res = _block->compare_at(_index_in_block, rhs._index_in_block, _num_key_columns, *rhs._block, -1); + } else { + cmp_res = _block->compare_at(_index_in_block, rhs._index_in_block, + &_key_group_cluster_key_idxes, *rhs._block, -1); + } if (cmp_res != 0) { return cmp_res > 0; } @@ -425,7 +431,8 @@ Status VerticalHeapMergeIterator::next_batch(Block* block) { tmp_row_sources.emplace_back(ctx->order(), false); } if (ctx->is_same() && - (_keys_type == KeysType::UNIQUE_KEYS || _keys_type == KeysType::AGG_KEYS)) { + ((_keys_type == KeysType::UNIQUE_KEYS && _key_group_cluster_key_idxes.empty()) || + _keys_type == KeysType::AGG_KEYS)) { // skip cur row, copy pre ctx ++_merged_rows; if (pre_ctx) { @@ -504,7 +511,8 @@ Status VerticalHeapMergeIterator::init(const StorageReadOptions& opts) { bool pre_iter_invalid = false; for (auto& iter : _origin_iters) { VerticalMergeIteratorContext* ctx = new VerticalMergeIteratorContext( - std::move(iter), _rowset_ids[seg_order], _ori_return_cols, seg_order, _seq_col_idx); + std::move(iter), _rowset_ids[seg_order], _ori_return_cols, seg_order, _seq_col_idx, + _key_group_cluster_key_idxes); _ori_iter_ctx.push_back(ctx); if (_iterator_init_flags[seg_order] || pre_iter_invalid) { RETURN_IF_ERROR(ctx->init(opts)); @@ -764,10 +772,11 @@ Status VerticalMaskMergeIterator::init(const StorageReadOptions& opts) { std::shared_ptr new_vertical_heap_merge_iterator( std::vector&& inputs, const std::vector& iterator_init_flag, const std::vector& rowset_ids, size_t ori_return_cols, KeysType keys_type, - uint32_t seq_col_idx, RowSourcesBuffer* row_sources) { - return std::make_shared(std::move(inputs), iterator_init_flag, - rowset_ids, ori_return_cols, keys_type, - seq_col_idx, row_sources); + uint32_t seq_col_idx, RowSourcesBuffer* row_sources, + std::vector key_group_cluster_key_idxes) { + return std::make_shared( + std::move(inputs), iterator_init_flag, rowset_ids, ori_return_cols, keys_type, + seq_col_idx, row_sources, key_group_cluster_key_idxes); } std::shared_ptr new_vertical_fifo_merge_iterator( diff --git a/be/src/vec/olap/vertical_merge_iterator.h b/be/src/vec/olap/vertical_merge_iterator.h index 70a452b2b6..760835e6d3 100644 --- a/be/src/vec/olap/vertical_merge_iterator.h +++ b/be/src/vec/olap/vertical_merge_iterator.h @@ -147,13 +147,15 @@ private: class VerticalMergeIteratorContext { public: VerticalMergeIteratorContext(RowwiseIteratorUPtr&& iter, RowsetId rowset_id, - size_t ori_return_cols, uint32_t order, uint32_t seq_col_idx) + size_t ori_return_cols, uint32_t order, uint32_t seq_col_idx, + std::vector key_group_cluster_key_idxes = {}) : _iter(std::move(iter)), _rowset_id(rowset_id), _ori_return_cols(ori_return_cols), _order(order), _seq_col_idx(seq_col_idx), - _num_key_columns(_iter->schema().num_key_columns()) {} + _num_key_columns(_iter->schema().num_key_columns()), + _key_group_cluster_key_idxes(key_group_cluster_key_idxes) {} VerticalMergeIteratorContext(const VerticalMergeIteratorContext&) = delete; VerticalMergeIteratorContext(VerticalMergeIteratorContext&&) = delete; @@ -217,6 +219,7 @@ private: int32_t _index_in_block = -1; size_t _block_row_max = 0; int _num_key_columns; + const std::vector _key_group_cluster_key_idxes; size_t _cur_batch_num = 0; // used to store data load from iterator->next_batch(Block*) @@ -237,14 +240,16 @@ public: std::vector iterator_init_flags, std::vector rowset_ids, size_t ori_return_cols, KeysType keys_type, int32_t seq_col_idx, - RowSourcesBuffer* row_sources_buf) + RowSourcesBuffer* row_sources_buf, + std::vector key_group_cluster_key_idxes) : _origin_iters(std::move(iters)), _iterator_init_flags(iterator_init_flags), _rowset_ids(rowset_ids), _ori_return_cols(ori_return_cols), _keys_type(keys_type), _seq_col_idx(seq_col_idx), - _row_sources_buf(row_sources_buf) {} + _row_sources_buf(row_sources_buf), + _key_group_cluster_key_idxes(key_group_cluster_key_idxes) {} ~VerticalHeapMergeIterator() override { while (!_merge_heap.empty()) { @@ -296,6 +301,7 @@ private: StorageReadOptions _opts; bool _record_rowids = false; std::vector _block_row_locations; + std::vector _key_group_cluster_key_idxes; }; // --------------- VerticalFifoMergeIterator ------------- // @@ -400,7 +406,8 @@ private: std::shared_ptr new_vertical_heap_merge_iterator( std::vector&& inputs, const std::vector& iterator_init_flag, const std::vector& rowset_ids, size_t _ori_return_cols, KeysType key_type, - uint32_t seq_col_idx, RowSourcesBuffer* row_sources_buf); + uint32_t seq_col_idx, RowSourcesBuffer* row_sources_buf, + std::vector key_group_cluster_key_idxes); std::shared_ptr new_vertical_fifo_merge_iterator( std::vector&& inputs, const std::vector& iterator_init_flag, diff --git a/be/test/olap/delete_bitmap_calculator_test.cpp b/be/test/olap/delete_bitmap_calculator_test.cpp index 898e6f2f5a..250ac9adff 100644 --- a/be/test/olap/delete_bitmap_calculator_test.cpp +++ b/be/test/olap/delete_bitmap_calculator_test.cpp @@ -227,7 +227,7 @@ public: MergeIndexDeleteBitmapCalculator calculator; size_t seq_col_len = 0; if (has_sequence_col) { - seq_col_len = tablet_schema->column(tablet_schema->sequence_col_idx()).length(); + seq_col_len = tablet_schema->column(tablet_schema->sequence_col_idx()).length() + 1; } ASSERT_TRUE(calculator.init(rowset_id, segments, seq_col_len).ok()); diff --git a/be/test/olap/primary_key_index_test.cpp b/be/test/olap/primary_key_index_test.cpp index fb96e7411e..a88614f054 100644 --- a/be/test/olap/primary_key_index_test.cpp +++ b/be/test/olap/primary_key_index_test.cpp @@ -57,7 +57,7 @@ TEST_F(PrimaryKeyIndexTest, builder) { auto fs = io::global_local_filesystem(); EXPECT_TRUE(fs->create_file(filename, &file_writer).ok()); - PrimaryKeyIndexBuilder builder(file_writer.get(), 0); + PrimaryKeyIndexBuilder builder(file_writer.get(), 0, 0); static_cast(builder.init()); size_t num_rows = 0; std::vector keys; @@ -174,7 +174,7 @@ TEST_F(PrimaryKeyIndexTest, multiple_pages) { EXPECT_TRUE(fs->create_file(filename, &file_writer).ok()); config::primary_key_data_page_size = 5 * 5; - PrimaryKeyIndexBuilder builder(file_writer.get(), 0); + PrimaryKeyIndexBuilder builder(file_writer.get(), 0, 0); static_cast(builder.init()); size_t num_rows = 0; std::vector keys {"00000", "00002", "00004", "00006", "00008", @@ -258,7 +258,7 @@ TEST_F(PrimaryKeyIndexTest, single_page) { EXPECT_TRUE(fs->create_file(filename, &file_writer).ok()); config::primary_key_data_page_size = 32768; - PrimaryKeyIndexBuilder builder(file_writer.get(), 0); + PrimaryKeyIndexBuilder builder(file_writer.get(), 0, 0); static_cast(builder.init()); size_t num_rows = 0; std::vector keys {"00000", "00002", "00004", "00006", "00008", diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 9f18216850..d6eef426ef 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -849,6 +849,7 @@ nonterminal DistributionDesc opt_distribution; nonterminal Integer opt_distribution_number; nonterminal Long opt_field_length; nonterminal KeysDesc opt_keys; +nonterminal List opt_cluster_keys; nonterminal Long opt_id; nonterminal PartitionKeyDesc partition_key_desc; @@ -3193,8 +3194,9 @@ opt_keys ::= :} /* unique_keys */ | KW_UNIQUE KW_KEY LPAREN ident_list:keys RPAREN + opt_cluster_keys:cluster_keys {: - RESULT = new KeysDesc(KeysType.UNIQUE_KEYS, keys); + RESULT = new KeysDesc(KeysType.UNIQUE_KEYS, keys, cluster_keys); :} /* agg_keys */ | KW_AGGREGATE KW_KEY LPAREN ident_list:keys RPAREN @@ -3203,6 +3205,17 @@ opt_keys ::= :} ; +opt_cluster_keys ::= + /* Empty */ + {: + RESULT = null; + :} + | KW_CLUSTER KW_BY LPAREN ident_list:keys RPAREN + {: + RESULT = keys; + :} + ; + opt_all_partition_desc_list ::= /* Empty */ {: diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 3a6cae6402..e77e7f11d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -310,24 +310,19 @@ public class SchemaChangeHandler extends AlterHandler { */ if (KeysType.UNIQUE_KEYS == olapTable.getKeysType()) { List baseSchema = indexSchemaMap.get(baseIndexId); - boolean isKey = false; for (Column column : baseSchema) { - if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { - lightSchemaChange = false; - isKey = true; - break; + if (column.getName().equalsIgnoreCase(dropColName)) { + if (column.isKey()) { + throw new DdlException("Can not drop key column in Unique data model table"); + } else if (column.isClusterKey()) { + throw new DdlException("Can not drop cluster key column in Unique data model table"); + } } } - - if (isKey) { - throw new DdlException("Can not drop key column in Unique data model table"); - } - if (olapTable.hasSequenceCol() && dropColName.equalsIgnoreCase(olapTable.getSequenceMapCol())) { throw new DdlException("Can not drop sequence mapping column[" + dropColName + "] in Unique data model table[" + olapTable.getName() + "]"); } - } else if (KeysType.AGG_KEYS == olapTable.getKeysType()) { if (null == targetIndexName) { // drop column in base table @@ -595,6 +590,9 @@ public class SchemaChangeHandler extends AlterHandler { col.checkSchemaChangeAllowed(modColumn); lightSchemaChange = olapTable.getEnableLightSchemaChange(); } + if (col.isClusterKey()) { + throw new DdlException("Can not modify cluster key column: " + col.getName()); + } } } if (hasColPos) { @@ -808,6 +806,9 @@ public class SchemaChangeHandler extends AlterHandler { if (!column.isVisible()) { newSchema.add(column); } + if (column.isClusterKey()) { + throw new DdlException("Can not modify column order in Unique data model table"); + } } } if (newSchema.size() != targetIndexSchema.size()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index 83fdbf792d..9cd1ca4481 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -181,6 +181,7 @@ public class ColumnDef { private DefaultValue defaultValue; private String comment; private boolean visible; + private int clusterKeyId = -1; public ColumnDef(String name, TypeDef typeDef) { this(name, typeDef, false, null, false, false, DefaultValue.NOT_SET, ""); @@ -306,6 +307,10 @@ public class ColumnDef { return visible; } + public void setClusterKeyId(int clusterKeyId) { + this.clusterKeyId = clusterKeyId; + } + public void analyze(boolean isOlap) throws AnalysisException { if (name == null || typeDef == null) { throw new AnalysisException("No column name or column type in column definition."); @@ -578,7 +583,8 @@ public class ColumnDef { } return new Column(name, type, isKey, aggregateType, isAllowNull, isAutoInc, defaultValue.value, comment, - visible, defaultValue.defaultValueExprDef, Column.COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue.getValue()); + visible, defaultValue.defaultValueExprDef, Column.COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue.getValue(), + clusterKeyId); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index a2ae5547ba..602793b7eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -450,6 +450,10 @@ public class CreateTableStmt extends DdlStmt { } keysDesc.analyze(columnDefs); + if (!CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnNames()) && !enableUniqueKeyMergeOnWrite) { + throw new AnalysisException("Cluster keys only support unique keys table which enabled " + + PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE); + } for (int i = 0; i < keysDesc.keysColumnSize(); ++i) { columnDefs.get(i).setIsKey(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/KeysDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/KeysDesc.java index 6f2eb96a7d..a3ed083ebd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/KeysDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/KeysDesc.java @@ -32,6 +32,8 @@ import java.util.List; public class KeysDesc implements Writable { private KeysType type; private List keysColumnNames; + private List clusterKeysColumnNames; + private List clusterKeysColumnIds = null; public KeysDesc() { this.type = KeysType.AGG_KEYS; @@ -43,6 +45,11 @@ public class KeysDesc implements Writable { this.keysColumnNames = keysColumnNames; } + public KeysDesc(KeysType type, List keysColumnNames, List clusterKeyColumnNames) { + this(type, keysColumnNames); + this.clusterKeysColumnNames = clusterKeyColumnNames; + } + public KeysType getKeysType() { return type; } @@ -51,6 +58,14 @@ public class KeysDesc implements Writable { return keysColumnNames.size(); } + public List getClusterKeysColumnNames() { + return clusterKeysColumnNames; + } + + public List getClusterKeysColumnIds() { + return clusterKeysColumnIds; + } + public boolean containsCol(String colName) { return keysColumnNames.contains(colName); } @@ -68,6 +83,14 @@ public class KeysDesc implements Writable { throw new AnalysisException("The number of key columns should be less than the number of columns."); } + if (clusterKeysColumnNames != null) { + if (type != KeysType.UNIQUE_KEYS) { + throw new AnalysisException("Cluster keys only support unique keys table."); + } + clusterKeysColumnIds = Lists.newArrayList(); + analyzeClusterKeys(cols); + } + for (int i = 0; i < keysColumnNames.size(); ++i) { String name = cols.get(i).getName(); if (!keysColumnNames.get(i).equalsIgnoreCase(name)) { @@ -100,6 +123,44 @@ public class KeysDesc implements Writable { } } } + + if (clusterKeysColumnNames != null) { + int minKeySize = keysColumnNames.size() < clusterKeysColumnNames.size() ? keysColumnNames.size() + : clusterKeysColumnNames.size(); + boolean sameKey = true; + for (int i = 0; i < minKeySize; ++i) { + if (!keysColumnNames.get(i).equalsIgnoreCase(clusterKeysColumnNames.get(i))) { + sameKey = false; + break; + } + } + if (sameKey) { + throw new AnalysisException("Unique keys and cluster keys should be different."); + } + } + } + + private void analyzeClusterKeys(List cols) throws AnalysisException { + for (int i = 0; i < clusterKeysColumnNames.size(); ++i) { + String name = clusterKeysColumnNames.get(i); + // check if key is duplicate + for (int j = 0; j < i; j++) { + if (clusterKeysColumnNames.get(j).equalsIgnoreCase(name)) { + throw new AnalysisException("Duplicate cluster key column[" + name + "]."); + } + } + // check if key exists and generate key column ids + for (int j = 0; j < cols.size(); j++) { + if (cols.get(j).getName().equalsIgnoreCase(name)) { + cols.get(j).setClusterKeyId(clusterKeysColumnIds.size()); + clusterKeysColumnIds.add(j); + break; + } + if (j == cols.size() - 1) { + throw new AnalysisException("Key cluster column[" + name + "] doesn't exist."); + } + } + } } public String toSql() { @@ -114,6 +175,18 @@ public class KeysDesc implements Writable { i++; } stringBuilder.append(")"); + if (clusterKeysColumnNames != null) { + stringBuilder.append("\nCLUSTER BY ("); + i = 0; + for (String columnName : clusterKeysColumnNames) { + if (i != 0) { + stringBuilder.append(", "); + } + stringBuilder.append("`").append(columnName).append("`"); + i++; + } + stringBuilder.append(")"); + } return stringBuilder.toString(); } @@ -132,6 +205,14 @@ public class KeysDesc implements Writable { for (String colName : keysColumnNames) { Text.writeString(out, colName); } + if (clusterKeysColumnNames == null) { + out.writeInt(0); + } else { + out.writeInt(clusterKeysColumnNames.size()); + for (String colName : clusterKeysColumnNames) { + Text.writeString(out, colName); + } + } } public void readFields(DataInput in) throws IOException { @@ -141,5 +222,12 @@ public class KeysDesc implements Writable { for (int i = 0; i < count; i++) { keysColumnNames.add(Text.readString(in)); } + count = in.readInt(); + if (count > 0) { + clusterKeysColumnNames = Lists.newArrayList(); + for (int i = 0; i < count; i++) { + clusterKeysColumnNames.add(Text.readString(in)); + } + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 81c1edc2e2..591b67d625 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -128,6 +128,9 @@ public class Column implements Writable, GsonPostProcessable { @SerializedName(value = "genericAggregationName") private String genericAggregationName; + @SerializedName(value = "clusterKeyId") + private int clusterKeyId = -1; + private boolean isCompoundKey = false; @SerializedName(value = "hasOnUpdateDefaultValue") @@ -242,6 +245,14 @@ public class Column implements Writable, GsonPostProcessable { } } + public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, + boolean isAutoInc, String defaultValue, String comment, boolean visible, + DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue, int clusterKeyId) { + this(name, type, isKey, aggregateType, isAllowNull, isAutoInc, defaultValue, comment, visible, + defaultValueExprDef, colUniqueId, realDefaultValue); + this.clusterKeyId = clusterKeyId; + } + public Column(Column column) { this.name = column.getName(); this.type = column.type; @@ -263,6 +274,7 @@ public class Column implements Writable, GsonPostProcessable { this.defineName = column.getDefineName(); this.hasOnUpdateDefaultValue = column.hasOnUpdateDefaultValue; this.onUpdateDefaultValueExprDef = column.onUpdateDefaultValueExprDef; + this.clusterKeyId = column.getClusterKeyId(); } public void createChildrenColumn(Type type, Column column) { @@ -551,6 +563,7 @@ public class Column implements Writable, GsonPostProcessable { tColumn.addToChildrenColumn(column.toThrift()); } } + tColumn.setClusterKeyId(this.clusterKeyId); // ATTN: // Currently, this `toThrift()` method is only used from CreateReplicaTask. // And CreateReplicaTask does not need `defineExpr` field. @@ -580,6 +593,7 @@ public class Column implements Writable, GsonPostProcessable { if (tColumn.getAggregationType() != null) { childrenTColumn.setAggregationType(tColumn.getAggregationType()); } + childrenTColumn.setClusterKeyId(children.clusterKeyId); tColumn.children_column.add(childrenTColumn); toChildrenThrift(children, childrenTColumn); @@ -660,6 +674,7 @@ public class Column implements Writable, GsonPostProcessable { && (other.getDataType() == PrimitiveType.VARCHAR || other.getDataType() == PrimitiveType.STRING)) { return; } + // TODO check cluster key } public boolean nameEquals(String otherColName, boolean ignorePrefix) { @@ -718,6 +733,14 @@ public class Column implements Writable, GsonPostProcessable { } } + public boolean isClusterKey() { + return clusterKeyId != -1; + } + + public int getClusterKeyId() { + return clusterKeyId; + } + public String toSql() { return toSql(false, false); } @@ -811,7 +834,7 @@ public class Column implements Writable, GsonPostProcessable { public int hashCode() { return Objects.hash(name, getDataType(), getStrLen(), getPrecision(), getScale(), aggregationType, isAggregationTypeImplicit, isKey, isAllowNull, isAutoInc, defaultValue, comment, children, visible, - realDefaultValue); + realDefaultValue, clusterKeyId); } @Override @@ -839,7 +862,8 @@ public class Column implements Writable, GsonPostProcessable { && Objects.equals(comment, other.comment) && visible == other.visible && Objects.equals(children, other.children) - && Objects.equals(realDefaultValue, other.realDefaultValue); + && Objects.equals(realDefaultValue, other.realDefaultValue) + && clusterKeyId == other.clusterKeyId; } // distribution column compare only care about attrs which affect data, @@ -861,20 +885,22 @@ public class Column implements Writable, GsonPostProcessable { && getScale() == other.getScale() && visible == other.visible && Objects.equals(children, other.children) - && Objects.equals(realDefaultValue, other.realDefaultValue); + && Objects.equals(realDefaultValue, other.realDefaultValue) + && clusterKeyId == other.clusterKeyId; if (!ok) { LOG.info("this column: name {} default value {} aggregationType {} isAggregationTypeImplicit {} " - + "isKey {}, isAllowNull {}, datatype {}, strlen {}, precision {}, scale {}, visible {} " - + "children {} realDefaultValue {}", - name, getDefaultValue(), aggregationType, isAggregationTypeImplicit, isKey, isAllowNull, - getDataType(), getStrLen(), getPrecision(), getScale(), visible, children, realDefaultValue); + + "isKey {}, isAllowNull {}, datatype {}, strlen {}, precision {}, scale {}, visible {} " + + "children {}, realDefaultValue {}, clusterKeyId {}", + name, getDefaultValue(), aggregationType, isAggregationTypeImplicit, isKey, isAllowNull, + getDataType(), getStrLen(), getPrecision(), getScale(), visible, children, realDefaultValue, + clusterKeyId); LOG.info("other column: name {} default value {} aggregationType {} isAggregationTypeImplicit {} " - + "isKey {}, isAllowNull {}, datatype {}, strlen {}, precision {}, scale {}, visible {} " - + "children {} realDefaultValue {}", - other.name, other.getDefaultValue(), other.aggregationType, other.isAggregationTypeImplicit, - other.isKey, other.isAllowNull, other.getDataType(), other.getStrLen(), other.getPrecision(), - other.getScale(), other.visible, other.children, other.realDefaultValue); + + "isKey {}, isAllowNull {}, datatype {}, strlen {}, precision {}, scale {}, visible {}, " + + "children {}, realDefaultValue {}, clusterKeyId {}", + other.name, other.getDefaultValue(), other.aggregationType, other.isAggregationTypeImplicit, + other.isKey, other.isAllowNull, other.getDataType(), other.getStrLen(), other.getPrecision(), + other.getScale(), other.visible, other.children, other.realDefaultValue, other.clusterKeyId); } return ok; } @@ -943,6 +969,7 @@ public class Column implements Writable, GsonPostProcessable { sb.append(isKey); sb.append(isAllowNull); sb.append(aggregationType); + sb.append(clusterKeyId); sb.append(defaultValue == null ? "" : defaultValue); return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 13e4b615fd..bed741c6e4 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -295,6 +295,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -3021,12 +3022,21 @@ public class Env { : keySql.substring("DUPLICATE ".length())) .append("("); List keysColumnNames = Lists.newArrayList(); + Map clusterKeysColumnNamesToId = new TreeMap<>(); for (Column column : olapTable.getBaseSchema()) { if (column.isKey()) { keysColumnNames.add("`" + column.getName() + "`"); } + if (column.isClusterKey()) { + clusterKeysColumnNamesToId.put(column.getClusterKeyId(), column.getName()); + } } sb.append(Joiner.on(", ").join(keysColumnNames)).append(")"); + // show cluster keys + if (!clusterKeysColumnNamesToId.isEmpty()) { + sb.append("\n").append("CLUSTER BY (`"); + sb.append(Joiner.on("`, `").join(clusterKeysColumnNamesToId.values())).append("`)"); + } } if (specificVersion != -1) { @@ -3952,15 +3962,22 @@ public class Env { public static short calcShortKeyColumnCount(List columns, Map properties, boolean isKeysRequired) throws DdlException { List indexColumns = new ArrayList(); + Map clusterColumns = new TreeMap<>(); for (Column column : columns) { if (column.isKey()) { indexColumns.add(column); } + if (column.isClusterKey()) { + clusterColumns.put(column.getClusterKeyId(), column); + } } - LOG.debug("index column size: {}", indexColumns.size()); + LOG.debug("index column size: {}, cluster column size: {}", indexColumns.size(), clusterColumns.size()); if (isKeysRequired) { Preconditions.checkArgument(indexColumns.size() > 0); } + // sort by cluster keys for mow if set, otherwise by index columns + List sortKeyColumns = clusterColumns.isEmpty() ? indexColumns + : clusterColumns.values().stream().collect(Collectors.toList()); // figure out shortKeyColumnCount short shortKeyColumnCount = (short) -1; @@ -3975,12 +3992,12 @@ public class Env { throw new DdlException("Invalid short key: " + shortKeyColumnCount); } - if (shortKeyColumnCount > indexColumns.size()) { - throw new DdlException("Short key is too large. should less than: " + indexColumns.size()); + if (shortKeyColumnCount > sortKeyColumns.size()) { + throw new DdlException("Short key is too large. should less than: " + sortKeyColumns.size()); } for (int pos = 0; pos < shortKeyColumnCount; pos++) { - if (indexColumns.get(pos).getDataType() == PrimitiveType.VARCHAR && pos != shortKeyColumnCount - 1) { + if (sortKeyColumns.get(pos).getDataType() == PrimitiveType.VARCHAR && pos != shortKeyColumnCount - 1) { throw new DdlException("Varchar should not in the middle of short keys."); } } @@ -3995,9 +4012,9 @@ public class Env { */ shortKeyColumnCount = 0; int shortKeySizeByte = 0; - int maxShortKeyColumnCount = Math.min(indexColumns.size(), FeConstants.shortkey_max_column_count); + int maxShortKeyColumnCount = Math.min(sortKeyColumns.size(), FeConstants.shortkey_max_column_count); for (int i = 0; i < maxShortKeyColumnCount; i++) { - Column column = indexColumns.get(i); + Column column = sortKeyColumns.get(i); shortKeySizeByte += column.getOlapColumnIndexSize(); if (shortKeySizeByte > FeConstants.shortkey_maxsize_bytes) { if (column.getDataType().isCharFamily()) { @@ -4020,6 +4037,18 @@ public class Env { } // end calc shortKeyColumnCount + if (clusterColumns.size() > 0 && shortKeyColumnCount < clusterColumns.size()) { + boolean sameKey = true; + for (int i = 0; i < shortKeyColumnCount; i++) { + if (!clusterColumns.get(i).getName().equals(indexColumns.get(i).getName())) { + sameKey = false; + break; + } + } + if (sameKey) { + throw new DdlException(shortKeyColumnCount + " short keys is a part of unique keys"); + } + } return shortKeyColumnCount; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 8706b36d01..5b94e7529d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -188,6 +188,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -1565,7 +1566,8 @@ public class InternalCatalog implements CatalogIf { olapTable.getTimeSeriesCompactionFileCountThreshold(), olapTable.getTimeSeriesCompactionTimeThresholdSeconds(), olapTable.storeRowColumn(), - binlogConfig, dataProperty.isStorageMediumSpecified()); + binlogConfig, dataProperty.isStorageMediumSpecified(), null); + // TODO cluster key ids // check again olapTable = db.getOlapTableOrDdlException(tableName); @@ -1819,7 +1821,7 @@ public class InternalCatalog implements CatalogIf { String compactionPolicy, Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold, Long timeSeriesCompactionTimeThresholdSeconds, boolean storeRowColumn, BinlogConfig binlogConfig, - boolean isStorageMediumSpecified) throws DdlException { + boolean isStorageMediumSpecified, List clusterKeyIndexes) throws DdlException { // create base index first. Preconditions.checkArgument(baseIndexId != -1); MaterializedIndex baseIndex = new MaterializedIndex(baseIndexId, IndexState.NORMAL); @@ -1887,6 +1889,7 @@ public class InternalCatalog implements CatalogIf { storeRowColumn, binlogConfig); task.setStorageFormat(storageFormat); + task.setClusterKeyIndexes(clusterKeyIndexes); batchTask.addTask(task); // add to AgentTaskQueue for handling finish report. // not for resending task @@ -2154,8 +2157,10 @@ public class InternalCatalog implements CatalogIf { olapTable.setCompressionType(compressionType); // check data sort properties + int keyColumnSize = CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnIds()) ? keysDesc.keysColumnSize() : + keysDesc.getClusterKeysColumnIds().size(); DataSortInfo dataSortInfo = PropertyAnalyzer.analyzeDataSortInfo(properties, keysType, - keysDesc.keysColumnSize(), storageFormat); + keyColumnSize, storageFormat); olapTable.setDataSortInfo(dataSortInfo); boolean enableUniqueKeyMergeOnWrite = false; @@ -2484,7 +2489,8 @@ public class InternalCatalog implements CatalogIf { olapTable.getTimeSeriesCompactionFileCountThreshold(), olapTable.getTimeSeriesCompactionTimeThresholdSeconds(), storeRowColumn, binlogConfigForTask, - partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified()); + partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified(), + keysDesc.getClusterKeysColumnIds()); olapTable.addPartition(partition); } else if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { @@ -2559,7 +2565,7 @@ public class InternalCatalog implements CatalogIf { olapTable.getTimeSeriesCompactionFileCountThreshold(), olapTable.getTimeSeriesCompactionTimeThresholdSeconds(), storeRowColumn, binlogConfigForTask, - dataProperty.isStorageMediumSpecified()); + dataProperty.isStorageMediumSpecified(), keysDesc.getClusterKeysColumnIds()); olapTable.addPartition(partition); olapTable.getPartitionInfo().getDataProperty(partition.getId()) .setStoragePolicy(partionStoragePolicy); @@ -2961,6 +2967,14 @@ public class InternalCatalog implements CatalogIf { Set tabletIdSet = Sets.newHashSet(); long bufferSize = IdGeneratorUtil.getBufferSizeForTruncateTable(copiedTbl, origPartitions.values()); IdGeneratorBuffer idGeneratorBuffer = Env.getCurrentEnv().getIdGeneratorBuffer(bufferSize); + Map clusterKeyMap = new TreeMap<>(); + for (int i = 0; i < olapTable.getBaseSchema().size(); i++) { + Column column = olapTable.getBaseSchema().get(i); + if (column.getClusterKeyId() != -1) { + clusterKeyMap.put(column.getClusterKeyId(), i); + } + } + List clusterKeyIdxes = clusterKeyMap.values().stream().collect(Collectors.toList()); try { for (Map.Entry entry : origPartitions.entrySet()) { // the new partition must use new id @@ -2986,7 +3000,8 @@ public class InternalCatalog implements CatalogIf { olapTable.getTimeSeriesCompactionFileCountThreshold(), olapTable.getTimeSeriesCompactionTimeThresholdSeconds(), olapTable.storeRowColumn(), binlogConfig, - copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified()); + copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified(), + clusterKeyIdxes); newPartitions.add(newPartition); } } catch (DdlException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index b488dbe926..6eb9f2d75e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -202,6 +202,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -2333,17 +2334,30 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor olapTable.getDataSortInfo().getColNum()) { return false; } + List sortKeyColumns = olapTable.getFullSchema(); + if (olapTable.getEnableUniqueKeyMergeOnWrite()) { + Map clusterKeyMap = new TreeMap<>(); + for (Column column : olapTable.getFullSchema()) { + if (column.getClusterKeyId() != -1) { + clusterKeyMap.put(column.getClusterKeyId(), column); + } + } + if (!clusterKeyMap.isEmpty()) { + sortKeyColumns.clear(); + sortKeyColumns.addAll(clusterKeyMap.values()); + } + } for (int i = 0; i < sortExprs.size(); i++) { - // table key. - Column tableKey = olapTable.getFullSchema().get(i); + // sort key. + Column sortColumn = sortKeyColumns.get(i); // sort slot. Expr sortExpr = sortExprs.get(i); if (sortExpr instanceof SlotRef) { SlotRef slotRef = (SlotRef) sortExpr; - if (tableKey.equals(slotRef.getColumn())) { + if (sortColumn.equals(slotRef.getColumn())) { // ORDER BY DESC NULLS FIRST can not be optimized to only read file tail, // since NULLS is at file head but data is at tail - if (tableKey.isAllowNull() && nullsFirsts.get(i) && !isAscOrders.get(i)) { + if (sortColumn.isAllowNull() && nullsFirsts.get(i) && !isAscOrders.get(i)) { return false; } } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 96d8c08f12..530a9a7830 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1391,6 +1391,10 @@ public class OlapScanNode extends ScanNode { } msg.node_type = TPlanNodeType.OLAP_SCAN_NODE; + if (olapTable.getBaseSchema().stream().anyMatch(Column::isClusterKey)) { + keyColumnNames.clear(); + keyColumnTypes.clear(); + } msg.olap_scan_node = new TOlapScanNode(desc.getId().asInt(), keyColumnNames, keyColumnTypes, isPreAggregation); msg.olap_scan_node.setColumnsDesc(columnsDesc); msg.olap_scan_node.setIndexesDesc(indexDesc); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java index abb6f6e23d..6842e273a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java @@ -114,6 +114,7 @@ public class CreateReplicaTask extends AgentTask { private boolean storeRowColumn; private BinlogConfig binlogConfig; + private List clusterKeyIndexes; public CreateReplicaTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, long replicaId, short shortKeyColumnCount, int schemaHash, long version, @@ -220,6 +221,10 @@ public class CreateReplicaTask extends AgentTask { this.storageFormat = storageFormat; } + public void setClusterKeyIndexes(List clusterKeyIndexes) { + this.clusterKeyIndexes = clusterKeyIndexes; + } + public TCreateTabletReq toThrift() { TCreateTabletReq createTabletReq = new TCreateTabletReq(); createTabletReq.setTabletId(tabletId); @@ -265,7 +270,10 @@ public class CreateReplicaTask extends AgentTask { tSchema.setDeleteSignIdx(deleteSign); tSchema.setSequenceColIdx(sequenceCol); tSchema.setVersionColIdx(versionCol); - + if (!CollectionUtils.isEmpty(clusterKeyIndexes)) { + tSchema.setClusterKeyIdxes(clusterKeyIndexes); + LOG.debug("cluster key index={}, table_id={}, tablet_id={}", clusterKeyIndexes, tableId, tabletId); + } if (CollectionUtils.isNotEmpty(indexes)) { List tIndexes = new ArrayList<>(); for (Index index : indexes) { diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 4c49e31a7f..68a792623a 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -254,6 +254,7 @@ message TabletSchemaPB { repeated string partial_update_input_columns = 21; // deprecated optional bool enable_single_replica_compaction = 22 [default=false]; optional bool skip_write_index_on_load = 23 [default=false]; + repeated int32 cluster_key_idxes = 24; } enum TabletStatePB { diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index c161fc99e7..79bb014a90 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -44,6 +44,7 @@ struct TTabletSchema { 16: optional bool store_row_column = false 17: optional bool enable_single_replica_compaction = false 18: optional bool skip_write_index_on_load = false + 19: optional list cluster_key_idxes } // this enum stands for different storage format in src_backends diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index abaf8f8967..7c2f70b19c 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -41,6 +41,7 @@ struct TColumn { 16: optional string aggregation 17: optional bool result_is_nullable 18: optional bool is_auto_increment = false; + 19: optional i32 cluster_key_id = -1 } struct TSlotDescriptor { diff --git a/regression-test/data/compaction/test_compaction_uniq_cluster_keys_with_delete.out b/regression-test/data/compaction/test_compaction_uniq_cluster_keys_with_delete.out new file mode 100644 index 0000000000..ea06a5aa3c --- /dev/null +++ b/regression-test/data/compaction/test_compaction_uniq_cluster_keys_with_delete.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.140 2017-10-01T11:11:11.120111 2020-01-03T00:00 1 32 20 + +-- !select_default1 -- +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 + +-- !select_default2 -- +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 +4 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 + +-- !select_default3 -- +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 +4 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 + diff --git a/regression-test/data/compaction/test_compaction_uniq_keys_cluster_key.out b/regression-test/data/compaction/test_compaction_uniq_keys_cluster_key.out new file mode 100644 index 0000000000..0ab6761ce2 --- /dev/null +++ b/regression-test/data/compaction/test_compaction_uniq_keys_cluster_key.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.160 2017-10-01T11:11:11.100111 2020-01-02T00:00 1 31 19 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.140 2017-10-01T11:11:11.120111 2020-01-03T00:00 1 32 20 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 +4 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 + +-- !select_default2 -- +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.160 2017-10-01T11:11:11.100111 2020-01-02T00:00 1 31 19 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.140 2017-10-01T11:11:11.120111 2020-01-03T00:00 1 32 20 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 +4 2017-10-01 2017-10-01 2017-10-01T11:11:11.110 2017-10-01T11:11:11.110111 Beijing 10 1 \N \N \N \N 2020-01-05T00:00 1 34 20 + diff --git a/regression-test/data/point_query_p0/test_point_query_cluster_key.out b/regression-test/data/point_query_p0/test_point_query_cluster_key.out new file mode 100644 index 0000000000..d23a62474c --- /dev/null +++ b/regression-test/data/point_query_p0/test_point_query_cluster_key.out @@ -0,0 +1,262 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !point_select -- +1231 119291.110000000 ddd laooq \N 2020-01-01 12:36:38 \N 1022-01-01 \N 1.111 \N [119181.111100000, 819019.119100000, null] + +-- !point_select -- +1231 119291.110000000 ddd laooq \N 2020-01-01 12:36:38 \N 1022-01-01 \N 1.111 \N [119181.111100000, 819019.119100000, null] + +-- !point_select -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] + +-- !point_select -- +1232 12222.991211350 xxx laooq 2023-01-02 2020-01-01 12:36:38 522.762 2022-01-01 true 212.111 \N \N + +-- !point_select -- +251 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01 12:36:38 251.0 7022-01-01 true 90696620686827832.374 [11111.000000000] [] + +-- !point_select -- +252 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01 12:36:38 252.0 7022-01-01 false 90696620686827832.374 \N [0.000000000] + +-- !point_select -- +298 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01 12:36:38 298.0 7022-01-01 true 90696620686827832.374 [] [] + +-- !point_select -- +1235 991129292901.111380000 dd \N 2120-01-02 2020-01-01 12:36:38 652.692 5022-01-01 false 90696620686827832.374 [119181.111100000] ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"] + +-- !point_select -- +646464 6C616F6F71 + +-- !point_select -- +646464 6C616F6F71 + +-- !point_select -- +646464 6C616F6F71 + +-- !point_select -- +1235 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 true 1.111 [119291.192910000] ["111", "222", "333"] 1 + +-- !point_select -- +1235 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 true 1.111 [119291.192910000] ["111", "222", "333"] 1 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 0 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 0 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 0 + +-- !sql -- +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N 1.111 [119181.111100000, 819019.119100000, null] \N 0 0 + +-- !sql -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] 0 0 + +-- !sql -- +6120202020646464 6C616F6F71 32.92200050354004 + +-- !sql -- +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N 1.111 [119181.111100000, 819019.119100000, null] \N 0 0 + +-- !sql -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] 0 0 + +-- !sql -- +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N 1.111 [119181.111100000, 819019.119100000, null] \N 0 0 + +-- !sql -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] 0 0 + +-- !sql -- +0 1 2 3 + +-- !point_select -- +1231 119291.110000000 ddd laooq \N 2020-01-01 12:36:38 \N 1022-01-01 \N 1.111 \N [119181.111100000, 819019.119100000, null] + +-- !point_select -- +1231 119291.110000000 ddd laooq \N 2020-01-01 12:36:38 \N 1022-01-01 \N 1.111 \N [119181.111100000, 819019.119100000, null] + +-- !point_select -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] + +-- !point_select -- +1232 12222.991211350 xxx laooq 2023-01-02 2020-01-01 12:36:38 522.762 2022-01-01 true 212.111 \N \N + +-- !point_select -- +251 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01 12:36:38 251.0 7022-01-01 true 90696620686827832.374 [11111.000000000] [] + +-- !point_select -- +252 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01 12:36:38 252.0 7022-01-01 false 90696620686827832.374 \N [0.000000000] + +-- !point_select -- +298 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01 12:36:38 298.0 7022-01-01 true 90696620686827832.374 [] [] + +-- !point_select -- +1235 991129292901.111380000 dd \N 2120-01-02 2020-01-01 12:36:38 652.692 5022-01-01 false 90696620686827832.374 [119181.111100000] ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"] + +-- !point_select -- +646464 6C616F6F71 + +-- !point_select -- +646464 6C616F6F71 + +-- !point_select -- +646464 6C616F6F71 + +-- !point_select -- +1235 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 true 1.111 [119291.192910000] ["111", "222", "333"] 1 + +-- !point_select -- +1235 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 true 1.111 [119291.192910000] ["111", "222", "333"] 1 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 0 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 0 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 0 + +-- !sql -- +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N 1.111 [119181.111100000, 819019.119100000, null] \N 0 0 + +-- !sql -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] 0 0 + +-- !sql -- +6120202020646464 6C616F6F71 32.92200050354004 + +-- !sql -- +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N 1.111 [119181.111100000, 819019.119100000, null] \N 0 0 + +-- !sql -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] 0 0 + +-- !sql -- +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N 1.111 [119181.111100000, 819019.119100000, null] \N 0 0 + +-- !sql -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] 0 0 + +-- !sql -- +0 1 2 3 + +-- !point_select -- +1231 119291.110000000 ddd laooq \N 2020-01-01 12:36:38 \N 1022-01-01 \N 1.111 \N [119181.111100000, 819019.119100000, null] + +-- !point_select -- +1231 119291.110000000 ddd laooq \N 2020-01-01 12:36:38 \N 1022-01-01 \N 1.111 \N [119181.111100000, 819019.119100000, null] + +-- !point_select -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] + +-- !point_select -- +1232 12222.991211350 xxx laooq 2023-01-02 2020-01-01 12:36:38 522.762 2022-01-01 true 212.111 \N \N + +-- !point_select -- +251 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01 12:36:38 251.0 7022-01-01 true 90696620686827832.374 [11111.000000000] [] + +-- !point_select -- +252 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01 12:36:38 252.0 7022-01-01 false 90696620686827832.374 \N [0.000000000] + +-- !point_select -- +298 120939.111300000 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa laooq 2030-01-02 2020-01-01 12:36:38 298.0 7022-01-01 true 90696620686827832.374 [] [] + +-- !point_select -- +1235 991129292901.111380000 dd \N 2120-01-02 2020-01-01 12:36:38 652.692 5022-01-01 false 90696620686827832.374 [119181.111100000] ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"] + +-- !point_select -- +646464 6C616F6F71 + +-- !point_select -- +646464 6C616F6F71 + +-- !point_select -- +646464 6C616F6F71 + +-- !point_select -- +1235 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 true 1.111 [119291.192910000] ["111", "222", "333"] 1 + +-- !point_select -- +1235 120939.111300000 a ddd laooq 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 true 1.111 [119291.192910000] ["111", "222", "333"] 1 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 0 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 0 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 + +-- !point_select -- +1235 120939.111300000 a ddd xxxxxx 2030-01-02 2020-01-01 12:36:38 22.822 7022-01-01 false 1929111.111 [119291.192910000] ["111", "222", "333"] 2 0 + +-- !sql -- +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N 1.111 [119181.111100000, 819019.119100000, null] \N 0 0 + +-- !sql -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] 0 0 + +-- !sql -- +6120202020646464 6C616F6F71 32.92200050354004 + +-- !sql -- +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N 1.111 [119181.111100000, 819019.119100000, null] \N 0 0 + +-- !sql -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] 0 0 + +-- !sql -- +1231 119291.110000000 ddd laooq \N 2020-01-01T12:36:38 \N 1022-01-01 \N 1.111 [119181.111100000, 819019.119100000, null] \N 0 0 + +-- !sql -- +1237 120939.111300000 a ddd laooq 2030-01-02 2020-01-01T12:36:38 22.822 7022-01-01 false 90696620686827832.374 [1.100000000, 2.200000000, 3.300000000, 4.400000000, 5.500000000] [] 0 0 + +-- !sql -- +0 1 2 3 + diff --git a/regression-test/data/unique_with_mow_p0/cluster_key/test_delete_sign_delete_bitmap.out b/regression-test/data/unique_with_mow_p0/cluster_key/test_delete_sign_delete_bitmap.out new file mode 100644 index 0000000000..687aeab54a --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/cluster_key/test_delete_sign_delete_bitmap.out @@ -0,0 +1,54 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 + +-- !after_delete -- +2 2 2 2 2 +4 4 4 4 4 + +-- !1 -- +1 1 1 1 1 0 +1 1 1 1 1 1 +2 2 2 2 2 0 +3 3 3 3 3 0 +3 3 3 3 3 1 +4 4 4 4 4 0 +5 5 5 5 5 0 +5 5 5 5 5 1 + +-- !2 -- +2 2 2 2 2 0 +4 4 4 4 4 0 + +-- !sql -- +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +5 5 5 5 5 + +-- !after_delete -- +2 2 2 2 2 +4 4 4 4 4 + +-- !1 -- +1 1 1 1 1 0 +1 1 1 1 1 1 +2 2 2 2 2 0 +3 3 3 3 3 0 +3 3 3 3 3 1 +4 4 4 4 4 0 +5 5 5 5 5 0 +5 5 5 5 5 1 + +-- !2 -- +1 1 1 1 1 1 +2 2 2 2 2 0 +3 3 3 3 3 1 +4 4 4 4 4 0 +5 5 5 5 5 1 + diff --git a/regression-test/data/unique_with_mow_p0/cluster_key/test_mow_with_null_sequence.out b/regression-test/data/unique_with_mow_p0/cluster_key/test_mow_with_null_sequence.out new file mode 100644 index 0000000000..2c14cb05ec --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/cluster_key/test_mow_with_null_sequence.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +a abc address2 2022-10-20 +aa1234 abc address4 2022-12-11 +aa1235 abc address6 \N +ab abc address6 2022-11-20 + +-- !sql -- +a abc address2 100 +aa1234 abc address4 -1 +aa1235 abc address6 -1 +aa1236 abc address6 0 +ab abc address6 110 + diff --git a/regression-test/data/unique_with_mow_p0/cluster_key/test_schema_change.out b/regression-test/data/unique_with_mow_p0/cluster_key/test_schema_change.out new file mode 100644 index 0000000000..7381d98730 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/cluster_key/test_schema_change.out @@ -0,0 +1,159 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 20 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 21 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 + +-- !sql -- +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 + +-- !sql -- +5 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 +6 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 +6 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 +7 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 25 + +-- !sql -- +6 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 20 +6 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 +7 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 25 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 21 +6 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 +7 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 25 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 +6 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 +7 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 25 + +-- !sql -- +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 + +-- !sql -- +7 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 +6 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 +7 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 25 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 +6 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 +7 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 25 + +-- !sql -- +6 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 20 +6 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 22 +7 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 25 + +-- !sql -- +6 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 22 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 21 +6 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 22 +7 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 25 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 22 +6 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 22 +7 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 25 + +-- !sql -- +5 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 22 + +-- !sql -- +7 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 22 +6 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 22 +7 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 25 + +-- !sql -- +1 2017-10-01 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 1 \N 30 20 +2 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 1 \N 31 21 +3 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 20 +4 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 1 \N 32 22 +5 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 22 +6 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 22 +7 2017-10-01 Beijing 10 1 \N 2020-01-05T00:00 1 \N 34 25 + +-- !sql -- +6 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 \N 34 22 + diff --git a/regression-test/data/unique_with_mow_p0/cluster_key/test_unique_mow_sequence.out b/regression-test/data/unique_with_mow_p0/cluster_key/test_unique_mow_sequence.out new file mode 100644 index 0000000000..65b68e4152 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/cluster_key/test_unique_mow_sequence.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 Customer#000000001 j5JsirBM9P MOROCCO 0 MOROCCO AFRICA 25-989-741-2988 BUILDING +2 Customer#000000002 487LW1dovn6Q4dMVym JORDAN 1 JORDAN MIDDLE EAST 23-768-687-3665 AUTOMOBILE +3 Customer#000000003 fkRGN8n ARGENTINA7 ARGENTINA AMERICA 11-719-748-3364 AUTOMOBILE +4 Customer#000000004 4u58h f EGYPT 4 EGYPT MIDDLE EAST 14-128-190-5944 MACHINERY +5 Customer#000000005 hwBtxkoBF qSW4KrI CANADA 5 CANADA AMERICA 13-750-942-6364 HOUSEHOLD + +-- !sql -- +2996 Customer#000002996 PFd,H,pC PERU 1 PERU AMERICA 27-412-836-3763 FURNITURE +2997 Customer#000002997 LiVKxN3lQHLunID ALGERIA 0 ALGERIA AFRICA 10-600-583-9608 FURNITURE +2998 Customer#000002998 waJRUwjblh3sJbglX9gS9w PERU 7 PERU AMERICA 27-747-219-4938 AUTOMOBILE +2999 Customer#000002999 HaPy4sQ MiANd0pR5uA7 VIETNAM 5 VIETNAM ASIA 31-297-683-9811 MACHINERY +3000 Customer#000003000 ,5Yw1O EGYPT 4 EGYPT MIDDLE EAST 14-645-615-5901 FURNITURE + +-- !sql -- +1 Customer#000000001 j5JsirBM9P MOROCCO 0 MOROCCO AFRICA 25-989-741-2988 BUILDING + +-- !sql -- +3000 Customer#000003000 ,5Yw1O EGYPT 4 EGYPT MIDDLE EAST 14-645-615-5901 FURNITURE + +-- !sql -- + +-- !sql -- + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.1.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.1.out new file mode 100644 index 0000000000..92604403fd --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.1.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q1.1 -- +\N + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.2.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.2.out new file mode 100644 index 0000000000..22731ac444 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.2.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q1.2 -- +\N + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.3.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.3.out new file mode 100644 index 0000000000..71908d1f12 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.3.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q1.3 -- +\N + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.1.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.1.out new file mode 100644 index 0000000000..9d56f6e633 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.1.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q2.1 -- +29165996 1992 MFGR#121 +23120066 1992 MFGR#1210 +52982362 1992 MFGR#1211 +30954680 1992 MFGR#1212 +15288453 1992 MFGR#1213 +7655070 1992 MFGR#1214 +22246540 1992 MFGR#1215 +19716439 1992 MFGR#1216 +43666251 1992 MFGR#1217 +22759602 1992 MFGR#1218 +23318799 1992 MFGR#1219 +74056106 1992 MFGR#122 +51050565 1992 MFGR#1220 +38878674 1992 MFGR#1221 +16558051 1992 MFGR#1222 +26690787 1992 MFGR#1223 +76498594 1992 MFGR#1224 +32608903 1992 MFGR#1225 +47636685 1992 MFGR#1226 +27691433 1992 MFGR#1227 +32513490 1992 MFGR#1228 +35514258 1992 MFGR#1229 +17199862 1992 MFGR#123 +24678908 1992 MFGR#1230 +26231337 1992 MFGR#1231 +36330900 1992 MFGR#1232 +24946678 1992 MFGR#1233 +36431683 1992 MFGR#1234 +39368479 1992 MFGR#1235 +44456974 1992 MFGR#1236 +31443810 1992 MFGR#1237 +49003021 1992 MFGR#1238 +31379822 1992 MFGR#1239 +24245603 1992 MFGR#124 +49870826 1992 MFGR#1240 +28194770 1992 MFGR#125 +40503844 1992 MFGR#126 +36027836 1992 MFGR#127 +35881895 1992 MFGR#128 +21732451 1992 MFGR#129 + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.2.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.2.out new file mode 100644 index 0000000000..debe195012 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.2.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q2.2 -- +28235270 1992 MFGR#2221 +64071827 1992 MFGR#2222 +48591160 1992 MFGR#2223 +20416501 1992 MFGR#2224 +74950776 1992 MFGR#2225 +60628045 1992 MFGR#2226 +39273349 1992 MFGR#2227 +66658087 1992 MFGR#2228 + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.3.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.3.out new file mode 100644 index 0000000000..40b3220406 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.3.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q2.3 -- +89380397 1992 MFGR#2239 + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.1.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.1.out new file mode 100644 index 0000000000..a50f6a20d5 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.1.out @@ -0,0 +1,28 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q3.1 -- +JAPAN CHINA 1992 637991852 +VIETNAM CHINA 1992 621845377 +INDONESIA CHINA 1992 621316255 +CHINA CHINA 1992 614550901 +INDIA CHINA 1992 561966207 +INDIA INDONESIA 1992 487449629 +INDONESIA INDONESIA 1992 477417717 +JAPAN INDONESIA 1992 476513261 +JAPAN VIETNAM 1992 468999429 +INDONESIA JAPAN 1992 465870469 +VIETNAM INDONESIA 1992 462424521 +INDIA JAPAN 1992 412186106 +JAPAN JAPAN 1992 399179790 +VIETNAM JAPAN 1992 395247587 +JAPAN INDIA 1992 393835589 +CHINA INDONESIA 1992 352903905 +CHINA INDIA 1992 348359904 +VIETNAM VIETNAM 1992 342176333 +INDIA VIETNAM 1992 334582962 +INDIA INDIA 1992 329354089 +CHINA JAPAN 1992 327558220 +CHINA VIETNAM 1992 324763767 +INDONESIA INDIA 1992 310417666 +VIETNAM INDIA 1992 296225919 +INDONESIA VIETNAM 1992 278083418 + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.2.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.2.out new file mode 100644 index 0000000000..1109fa3ce8 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.2.out @@ -0,0 +1,51 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q3.2 -- +UNITED ST4 UNITED ST0 1992 34626982 +UNITED ST4 UNITED ST3 1992 29767238 +UNITED ST1 UNITED ST9 1992 25644597 +UNITED ST2 UNITED ST0 1992 23943154 +UNITED ST4 UNITED ST9 1992 21189183 +UNITED ST0 UNITED ST0 1992 18293852 +UNITED ST7 UNITED ST3 1992 17996772 +UNITED ST9 UNITED ST3 1992 17863433 +UNITED ST1 UNITED ST7 1992 17410800 +UNITED ST2 UNITED ST3 1992 15331073 +UNITED ST5 UNITED ST9 1992 14448179 +UNITED ST1 UNITED ST3 1992 13938002 +UNITED ST5 UNITED ST6 1992 12398029 +UNITED ST9 UNITED ST2 1992 12370917 +UNITED ST2 UNITED ST9 1992 12343455 +UNITED ST5 UNITED ST0 1992 12301234 +UNITED ST6 UNITED ST0 1992 11900889 +UNITED ST4 UNITED ST2 1992 11696334 +UNITED ST4 UNITED ST6 1992 11369008 +UNITED ST1 UNITED ST6 1992 11000283 +UNITED ST1 UNITED ST0 1992 10878084 +UNITED ST4 UNITED ST7 1992 10151573 +UNITED ST5 UNITED ST2 1992 9917834 +UNITED ST7 UNITED ST7 1992 9715656 +UNITED ST6 UNITED ST6 1992 8685228 +UNITED ST2 UNITED ST2 1992 8313714 +UNITED ST1 UNITED ST2 1992 8004700 +UNITED ST2 UNITED ST7 1992 7759164 +UNITED ST0 UNITED ST7 1992 7137641 +UNITED ST9 UNITED ST7 1992 6703890 +UNITED ST6 UNITED ST9 1992 6597261 +UNITED ST7 UNITED ST2 1992 6125476 +UNITED ST7 UNITED ST6 1992 6058017 +UNITED ST5 UNITED ST3 1992 5862031 +UNITED ST8 UNITED ST9 1992 5690491 +UNITED ST7 UNITED ST9 1992 5403152 +UNITED ST9 UNITED ST0 1992 4816370 +UNITED ST9 UNITED ST9 1992 4234523 +UNITED ST3 UNITED ST3 1992 4080199 +UNITED ST5 UNITED ST7 1992 3936271 +UNITED ST8 UNITED ST0 1992 3574169 +UNITED ST0 UNITED ST3 1992 3201624 +UNITED ST3 UNITED ST9 1992 2614811 +UNITED ST8 UNITED ST7 1992 2373825 +UNITED ST9 UNITED ST6 1992 2066609 +UNITED ST7 UNITED ST0 1992 1882015 +UNITED ST6 UNITED ST3 1992 1873819 +UNITED ST6 UNITED ST2 1992 291566 + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.3.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.3.out new file mode 100644 index 0000000000..6f33841912 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.3.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q3.3 -- +UNITED KI5 UNITED KI1 1992 4397192 + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.4.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.4.out new file mode 100644 index 0000000000..3738fc2859 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.4.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q3.4 -- + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.1.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.1.out new file mode 100644 index 0000000000..00bc9ddd7c --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.1.out @@ -0,0 +1,8 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q4.1 -- +1992 ARGENTINA 533196600 +1992 BRAZIL 684224630 +1992 CANADA 532686194 +1992 PERU 586223155 +1992 UNITED STATES 682387184 + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.2.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.2.out new file mode 100644 index 0000000000..30fae3d4bf --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.2.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q4.2 -- + diff --git a/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.3.out b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.3.out new file mode 100644 index 0000000000..741016a89d --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.3.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q4.3 -- + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.1.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.1.out new file mode 100644 index 0000000000..92604403fd --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.1.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q1.1 -- +\N + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.2.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.2.out new file mode 100644 index 0000000000..22731ac444 --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.2.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q1.2 -- +\N + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.3.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.3.out new file mode 100644 index 0000000000..71908d1f12 --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.3.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q1.3 -- +\N + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.1.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.1.out new file mode 100644 index 0000000000..9d56f6e633 --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.1.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q2.1 -- +29165996 1992 MFGR#121 +23120066 1992 MFGR#1210 +52982362 1992 MFGR#1211 +30954680 1992 MFGR#1212 +15288453 1992 MFGR#1213 +7655070 1992 MFGR#1214 +22246540 1992 MFGR#1215 +19716439 1992 MFGR#1216 +43666251 1992 MFGR#1217 +22759602 1992 MFGR#1218 +23318799 1992 MFGR#1219 +74056106 1992 MFGR#122 +51050565 1992 MFGR#1220 +38878674 1992 MFGR#1221 +16558051 1992 MFGR#1222 +26690787 1992 MFGR#1223 +76498594 1992 MFGR#1224 +32608903 1992 MFGR#1225 +47636685 1992 MFGR#1226 +27691433 1992 MFGR#1227 +32513490 1992 MFGR#1228 +35514258 1992 MFGR#1229 +17199862 1992 MFGR#123 +24678908 1992 MFGR#1230 +26231337 1992 MFGR#1231 +36330900 1992 MFGR#1232 +24946678 1992 MFGR#1233 +36431683 1992 MFGR#1234 +39368479 1992 MFGR#1235 +44456974 1992 MFGR#1236 +31443810 1992 MFGR#1237 +49003021 1992 MFGR#1238 +31379822 1992 MFGR#1239 +24245603 1992 MFGR#124 +49870826 1992 MFGR#1240 +28194770 1992 MFGR#125 +40503844 1992 MFGR#126 +36027836 1992 MFGR#127 +35881895 1992 MFGR#128 +21732451 1992 MFGR#129 + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.2.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.2.out new file mode 100644 index 0000000000..debe195012 --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.2.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q2.2 -- +28235270 1992 MFGR#2221 +64071827 1992 MFGR#2222 +48591160 1992 MFGR#2223 +20416501 1992 MFGR#2224 +74950776 1992 MFGR#2225 +60628045 1992 MFGR#2226 +39273349 1992 MFGR#2227 +66658087 1992 MFGR#2228 + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.3.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.3.out new file mode 100644 index 0000000000..40b3220406 --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.3.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q2.3 -- +89380397 1992 MFGR#2239 + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.1.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.1.out new file mode 100644 index 0000000000..a50f6a20d5 --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.1.out @@ -0,0 +1,28 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q3.1 -- +JAPAN CHINA 1992 637991852 +VIETNAM CHINA 1992 621845377 +INDONESIA CHINA 1992 621316255 +CHINA CHINA 1992 614550901 +INDIA CHINA 1992 561966207 +INDIA INDONESIA 1992 487449629 +INDONESIA INDONESIA 1992 477417717 +JAPAN INDONESIA 1992 476513261 +JAPAN VIETNAM 1992 468999429 +INDONESIA JAPAN 1992 465870469 +VIETNAM INDONESIA 1992 462424521 +INDIA JAPAN 1992 412186106 +JAPAN JAPAN 1992 399179790 +VIETNAM JAPAN 1992 395247587 +JAPAN INDIA 1992 393835589 +CHINA INDONESIA 1992 352903905 +CHINA INDIA 1992 348359904 +VIETNAM VIETNAM 1992 342176333 +INDIA VIETNAM 1992 334582962 +INDIA INDIA 1992 329354089 +CHINA JAPAN 1992 327558220 +CHINA VIETNAM 1992 324763767 +INDONESIA INDIA 1992 310417666 +VIETNAM INDIA 1992 296225919 +INDONESIA VIETNAM 1992 278083418 + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.2.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.2.out new file mode 100644 index 0000000000..1109fa3ce8 --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.2.out @@ -0,0 +1,51 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q3.2 -- +UNITED ST4 UNITED ST0 1992 34626982 +UNITED ST4 UNITED ST3 1992 29767238 +UNITED ST1 UNITED ST9 1992 25644597 +UNITED ST2 UNITED ST0 1992 23943154 +UNITED ST4 UNITED ST9 1992 21189183 +UNITED ST0 UNITED ST0 1992 18293852 +UNITED ST7 UNITED ST3 1992 17996772 +UNITED ST9 UNITED ST3 1992 17863433 +UNITED ST1 UNITED ST7 1992 17410800 +UNITED ST2 UNITED ST3 1992 15331073 +UNITED ST5 UNITED ST9 1992 14448179 +UNITED ST1 UNITED ST3 1992 13938002 +UNITED ST5 UNITED ST6 1992 12398029 +UNITED ST9 UNITED ST2 1992 12370917 +UNITED ST2 UNITED ST9 1992 12343455 +UNITED ST5 UNITED ST0 1992 12301234 +UNITED ST6 UNITED ST0 1992 11900889 +UNITED ST4 UNITED ST2 1992 11696334 +UNITED ST4 UNITED ST6 1992 11369008 +UNITED ST1 UNITED ST6 1992 11000283 +UNITED ST1 UNITED ST0 1992 10878084 +UNITED ST4 UNITED ST7 1992 10151573 +UNITED ST5 UNITED ST2 1992 9917834 +UNITED ST7 UNITED ST7 1992 9715656 +UNITED ST6 UNITED ST6 1992 8685228 +UNITED ST2 UNITED ST2 1992 8313714 +UNITED ST1 UNITED ST2 1992 8004700 +UNITED ST2 UNITED ST7 1992 7759164 +UNITED ST0 UNITED ST7 1992 7137641 +UNITED ST9 UNITED ST7 1992 6703890 +UNITED ST6 UNITED ST9 1992 6597261 +UNITED ST7 UNITED ST2 1992 6125476 +UNITED ST7 UNITED ST6 1992 6058017 +UNITED ST5 UNITED ST3 1992 5862031 +UNITED ST8 UNITED ST9 1992 5690491 +UNITED ST7 UNITED ST9 1992 5403152 +UNITED ST9 UNITED ST0 1992 4816370 +UNITED ST9 UNITED ST9 1992 4234523 +UNITED ST3 UNITED ST3 1992 4080199 +UNITED ST5 UNITED ST7 1992 3936271 +UNITED ST8 UNITED ST0 1992 3574169 +UNITED ST0 UNITED ST3 1992 3201624 +UNITED ST3 UNITED ST9 1992 2614811 +UNITED ST8 UNITED ST7 1992 2373825 +UNITED ST9 UNITED ST6 1992 2066609 +UNITED ST7 UNITED ST0 1992 1882015 +UNITED ST6 UNITED ST3 1992 1873819 +UNITED ST6 UNITED ST2 1992 291566 + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.3.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.3.out new file mode 100644 index 0000000000..6f33841912 --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.3.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q3.3 -- +UNITED KI5 UNITED KI1 1992 4397192 + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.4.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.4.out new file mode 100644 index 0000000000..3738fc2859 --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.4.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q3.4 -- + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.1.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.1.out new file mode 100644 index 0000000000..00bc9ddd7c --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.1.out @@ -0,0 +1,8 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q4.1 -- +1992 ARGENTINA 533196600 +1992 BRAZIL 684224630 +1992 CANADA 532686194 +1992 PERU 586223155 +1992 UNITED STATES 682387184 + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.2.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.2.out new file mode 100644 index 0000000000..30fae3d4bf --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.2.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q4.2 -- + diff --git a/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.3.out b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.3.out new file mode 100644 index 0000000000..741016a89d --- /dev/null +++ b/regression-test/data/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.3.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q4.3 -- + diff --git a/regression-test/suites/compaction/test_compaction_uniq_cluster_keys_with_delete.groovy b/regression-test/suites/compaction/test_compaction_uniq_cluster_keys_with_delete.groovy new file mode 100644 index 0000000000..dc6cfe7208 --- /dev/null +++ b/regression-test/suites/compaction/test_compaction_uniq_cluster_keys_with_delete.groovy @@ -0,0 +1,177 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_compaction_uniq_cluster_keys_with_delete") { + def tableName = "test_compaction_uniq_cluster_keys_with_delete" + + try { + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + backend_id = backendId_to_backendIP.keySet()[0] + def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) + + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `datev2` DATEV2 NOT NULL COMMENT "数据灌入日期时间", + `datetimev2_1` DATETIMEV2(3) NOT NULL COMMENT "数据灌入日期时间", + `datetimev2_2` DATETIMEV2(6) NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `datetime_val1` DATETIMEV2(3) DEFAULT "1970-01-01 00:00:00.111" COMMENT "用户最后一次访问时间", + `datetime_val2` DATETIME(6) DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + UNIQUE KEY(`user_id`, `date`, `datev2`, `datetimev2_1`, `datetimev2_2`, `city`, `age`, `sex`) + CLUSTER BY(`sex`, `date`, `cost`) + DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2017-10-01 11:11:11.170000', '2017-10-01 11:11:11.110111', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2017-10-01 11:11:11.160000', '2017-10-01 11:11:11.100111', '2020-01-02', 1, 31, 19) + """ + + sql """ + DELETE FROM ${tableName} where user_id <= 5 + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2017-10-01 11:11:11.150000', '2017-10-01 11:11:11.130111', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2017-10-01 11:11:11.140000', '2017-10-01 11:11:11.120111', '2020-01-03', 1, 32, 20) + """ + + sql """ + DELETE FROM ${tableName} where user_id <= 1 + """ + + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2017-10-01 11:11:11.100000', '2017-10-01 11:11:11.140111', '2020-01-03', 1, 32, 22) + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-04', '2020-01-04', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.150111', '2020-01-04', 1, 33, 21) + """ + + sql """ + DELETE FROM ${tableName} where user_id <= 2 + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, NULL, NULL, NULL, NULL, '2020-01-05', 1, 34, 20) + """ + + qt_select_default1 """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, NULL, NULL, NULL, NULL, '2020-01-05', 1, 34, 20) + """ + + qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + + //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus + String[][] tablets = sql """ show tablets from ${tableName}; """ + + // trigger compactions for all tablets in ${tableName} + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactJson = parseJson(out.trim()) + if (compactJson.status.toLowerCase() == "fail") { + assertEquals(disableAutoCompaction, false) + logger.info("Compaction was done automatically!") + } + if (disableAutoCompaction) { + assertEquals("success", compactJson.status.toLowerCase()) + } + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + def compactionStatusUrlIndex = 18 + (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex]) + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + assert (rowCount < 8) + qt_select_default3 """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + } finally { + try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys_cluster_key.groovy b/regression-test/suites/compaction/test_compaction_uniq_keys_cluster_key.groovy new file mode 100644 index 0000000000..39d485c35f --- /dev/null +++ b/regression-test/suites/compaction/test_compaction_uniq_keys_cluster_key.groovy @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_compaction_uniq_keys_cluster_key") { + def tableName = "compaction_uniq_keys_cluster_key" + + try { + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + backend_id = backendId_to_backendIP.keySet()[0] + def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) + + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def configList = parseJson(out.trim()) + assert configList instanceof List + + boolean disableAutoCompaction = true + for (Object ele in (List) configList) { + assert ele instanceof List + if (((List) ele)[0] == "disable_auto_compaction") { + disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) + } + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `datev2` DATEV2 NOT NULL COMMENT "数据灌入日期时间", + `datetimev2_1` DATETIMEV2(3) NOT NULL COMMENT "数据灌入日期时间", + `datetimev2_2` DATETIMEV2(6) NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `datetime_val1` DATETIMEV2(3) DEFAULT "1970-01-01 00:00:00.111" COMMENT "用户最后一次访问时间", + `datetime_val2` DATETIME(6) DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + UNIQUE KEY(`user_id`, `date`, `datev2`, `datetimev2_1`, `datetimev2_2`, `city`, `age`, `sex`) + CLUSTER BY(`last_visit_date_not_null`, `age`, `sex`, `city`) + DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2017-10-01 11:11:11.170000', '2017-10-01 11:11:11.110111', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2017-10-01 11:11:11.160000', '2017-10-01 11:11:11.100111', '2020-01-02', 1, 31, 19) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2017-10-01 11:11:11.150000', '2017-10-01 11:11:11.130111', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2017-10-01 11:11:11.140000', '2017-10-01 11:11:11.120111', '2020-01-03', 1, 32, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2017-10-01 11:11:11.100000', '2017-10-01 11:11:11.140111', '2020-01-03', 1, 32, 22) + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, '2020-01-04', '2020-01-04', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.150111', '2020-01-04', 1, 33, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, NULL, NULL, NULL, NULL, '2020-01-05', 1, 34, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', '2017-10-01', '2017-10-01 11:11:11.110000', '2017-10-01 11:11:11.110111', 'Beijing', 10, 1, NULL, NULL, NULL, NULL, '2020-01-05', 1, 34, 20) + """ + + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + + //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus + String[][] tablets = sql """ show tablets from ${tableName}; """ + + // trigger compactions for all tablets in ${tableName} + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + backend_id = tablet[2] + (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactJson = parseJson(out.trim()) + if (compactJson.status.toLowerCase() == "fail") { + assertEquals(disableAutoCompaction, false) + logger.info("Compaction was done automatically!") + } + if (disableAutoCompaction) { + assertEquals("success", compactJson.status.toLowerCase()) + } + } + + // wait for all compactions done + for (String[] tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet[0] + backend_id = tablet[2] + (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + int rowCount = 0 + for (String[] tablet in tablets) { + String tablet_id = tablet[0] + def compactionStatusUrlIndex = 18 + (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex]) + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + rowCount += Integer.parseInt(rowset.split(" ")[1]) + } + } + assert (rowCount < 8) + qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + } finally { + // try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/point_query_p0/test_point_query.groovy b/regression-test/suites/point_query_p0/test_point_query.groovy index 8c01c8cc20..de562c086d 100644 --- a/regression-test/suites/point_query_p0/test_point_query.groovy +++ b/regression-test/suites/point_query_p0/test_point_query.groovy @@ -254,7 +254,7 @@ suite("test_point_query") { "disable_auto_compaction" = "false" );""" sql """insert into ${tableName} values (0, "1", "2", "3")""" - qt_sql "select /*+ SET_VAR(enable_nereids_planner=false) */ * from test_query where customer_key = 0" + qt_sql """select /*+ SET_VAR(enable_nereids_planner=false) */ * from ${tableName} where customer_key = 0""" } } } finally { diff --git a/regression-test/suites/point_query_p0/test_point_query_cluster_key.groovy b/regression-test/suites/point_query_p0/test_point_query_cluster_key.groovy new file mode 100644 index 0000000000..f1bc1e8913 --- /dev/null +++ b/regression-test/suites/point_query_p0/test_point_query_cluster_key.groovy @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.math.BigDecimal; + +suite("test_point_query_cluster_key") { + def set_be_config = { key, value -> + String backend_id; + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + backend_id = backendId_to_backendIP.keySet()[0] + def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) + logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) + } + try { + set_be_config.call("disable_storage_row_cache", "false") + // nereids do not support point query now + sql """set enable_nereids_planner=false""" + + def user = context.config.jdbcUser + def password = context.config.jdbcPassword + def realDb = "regression_test_serving_p0" + def tableName = realDb + ".tbl_point_query_cluster_key" + sql "CREATE DATABASE IF NOT EXISTS ${realDb}" + + // Parse url + String jdbcUrl = context.config.jdbcUrl + String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3) + def sql_ip = urlWithoutSchema.substring(0, urlWithoutSchema.indexOf(":")) + def sql_port + if (urlWithoutSchema.indexOf("/") >= 0) { + // e.g: jdbc:mysql://locahost:8080/?a=b + sql_port = urlWithoutSchema.substring(urlWithoutSchema.indexOf(":") + 1, urlWithoutSchema.indexOf("/")) + } else { + // e.g: jdbc:mysql://locahost:8080 + sql_port = urlWithoutSchema.substring(urlWithoutSchema.indexOf(":") + 1) + } + // set server side prepared statement url + def prepare_url = "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb + "?&useServerPrepStmts=true" + + def generateString = {len -> + def str = "" + for (int i = 0; i < len; i++) { + str += "a" + } + return str + } + + def nprep_sql = { sql_str -> + def url_without_prep = "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb + connect(user = user, password = password, url = url_without_prep) { + sql sql_str + } + } + + sql """DROP TABLE IF EXISTS ${tableName}""" + test { + // abnormal case + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int NULL COMMENT "" + ) ENGINE=OLAP + UNIQUE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "store_row_column" = "true", + "light_schema_change" = "false" + ) + """ + exception "errCode = 2, detailMessage = Row store column rely on light schema change, enable light schema change first" + } + + def create_table_sql = { property -> + return String.format(""" + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL COMMENT "", + `k2` decimalv3(27, 9) NULL COMMENT "", + `k3` varchar(300) NULL COMMENT "", + `k4` varchar(30) NULL COMMENT "", + `k5` date NULL COMMENT "", + `k6` datetime NULL COMMENT "", + `k7` float NULL COMMENT "", + `k8` datev2 NULL COMMENT "", + `k9` boolean NULL COMMENT "", + `k10` decimalv3(20, 3) NULL COMMENT "", + `k11` array NULL COMMENT "", + `k12` array NULL COMMENT "" + ) ENGINE=OLAP + UNIQUE KEY(`k1`, `k2`, `k3`) + CLUSTER BY(`k9`, `k5`, `k4`, `k2`) + DISTRIBUTED BY HASH(`k1`, k2, k3) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "store_row_column" = "true", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + %s + "storage_format" = "V2") + """, property) + } + + for (int i = 0; i < 3; i++) { + tableName = realDb + ".tbl_point_query_cluster_key" + i + sql """DROP TABLE IF EXISTS ${tableName}""" + if (i == 0) { + def sql0 = create_table_sql("") + sql """ ${sql0} """ + } else if (i == 1) { + def sql1 = create_table_sql("\"function_column.sequence_type\" = 'int',") + sql """ ${sql1} """ + } else { + def sql2 = create_table_sql("\"function_column.sequence_col\" = 'k6',") + sql """ ${sql2} """ + } + sql """ INSERT INTO ${tableName} VALUES(1231, 119291.11, "ddd", "laooq", null, "2020-01-01 12:36:38", null, "1022-01-01 11:30:38", null, 1.111112, [119181.1111, 819019.1191, null], null) """ + sql """ INSERT INTO ${tableName} VALUES(1232, 12222.99121135, "xxx", "laooq", "2023-01-02", "2020-01-01 12:36:38", 522.762, "2022-01-01 11:30:38", 1, 212.111, null, null) """ + sql """ INSERT INTO ${tableName} VALUES(1233, 1.392932911, "yyy", "laooq", "2024-01-02", "2020-01-01 12:36:38", 52.862, "3022-01-01 11:30:38", 0, 5973903488739435.668, [119181.1111, null, 819019.1191], ["dijiiixxx"]) """ + sql """ INSERT INTO ${tableName} VALUES(1234, 12919291.129191137, "xxddd", "laooq", "2025-01-02", "2020-01-01 12:36:38", 552.872, "4022-01-01 11:30:38", 1, 5973903488739435.668, [1888109181.192111, 192129019.1191], ["1", "2", "3"]) """ + sql """ INSERT INTO ${tableName} VALUES(1235, 991129292901.11138, "dd", null, "2120-01-02", "2020-01-01 12:36:38", 652.692, "5022-01-01 11:30:38", 0, 90696620686827832.374, [119181.1111], ["${generateString(251)}"]) """ + sql """ INSERT INTO ${tableName} VALUES(1236, 100320.11139, "laa ddd", "laooq", "2220-01-02", "2020-01-01 12:36:38", 2.7692, "6022-01-01 11:30:38", 1, 23698.299, [], ["${generateString(251)}"]) """ + sql """ INSERT INTO ${tableName} VALUES(1237, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", 0, 90696620686827832.374, [1.1, 2.2, 3.3, 4.4, 5.5], []) """ + sql """ INSERT INTO ${tableName} VALUES(251, 120939.11130, "${generateString(251)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 251, "7022-01-01 11:30:38", 1, 90696620686827832.374, [11111], []) """ + sql """ INSERT INTO ${tableName} VALUES(252, 120939.11130, "${generateString(252)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 252, "7022-01-01 11:30:38", 0, 90696620686827832.374, [0], null) """ + sql """ INSERT INTO ${tableName} VALUES(298, 120939.11130, "${generateString(298)}", "laooq", "2030-01-02", "2020-01-01 12:36:38", 298, "7022-01-01 11:30:38", 1, 90696620686827832.374, [], []) """ + + def result1 = connect(user=user, password=password, url=prepare_url) { + def stmt = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=false) */ * from ${tableName} where k1 = ? and k2 = ? and k3 = ?" + assertEquals(stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement); + stmt.setInt(1, 1231) + stmt.setBigDecimal(2, new BigDecimal("119291.11")) + stmt.setString(3, "ddd") + qe_point_select stmt + stmt.setInt(1, 1231) + stmt.setBigDecimal(2, new BigDecimal("119291.11")) + stmt.setString(3, "ddd") + qe_point_select stmt + stmt.setInt(1, 1237) + stmt.setBigDecimal(2, new BigDecimal("120939.11130")) + stmt.setString(3, "a ddd") + qe_point_select stmt + + stmt.setInt(1, 1232) + stmt.setBigDecimal(2, new BigDecimal("12222.99121135")) + stmt.setString(3, 'xxx') + qe_point_select stmt + + stmt.setInt(1, 251) + stmt.setBigDecimal(2, new BigDecimal("120939.11130")) + stmt.setString(3, generateString(251)) + qe_point_select stmt + + stmt.setInt(1, 252) + stmt.setBigDecimal(2, new BigDecimal("120939.11130")) + stmt.setString(3, generateString(252)) + qe_point_select stmt + + stmt.setInt(1, 298) + stmt.setBigDecimal(2, new BigDecimal("120939.11130")) + stmt.setString(3, generateString(298)) + qe_point_select stmt + stmt.close() + + stmt = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=false) */ * from ${tableName} where k1 = 1235 and k2 = ? and k3 = ?" + assertEquals(stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement); + stmt.setBigDecimal(1, new BigDecimal("991129292901.11138")) + stmt.setString(2, "dd") + qe_point_select stmt + + def stmt_fn = prepareStatement "select /*+ SET_VAR(enable_nereids_planner=false) */ hex(k3), hex(k4) from ${tableName} where k1 = ? and k2 =? and k3 = ?" + assertEquals(stmt_fn.class, com.mysql.cj.jdbc.ServerPreparedStatement); + stmt_fn.setInt(1, 1231) + stmt_fn.setBigDecimal(2, new BigDecimal("119291.11")) + stmt_fn.setString(3, "ddd") + qe_point_select stmt_fn + qe_point_select stmt_fn + qe_point_select stmt_fn + + nprep_sql """ + ALTER table ${tableName} ADD COLUMN new_column0 INT default "0"; + """ + sleep(1); + nprep_sql """ INSERT INTO ${tableName} VALUES(1235, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", 1, 1.1111299, [119291.19291], ["111", "222", "333"], 1) """ + stmt.setBigDecimal(1, new BigDecimal("120939.11130")) + stmt.setString(2, "a ddd") + qe_point_select stmt + qe_point_select stmt + // invalidate cache + nprep_sql """ INSERT INTO ${tableName} VALUES(1235, 120939.11130, "a ddd", "xxxxxx", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", 0, 1929111.1111,[119291.19291], ["111", "222", "333"], 2) """ + qe_point_select stmt + qe_point_select stmt + qe_point_select stmt + nprep_sql """ + ALTER table ${tableName} ADD COLUMN new_column1 INT default "0"; + """ + qe_point_select stmt + qe_point_select stmt + nprep_sql """ + ALTER table ${tableName} DROP COLUMN new_column1; + """ + qe_point_select stmt + qe_point_select stmt + + sql """ + ALTER table ${tableName} ADD COLUMN new_column1 INT default "0"; + """ + qe_point_select stmt + } + // disable useServerPrepStmts + def result2 = connect(user=user, password=password, url=context.config.jdbcUrl) { + qt_sql """select /*+ SET_VAR(enable_nereids_planner=false) */ * from ${tableName} where k1 = 1231 and k2 = 119291.11 and k3 = 'ddd'""" + qt_sql """select /*+ SET_VAR(enable_nereids_planner=false) */ * from ${tableName} where k1 = 1237 and k2 = 120939.11130 and k3 = 'a ddd'""" + qt_sql """select /*+ SET_VAR(enable_nereids_planner=false) */ hex(k3), hex(k4), k7 + 10.1 from ${tableName} where k1 = 1237 and k2 = 120939.11130 and k3 = 'a ddd'""" + // prepared text + sql """ prepare stmt1 from select * from ${tableName} where k1 = % and k2 = % and k3 = % """ + qt_sql """execute stmt1 using (1231, 119291.11, 'ddd')""" + qt_sql """execute stmt1 using (1237, 120939.11130, 'a ddd')""" + + sql """prepare stmt2 from select * from ${tableName} where k1 = % and k2 = % and k3 = %""" + qt_sql """execute stmt2 using (1231, 119291.11, 'ddd')""" + qt_sql """execute stmt2 using (1237, 120939.11130, 'a ddd')""" + tableName = "test_query_cluster_key" + sql """DROP TABLE IF EXISTS ${tableName}""" + sql """CREATE TABLE ${tableName} ( + `customer_key` bigint(20) NULL, + `customer_btm_value_0` text NULL, + `customer_btm_value_1` text NULL, + `customer_btm_value_2` text NULL + ) ENGINE=OLAP + UNIQUE KEY(`customer_key`) + CLUSTER BY(`customer_btm_value_1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`customer_key`) BUCKETS 16 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "store_row_column" = "true", + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "false" + );""" + sql """insert into ${tableName} values (0, "1", "2", "3")""" + qt_sql """select /*+ SET_VAR(enable_nereids_planner=false) */ * from ${tableName} where customer_key = 0""" + } + } + } finally { + set_be_config.call("disable_storage_row_cache", "true") + } +} \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/cluster_key/test_create_table.groovy b/regression-test/suites/unique_with_mow_p0/cluster_key/test_create_table.groovy new file mode 100644 index 0000000000..91c2bc6ba6 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/cluster_key/test_create_table.groovy @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_create_table") { + def tableName = "cluster_key_test_create_table" + sql """ DROP TABLE IF EXISTS ${tableName} """ + onFinish { + try_sql("DROP TABLE IF EXISTS ${tableName}") + } + + // duplicate table with cluster keys + test { + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "" + ) + DUPLICATE KEY (`c_custkey`) + CLUSTER BY (`c_name`, `c_address`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + exception "Syntax error" + } + + // mor unique table with cluster keys + test { + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "" + ) + UNIQUE KEY (`c_custkey`) + CLUSTER BY (`c_name`, `c_address`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + exception "Cluster keys only support unique keys table which enabled enable_unique_key_merge_on_write" + } + + // mor unique table with cluster keys + test { + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "" + ) + UNIQUE KEY (`c_custkey`) + CLUSTER BY (`c_name`, `c_address`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false" + ); + """ + exception "Cluster keys only support unique keys table which enabled enable_unique_key_merge_on_write" + } + + // mow unique table with invalid cluster keys + test { + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "" + ) + UNIQUE KEY (`c_custkey`) + CLUSTER BY (`c_name`, `c_addresses`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + exception "Key cluster column[c_addresses] doesn't exist" + } + + // mow unique table with duplicate cluster keys + test { + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "" + ) + UNIQUE KEY (`c_custkey`) + CLUSTER BY (`c_name`, `c_address`, `c_name`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + exception "Duplicate cluster key column[c_name]" + } + + // mow unique table with same cluster and unique keys + test { + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "" + ) + UNIQUE KEY (`c_custkey`) + CLUSTER BY (`c_custkey`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + exception "Unique keys and cluster keys should be different" + } + + // mow unique table with short key is part of unique keys + test { + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_age` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "" + ) + UNIQUE KEY (`c_custkey`, `c_age`, `c_name`) + CLUSTER BY (`c_custkey`, `c_age`, `c_address`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "short_key" = "2" + ); + """ + exception "2 short keys is a part of unique keys" + } + + // mow unique table with short key is part of unique keys + test { + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_age` int(11) NOT NULL COMMENT "", + `c_name` varchar(100) NOT NULL COMMENT "", + `c_address` varchar(100) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "" + ) + UNIQUE KEY (`c_custkey`, `c_age`, `c_name`, `c_address`) + CLUSTER BY (`c_custkey`, `c_age`, `c_name`, `c_city`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + exception "3 short keys is a part of unique keys" + } + + // success to create mow unique table with cluster keys + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "" + ) + UNIQUE KEY (`c_custkey`) + CLUSTER BY (`c_name`, `c_city`, `c_address`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ +} diff --git a/regression-test/suites/unique_with_mow_p0/cluster_key/test_delete_sign_delete_bitmap.groovy b/regression-test/suites/unique_with_mow_p0/cluster_key/test_delete_sign_delete_bitmap.groovy new file mode 100644 index 0000000000..ea7e2c5e7d --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/cluster_key/test_delete_sign_delete_bitmap.groovy @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_delete_sign_delete_bitmap') { + +// def tableName1 = "test_delete_sign_delete_bitmap1" +// sql "DROP TABLE IF EXISTS ${tableName1};" +// sql """ CREATE TABLE IF NOT EXISTS ${tableName1} ( +// `k1` int NOT NULL, +// `c1` int, +// `c2` int, +// `c3` int, +// `c4` int +// )UNIQUE KEY(k1) +// CLUSTER BY(c1, c2) +// DISTRIBUTED BY HASH(k1) BUCKETS 1 +// PROPERTIES ( +// "enable_unique_key_merge_on_write" = "true", +// "disable_auto_compaction" = "true", +// "replication_num" = "1" +// );""" +// +// sql "insert into ${tableName1} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" +// qt_sql "select * from ${tableName1} order by k1,c1,c2,c3,c4;" +// // sql "insert into ${tableName1}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__) select k1,c1,c2,c3,c4,1 from ${tableName1} where k1 in (1,3,5);" +// sql """insert into ${tableName1}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__) values(1,1,1,1,1,1),(3,3,3,3,3,1),(5,5,5,5,5,1);""" +// sql "sync" +// qt_after_delete "select * from ${tableName1} order by k1,c1,c2,c3,c4;" +// sql "set skip_delete_sign=true;" +// sql "set skip_storage_engine_merge=true;" +// sql "set skip_delete_bitmap=true;" +// sql "sync" +// // skip_delete_bitmap=true, skip_delete_sign=true +// qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" +// +// sql "set skip_delete_sign=true;" +// sql "set skip_delete_bitmap=false;" +// sql "sync" +// // skip_delete_bitmap=false, skip_delete_sign=true +// qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" +// sql "drop table if exists ${tableName1};" +// +// +// sql "set skip_delete_sign=false;" +// sql "set skip_storage_engine_merge=false;" +// sql "set skip_delete_bitmap=false;" +// sql "sync" +// def tableName2 = "test_delete_sign_delete_bitmap2" +// sql "DROP TABLE IF EXISTS ${tableName2};" +// sql """ CREATE TABLE IF NOT EXISTS ${tableName2} ( +// `k1` int NOT NULL, +// `c1` int, +// `c2` int, +// `c3` int, +// `c4` int +// )UNIQUE KEY(k1) +// CLUSTER BY(c4, c3) +// DISTRIBUTED BY HASH(k1) BUCKETS 1 +// PROPERTIES ( +// "enable_unique_key_merge_on_write" = "true", +// "disable_auto_compaction" = "true", +// "replication_num" = "1", +// "function_column.sequence_col" = 'c4' +// );""" +// +// sql "insert into ${tableName2} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);" +// qt_sql "select * from ${tableName2} order by k1,c1,c2,c3,c4;" +// sql """insert into ${tableName2}(k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__) values(1,1,1,1,1,1),(3,3,3,3,3,1),(5,5,5,5,5,1);""" +// sql "sync" +// qt_after_delete "select * from ${tableName2} order by k1,c1,c2,c3,c4;" +// sql "set skip_delete_sign=true;" +// sql "set skip_storage_engine_merge=true;" +// sql "set skip_delete_bitmap=true;" +// sql "sync" +// // skip_delete_bitmap=true, skip_delete_sign=true +// qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" +// +// sql "set skip_delete_sign=true;" +// sql "set skip_delete_bitmap=false;" +// sql "sync" +// // skip_delete_bitmap=false, skip_delete_sign=true +// qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;" +// sql "drop table if exists ${tableName2};" +} diff --git a/regression-test/suites/unique_with_mow_p0/cluster_key/test_mow_with_null_sequence.groovy b/regression-test/suites/unique_with_mow_p0/cluster_key/test_mow_with_null_sequence.groovy new file mode 100644 index 0000000000..bf3ce215a9 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/cluster_key/test_mow_with_null_sequence.groovy @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_mow_with_null_sequence") { + def tableName = "test_null_sequence" + sql """ DROP TABLE IF EXISTS $tableName """ + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` varchar(20) NOT NULL COMMENT "", + `c_name` varchar(20) NOT NULL COMMENT "", + `c_address` varchar(20) NOT NULL COMMENT "", + `c_date` date NULL COMMENT "" + ) + UNIQUE KEY (`c_custkey`) + CLUSTER BY(`c_name`, `c_date`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10 + PROPERTIES ( + "function_column.sequence_col" = 'c_date', + "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "true" + ); + """ + + + sql """ insert into $tableName values('a', 'abc', 'address1', NULL) """ + sql """ insert into $tableName values('a', 'abc', 'address2', '2022-10-20') """ + sql """ insert into $tableName values('a', 'abc', 'address3', NULL) """ + sql """ insert into $tableName values('ab', 'abc', 'address4', NULL) """ + sql """ insert into $tableName values('ab', 'abc', 'address5', '2022-10-20') """ + sql """ insert into $tableName values('ab', 'abc', 'address6', '2022-11-20') """ + sql """ insert into $tableName values('ab', 'abc', 'address6', '2022-11-15') """ + sql """ insert into $tableName values('aa1234', 'abc', 'address4', '2022-12-11') """ + sql """ insert into $tableName values('aa1234', 'abc', 'address5', NULL) """ + sql """ insert into $tableName values('aa1235', 'abc', 'address6', NULL) """ + + order_qt_sql "select * from $tableName" + + sql """ DROP TABLE IF EXISTS $tableName """ + + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` varchar(20) NOT NULL COMMENT "", + `c_name` varchar(20) NOT NULL COMMENT "", + `c_address` varchar(20) NOT NULL COMMENT "", + `c_int` int NULL COMMENT "" + ) + UNIQUE KEY (`c_custkey`) + CLUSTER BY(`c_int`, `c_name`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10 + PROPERTIES ( + "function_column.sequence_col" = 'c_int', + "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "true" + ); + """ + sql """ insert into $tableName values('a', 'abc', 'address1', NULL) """ + sql """ insert into $tableName values('a', 'abc', 'address2', 100) """ + sql """ insert into $tableName values('a', 'abc', 'address3', NULL) """ + + sql """ insert into $tableName values('ab', 'abc', 'address4', NULL) """ + sql """ insert into $tableName values('ab', 'abc', 'address5', -10) """ + sql """ insert into $tableName values('ab', 'abc', 'address6', 110) """ + sql """ insert into $tableName values('ab', 'abc', 'address6', 100) """ + + sql """ insert into $tableName values('aa1234', 'abc', 'address4', -1) """ + sql """ insert into $tableName values('aa1234', 'abc', 'address5', NULL) """ + + sql """ insert into $tableName values('aa1235', 'abc', 'address6', NULL) """ + sql """ insert into $tableName values('aa1235', 'abc', 'address6', -1) """ + + sql """ insert into $tableName values('aa1236', 'abc', 'address6', NULL) """ + sql """ insert into $tableName values('aa1236', 'abc', 'address6', 0) """ + sql """ insert into $tableName values('aa1236', 'abc', 'address6', NULL) """ + + order_qt_sql "select * from $tableName" + + sql """ DROP TABLE IF EXISTS $tableName """ +} diff --git a/regression-test/suites/unique_with_mow_p0/cluster_key/test_pk_uk_case.groovy b/regression-test/suites/unique_with_mow_p0/cluster_key/test_pk_uk_case.groovy new file mode 100644 index 0000000000..337bf3c1a7 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/cluster_key/test_pk_uk_case.groovy @@ -0,0 +1,260 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods; +import java.util.Random; +import org.apache.commons.lang.RandomStringUtils; +import java.util.Date; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.Map; +import java.util.UUID; +import java.time.format.DateTimeFormatter; + +suite("test_pk_uk_case") { + def tableNamePk = "primary_key_pk_uk" + def tableNameUk = "unique_key_pk_uk" + + onFinish { + try_sql("DROP TABLE IF EXISTS ${tableNamePk}") + try_sql("DROP TABLE IF EXISTS ${tableNameUk}") + } + + sql """ DROP TABLE IF EXISTS ${tableNamePk} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableNamePk} ( + L_ORDERKEY INTEGER NOT NULL, + L_PARTKEY INTEGER NOT NULL, + L_SUPPKEY INTEGER NOT NULL, + L_LINENUMBER INTEGER NOT NULL, + L_QUANTITY DECIMAL(15,2) NOT NULL, + L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL, + L_DISCOUNT DECIMAL(15,2) NOT NULL, + L_TAX DECIMAL(15,2) NOT NULL, + L_RETURNFLAG CHAR(1) NOT NULL, + L_LINESTATUS CHAR(1) NOT NULL, + L_SHIPDATE DATE NOT NULL, + L_COMMITDATE DATE NOT NULL, + L_RECEIPTDATE DATE NOT NULL, + L_SHIPINSTRUCT CHAR(60) NOT NULL, + L_SHIPMODE CHAR(60) NOT NULL, + L_COMMENT VARCHAR(60) NOT NULL + ) + UNIQUE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER) + CLUSTER BY(L_PARTKEY, L_RETURNFLAG) + DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "true" + ) + """ + + sql """ DROP TABLE IF EXISTS ${tableNameUk} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableNameUk} ( + L_ORDERKEY INTEGER NOT NULL, + L_PARTKEY INTEGER NOT NULL, + L_SUPPKEY INTEGER NOT NULL, + L_LINENUMBER INTEGER NOT NULL, + L_QUANTITY DECIMAL(15,2) NOT NULL, + L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL, + L_DISCOUNT DECIMAL(15,2) NOT NULL, + L_TAX DECIMAL(15,2) NOT NULL, + L_RETURNFLAG CHAR(1) NOT NULL, + L_LINESTATUS CHAR(1) NOT NULL, + L_SHIPDATE DATE NOT NULL, + L_COMMITDATE DATE NOT NULL, + L_RECEIPTDATE DATE NOT NULL, + L_SHIPINSTRUCT CHAR(60) NOT NULL, + L_SHIPMODE CHAR(60) NOT NULL, + L_COMMENT VARCHAR(60) NOT NULL + ) + UNIQUE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER) + DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "false" + ) + """ + + Random rd = new Random() + def order_key = rd.nextInt(1000) + def part_key = rd.nextInt(1000) + def sub_key = 13 + def line_num = 29 + def decimal = 111.11 + def city = RandomStringUtils.randomAlphabetic(10) + def name = UUID.randomUUID().toString() + def date = DateTimeFormatter.ofPattern("yyyy-MM-dd").format(LocalDateTime.now()) + for (int idx = 0; idx < 10; idx++) { + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + // insert batch key + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + sql "sync" + + // count(*) + def result0 = sql """ SELECT count(*) FROM ${tableNamePk}; """ + def result1 = sql """ SELECT count(*) FROM ${tableNameUk}; """ + logger.info("result:" + result0[0][0] + "|" + result1[0][0]) + assertTrue(result0[0]==result1[0]) + if (result0[0][0]!=result1[0][0]) { + logger.info("result:" + result0[0][0] + "|" + result1[0][0]) + } + + result0 = sql """ SELECT + l_returnflag, + l_linestatus, + sum(l_quantity) AS sum_qty, + sum(l_extendedprice) AS sum_base_price, + sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + avg(l_quantity) AS avg_qty, + avg(l_extendedprice) AS avg_price, + avg(l_discount) AS avg_disc, + count(*) AS count_order + FROM + ${tableNamePk} + GROUP BY + l_returnflag, + l_linestatus + ORDER BY + l_returnflag, + l_linestatus + """ + result1 = sql """ SELECT + l_returnflag, + l_linestatus, + sum(l_quantity) AS sum_qty, + sum(l_extendedprice) AS sum_base_price, + sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + avg(l_quantity) AS avg_qty, + avg(l_extendedprice) AS avg_price, + avg(l_discount) AS avg_disc, + count(*) AS count_order + FROM + ${tableNameUk} + GROUP BY + l_returnflag, + l_linestatus + ORDER BY + l_returnflag, + l_linestatus + """ + assertTrue(result0.size()==result1.size()) + for (int i = 0; i < result0.size(); ++i) { + for (j = 0; j < result0[0].size(); j++) { + logger.info("result: " + result0[i][j] + "|" + result1[i][j]) + assertTrue(result0[i][j]==result1[i][j]) + } + } + + // delete + if (idx % 10 == 0) { + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + result0 = sql """ SELECT count(*) FROM ${tableNamePk} where L_ORDERKEY < $order_key and L_PARTKEY < $part_key; """ + result1 = sql """ SELECT count(*) FROM ${tableNameUk} where L_ORDERKEY < $order_key and L_PARTKEY < $part_key""" + logger.info("result:" + result0[0][0] + "|" + result1[0][0]) + sql "DELETE FROM ${tableNamePk} where L_ORDERKEY < $order_key and L_PARTKEY < $part_key" + sql "DELETE FROM ${tableNameUk} where L_ORDERKEY < $order_key and L_PARTKEY < $part_key" + } + } +} diff --git a/regression-test/suites/unique_with_mow_p0/cluster_key/test_primary_key_simple_case.groovy b/regression-test/suites/unique_with_mow_p0/cluster_key/test_primary_key_simple_case.groovy new file mode 100644 index 0000000000..788f934fdf --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/cluster_key/test_primary_key_simple_case.groovy @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_primary_key_simple_case") { + def tableName = "cluster_key_test_primary_key_simple_case" + onFinish { + // try_sql("DROP TABLE IF EXISTS ${tableName}") + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) + CLUSTER BY(`user_id`, `age`, `cost`, `sex`) + DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "true" + ); + """ + + sql """ INSERT INTO ${tableName} VALUES + (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} VALUES + (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + sql """ INSERT INTO ${tableName} VALUES + (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 22) + """ + + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 20) + """ + + result = sql """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + assertTrue(result.size() == 5) + assertTrue(result[0].size() == 11) + + // insert a duplicate key + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 21) + """ + result = sql """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + assertTrue(result.size() == 5) + assertTrue(result[4][10] == 21) + + // insert a duplicate key + sql """ INSERT INTO ${tableName} VALUES + (5, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 22) + """ + result = sql """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + assertTrue(result.size() == 5) + logger.info("fuck: " + result.size()) + assertTrue(result[4][10] == 22) + + result = sql """ SELECT * FROM ${tableName} t where user_id = 5; """ + assertTrue(result.size() == 1) + assertTrue(result[0][10] == 22) + + result = sql """ SELECT COUNT(*) FROM ${tableName};""" + assertTrue(result.size() == 1) + assertTrue(result[0][0] == 5) + + // insert a new key + sql """ INSERT INTO ${tableName} VALUES + (6, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 22) + """ + result = sql """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + assertTrue(result.size() == 6) + + // insert batch key + sql """ INSERT INTO ${tableName} VALUES + (7, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 22), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 23), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 24), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 25) + """ + result = sql """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + assertTrue(result.size() == 7) + assertTrue(result[6][10] == 25) +} diff --git a/regression-test/suites/unique_with_mow_p0/cluster_key/test_schema_change.groovy b/regression-test/suites/unique_with_mow_p0/cluster_key/test_schema_change.groovy new file mode 100644 index 0000000000..3dafd0d0a8 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/cluster_key/test_schema_change.groovy @@ -0,0 +1,334 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_schema_change") { + def tableName = "test_schema_change" + onFinish { + // try_sql("DROP TABLE IF EXISTS ${tableName}") + } + + def getAlterTableState = { + def retry = 0 + while (true) { + sleep(2000) + def state = sql "show alter table column where tablename = '${tableName}' order by CreateTime desc " + logger.info("alter table state: ${state}") + if (state.size()> 0 && state[0][9] == "FINISHED") { + return true + } + retry++ + if (retry >= 10) { + return false + } + } + return false + } + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `date` DATE NOT NULL COMMENT "数据灌入日期时间", + `city` VARCHAR(20) COMMENT "用户所在城市", + `age` SMALLINT COMMENT "用户年龄", + `sex` TINYINT COMMENT "用户性别", + `last_visit_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `last_update_date` DATETIME DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间", + `last_visit_date_not_null` DATETIME NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间", + `cost` BIGINT DEFAULT "0" COMMENT "用户总消费", + `comment` VARCHAR(5), + `max_dwell_time` INT DEFAULT "0" COMMENT "用户最大停留时间", + `min_dwell_time` INT DEFAULT "99999" COMMENT "用户最小停留时间") + UNIQUE KEY(`user_id`, `date`, `city`, `age`, `sex`) + CLUSTER BY(`cost`, `comment`) + DISTRIBUTED BY HASH(`user_id`) + PROPERTIES ( "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "true" + ); + """ + + // 1. add a value column(any position after key column) + for (int i = 0; i < 2; i++) { + if (i == 1) { + sql """ alter table ${tableName} ADD column score int after sex; """ + assertTrue(getAlterTableState(), "add column should success") + } + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 22) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (5, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 20) + """ + + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + + // insert a duplicate key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (5, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 21) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + + // insert a duplicate key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (5, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 22) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + + qt_sql """ + SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t + where user_id = 5; + """ + + qt_sql """ SELECT COUNT(*) FROM ${tableName};""" + + // insert a new key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (6, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 22) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + + // insert batch key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES + (7, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 22), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 23), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 24), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 25) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + qt_sql """ + SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_visit_date`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t + where user_id = 6 and sex = 1 ORDER BY user_id; + """ + } + + // 2. drop a value column + sql """ alter table ${tableName} DROP column last_visit_date; """ + assertTrue(getAlterTableState(), "drop column should success"); + { + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 22) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (5, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 20) + """ + + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t + where user_id = 6 and sex = 1 ORDER BY user_id; """ + + // insert a duplicate key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (5, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 21) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + + // insert a duplicate key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (5, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 22) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t + where user_id = 5; """ + + qt_sql """ SELECT COUNT(*) FROM ${tableName};""" + + // insert a new key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (6, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 22) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + + // insert batch key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES + (7, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 22), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 23), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 24), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 25) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + qt_sql """ SELECT * FROM ${tableName} t + where user_id = 6 and sex = 1 ORDER BY user_id; """ + } + + // 3.0 add a cluster key key column is not support + // 3.1 drop a cluster key column is not support + // unique table key column also can not be dropped: Can not drop key column in Unique data model table + test { + sql """ alter table ${tableName} DROP column cost; """ + exception "Can not drop cluster key column in Unique data model table" + } + + // 4. modify a cluster key column + test { + sql """ alter table ${tableName} MODIFY column `comment` varchar(20); """ + exception "Can not modify cluster key column" + } + + // 5. modify column order should success (Temporarily throw exception) + test { + sql """ + alter table ${tableName} ORDER BY (`user_id`, `date`, `city`, `age`, `sex`, `max_dwell_time`, `comment`, `min_dwell_time`, `last_visit_date_not_null`, `cost`, `score`, `last_update_date`); + """ + exception "Can not modify column order in Unique data model table" + } + /*assertTrue(getAlterTableState(), "alter column order should success"); + { + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', 1, 30, 20) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', 1, 31, 21) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 20) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (4, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', 1, 32, 22) + """ + + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (5, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 20) + """ + + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t + where user_id = 6 and sex = 1 ORDER BY user_id; """ + + // insert a duplicate key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (5, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 21) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + + // insert a duplicate key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (5, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 22) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t where user_id = 5; """ + + qt_sql """ SELECT COUNT(*) FROM ${tableName};""" + + // insert a new key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES (6, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 22) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + + // insert batch key + sql """ INSERT INTO ${tableName} + (`user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, + `cost`, `max_dwell_time`, `min_dwell_time`) + VALUES + (7, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 22), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 23), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 24), + (7, '2017-10-01', 'Beijing', 10, 1, NULL, '2020-01-05', 1, 34, 25) + """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t ORDER BY user_id; """ + qt_sql """ SELECT `user_id`, `date`, `city`, `age`, `sex`, `last_update_date`, `last_visit_date_not_null`, `cost`, `comment`, `max_dwell_time`, `min_dwell_time` FROM ${tableName} t + where user_id = 6 and sex = 1 ORDER BY user_id; """ + }*/ +} diff --git a/regression-test/suites/unique_with_mow_p0/cluster_key/test_unique_mow_sequence.groovy b/regression-test/suites/unique_with_mow_p0/cluster_key/test_unique_mow_sequence.groovy new file mode 100644 index 0000000000..c3ded30c04 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/cluster_key/test_unique_mow_sequence.groovy @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_unique_mow_sequence") { + def tableName = "test_mow_sequence" + sql """ DROP TABLE IF EXISTS $tableName """ + sql """ + CREATE TABLE `$tableName` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "", + `c_nation` varchar(16) NOT NULL COMMENT "", + `c_region` varchar(13) NOT NULL COMMENT "", + `c_phone` varchar(16) NOT NULL COMMENT "", + `c_mktsegment` varchar(11) NOT NULL COMMENT "" + ) + UNIQUE KEY (`c_custkey`) + CLUSTER BY (`c_nation`, `c_mktsegment`, `c_region`) + DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10 + PROPERTIES ( + "function_column.sequence_type" = 'int', + "compression"="zstd", + "replication_num" = "1", + "disable_auto_compaction" = "true", + "enable_unique_key_merge_on_write" = "true" + ); + """ + + streamLoad { + table "${tableName}" + + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', 'c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use' + set 'function_column.sequence_col', 'c_custkey' + + file """${getS3Url()}/regression/ssb/sf0.1/customer.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + + sql "sync" + + order_qt_sql "select * from $tableName where c_custkey < 6;" + + order_qt_sql "select * from $tableName where c_custkey > 2995;" + + qt_sql "select * from $tableName where c_custkey = 1;" + + qt_sql "select * from $tableName where c_custkey = 3000;" + + qt_sql "select * from $tableName where c_custkey = 3001;" + + qt_sql "select * from $tableName where c_custkey = 0;" +} diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_create.sql new file mode 100644 index 0000000000..789c8fd79b --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_create.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS `customer` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "", + `c_nation` varchar(16) NOT NULL COMMENT "", + `c_region` varchar(13) NOT NULL COMMENT "", + `c_phone` varchar(16) NOT NULL COMMENT "", + `c_mktsegment` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`c_custkey`) +CLUSTER BY (`c_name`, `c_phone`, `c_city`) +DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_delete.sql new file mode 100644 index 0000000000..fe22a226fe --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_delete.sql @@ -0,0 +1 @@ +truncate table customer; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_part_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_part_delete.sql new file mode 100644 index 0000000000..a9d1b34d68 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_part_delete.sql @@ -0,0 +1 @@ +delete from customer where c_custkey > 1500 ; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_sequence_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_sequence_create.sql new file mode 100644 index 0000000000..2b560cfdac --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/customer_sequence_create.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS `customer` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "", + `c_nation` varchar(16) NOT NULL COMMENT "", + `c_region` varchar(13) NOT NULL COMMENT "", + `c_phone` varchar(16) NOT NULL COMMENT "", + `c_mktsegment` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`c_custkey`) +CLUSTER BY (`c_nation`, `c_phone`, `c_city`, `c_name`) +DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10 +PROPERTIES ( +"function_column.sequence_type" = 'int', +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_create.sql new file mode 100644 index 0000000000..cf6b4b6a73 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_create.sql @@ -0,0 +1,28 @@ +CREATE TABLE IF NOT EXISTS `date` ( + `d_datekey` int(11) NOT NULL COMMENT "", + `d_date` varchar(20) NOT NULL COMMENT "", + `d_dayofweek` varchar(10) NOT NULL COMMENT "", + `d_month` varchar(11) NOT NULL COMMENT "", + `d_year` int(11) NOT NULL COMMENT "", + `d_yearmonthnum` int(11) NOT NULL COMMENT "", + `d_yearmonth` varchar(9) NOT NULL COMMENT "", + `d_daynuminweek` int(11) NOT NULL COMMENT "", + `d_daynuminmonth` int(11) NOT NULL COMMENT "", + `d_daynuminyear` int(11) NOT NULL COMMENT "", + `d_monthnuminyear` int(11) NOT NULL COMMENT "", + `d_weeknuminyear` int(11) NOT NULL COMMENT "", + `d_sellingseason` varchar(14) NOT NULL COMMENT "", + `d_lastdayinweekfl` int(11) NOT NULL COMMENT "", + `d_lastdayinmonthfl` int(11) NOT NULL COMMENT "", + `d_holidayfl` int(11) NOT NULL COMMENT "", + `d_weekdayfl` int(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`d_datekey`) +CLUSTER BY (`d_holidayfl`) +DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_delete.sql new file mode 100644 index 0000000000..12933cbbad --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_delete.sql @@ -0,0 +1 @@ +truncate table `date`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_part_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_part_delete.sql new file mode 100644 index 0000000000..0c21b27cc4 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_part_delete.sql @@ -0,0 +1 @@ +delete from `date` where d_datekey >= '19920701' and d_datekey <= '19920731'; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_sequence_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_sequence_create.sql new file mode 100644 index 0000000000..4ff7ba2208 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/date_sequence_create.sql @@ -0,0 +1,29 @@ +CREATE TABLE IF NOT EXISTS `date` ( + `d_datekey` int(11) NOT NULL COMMENT "", + `d_date` varchar(20) NOT NULL COMMENT "", + `d_dayofweek` varchar(10) NOT NULL COMMENT "", + `d_month` varchar(11) NOT NULL COMMENT "", + `d_year` int(11) NOT NULL COMMENT "", + `d_yearmonthnum` int(11) NOT NULL COMMENT "", + `d_yearmonth` varchar(9) NOT NULL COMMENT "", + `d_daynuminweek` int(11) NOT NULL COMMENT "", + `d_daynuminmonth` int(11) NOT NULL COMMENT "", + `d_daynuminyear` int(11) NOT NULL COMMENT "", + `d_monthnuminyear` int(11) NOT NULL COMMENT "", + `d_weeknuminyear` int(11) NOT NULL COMMENT "", + `d_sellingseason` varchar(14) NOT NULL COMMENT "", + `d_lastdayinweekfl` int(11) NOT NULL COMMENT "", + `d_lastdayinmonthfl` int(11) NOT NULL COMMENT "", + `d_holidayfl` int(11) NOT NULL COMMENT "", + `d_weekdayfl` int(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`d_datekey`) +CLUSTER BY (`d_yearmonth`, `d_holidayfl`) +DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1 +PROPERTIES ( +"function_column.sequence_type" = 'int', +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_create.sql new file mode 100644 index 0000000000..2dff3181c9 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_create.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS `lineorder` ( + `lo_orderdate` int(11) NOT NULL COMMENT "", + `lo_orderkey` bigint(20) NOT NULL COMMENT "", + `lo_linenumber` bigint(20) NOT NULL COMMENT "", + `lo_custkey` int(11) NOT NULL COMMENT "", + `lo_partkey` int(11) NOT NULL COMMENT "", + `lo_suppkey` int(11) NOT NULL COMMENT "", + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", + `lo_shippriority` int(11) NOT NULL COMMENT "", + `lo_quantity` bigint(20) NOT NULL COMMENT "", + `lo_extendedprice` bigint(20) NOT NULL COMMENT "", + `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "", + `lo_discount` bigint(20) NOT NULL COMMENT "", + `lo_revenue` bigint(20) NOT NULL COMMENT "", + `lo_supplycost` bigint(20) NOT NULL COMMENT "", + `lo_tax` bigint(20) NOT NULL COMMENT "", + `lo_commitdate` bigint(20) NOT NULL COMMENT "", + `lo_shipmode` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`lo_orderdate`,`lo_orderkey`,`lo_linenumber`) +CLUSTER BY (`lo_orderdate`, `lo_suppkey`, `lo_shipmode`) +PARTITION BY RANGE(`lo_orderdate`) +(PARTITION p1992 VALUES [("-2147483648"), ("19930101")), +PARTITION p1993 VALUES [("19930101"), ("19940101")), +PARTITION p1994 VALUES [("19940101"), ("19950101")), +PARTITION p1995 VALUES [("19950101"), ("19960101")), +PARTITION p1996 VALUES [("19960101"), ("19970101")), +PARTITION p1997 VALUES [("19970101"), ("19980101")), +PARTITION p1998 VALUES [("19980101"), ("19990101"))) +DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 48 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_delete.sql new file mode 100644 index 0000000000..329e040060 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_delete.sql @@ -0,0 +1 @@ +truncate table lineorder; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_part_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_part_delete.sql new file mode 100644 index 0000000000..abb7ded433 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_part_delete.sql @@ -0,0 +1 @@ +delete from lineorder where lo_orderkey >= 240001 and lo_orderkey <= 360000; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_sequence_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_sequence_create.sql new file mode 100644 index 0000000000..d639e7f2bb --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/lineorder_sequence_create.sql @@ -0,0 +1,37 @@ +CREATE TABLE IF NOT EXISTS `lineorder` ( + `lo_orderdate` int(11) NOT NULL COMMENT "", + `lo_orderkey` bigint(20) NOT NULL COMMENT "", + `lo_linenumber` bigint(20) NOT NULL COMMENT "", + `lo_custkey` int(11) NOT NULL COMMENT "", + `lo_partkey` int(11) NOT NULL COMMENT "", + `lo_suppkey` int(11) NOT NULL COMMENT "", + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", + `lo_shippriority` int(11) NOT NULL COMMENT "", + `lo_quantity` bigint(20) NOT NULL COMMENT "", + `lo_extendedprice` bigint(20) NOT NULL COMMENT "", + `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "", + `lo_discount` bigint(20) NOT NULL COMMENT "", + `lo_revenue` bigint(20) NOT NULL COMMENT "", + `lo_supplycost` bigint(20) NOT NULL COMMENT "", + `lo_tax` bigint(20) NOT NULL COMMENT "", + `lo_commitdate` bigint(20) NOT NULL COMMENT "", + `lo_shipmode` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`lo_orderdate`,`lo_orderkey`,`lo_linenumber`) +CLUSTER BY (`lo_quantity`, `lo_suppkey`) +PARTITION BY RANGE(`lo_orderdate`) +(PARTITION p1992 VALUES [("-2147483648"), ("19930101")), +PARTITION p1993 VALUES [("19930101"), ("19940101")), +PARTITION p1994 VALUES [("19940101"), ("19950101")), +PARTITION p1995 VALUES [("19950101"), ("19960101")), +PARTITION p1996 VALUES [("19960101"), ("19970101")), +PARTITION p1997 VALUES [("19970101"), ("19980101")), +PARTITION p1998 VALUES [("19980101"), ("19990101"))) +DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 48 +PROPERTIES ( +"function_column.sequence_type" = 'int', +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_create.sql new file mode 100644 index 0000000000..b1b01bcaec --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_create.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS `part` ( + `p_partkey` int(11) NOT NULL COMMENT "", + `p_name` varchar(23) NOT NULL COMMENT "", + `p_mfgr` varchar(7) NOT NULL COMMENT "", + `p_category` varchar(8) NOT NULL COMMENT "", + `p_brand` varchar(10) NOT NULL COMMENT "", + `p_color` varchar(12) NOT NULL COMMENT "", + `p_type` varchar(26) NOT NULL COMMENT "", + `p_size` int(11) NOT NULL COMMENT "", + `p_container` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`p_partkey`) +CLUSTER BY (`p_color`, `p_name`) +DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_delete.sql new file mode 100644 index 0000000000..02c6abd253 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_delete.sql @@ -0,0 +1 @@ +truncate table `part`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_part_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_part_delete.sql new file mode 100644 index 0000000000..32ec2aa18b --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_part_delete.sql @@ -0,0 +1 @@ +delete from `part` where p_partkey > 10000; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_sequence_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_sequence_create.sql new file mode 100644 index 0000000000..ab2b62a52d --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/part_sequence_create.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS `part` ( + `p_partkey` int(11) NOT NULL COMMENT "", + `p_name` varchar(23) NOT NULL COMMENT "", + `p_mfgr` varchar(7) NOT NULL COMMENT "", + `p_category` varchar(8) NOT NULL COMMENT "", + `p_brand` varchar(10) NOT NULL COMMENT "", + `p_color` varchar(12) NOT NULL COMMENT "", + `p_type` varchar(26) NOT NULL COMMENT "", + `p_size` int(11) NOT NULL COMMENT "", + `p_container` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`p_partkey`) +CLUSTER BY (`p_size`) +DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 10 +PROPERTIES ( +"function_column.sequence_type" = 'int', +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_create.sql new file mode 100644 index 0000000000..53b607a53f --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_create.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS `supplier` ( + `s_suppkey` int(11) NOT NULL COMMENT "", + `s_name` varchar(26) NOT NULL COMMENT "", + `s_address` varchar(26) NOT NULL COMMENT "", + `s_city` varchar(11) NOT NULL COMMENT "", + `s_nation` varchar(16) NOT NULL COMMENT "", + `s_region` varchar(13) NOT NULL COMMENT "", + `s_phone` varchar(16) NOT NULL COMMENT "" +) +UNIQUE KEY (`s_suppkey`) +CLUSTER BY (`s_region`, `s_address`) +DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_delete.sql new file mode 100644 index 0000000000..39e663134c --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_delete.sql @@ -0,0 +1 @@ +truncate table `supplier`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_part_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_part_delete.sql new file mode 100644 index 0000000000..ac6a7030fd --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_part_delete.sql @@ -0,0 +1 @@ +delete from `supplier` where s_suppkey > 100; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_sequence_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_sequence_create.sql new file mode 100644 index 0000000000..9fef263bf0 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/ddl/supplier_sequence_create.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS `supplier` ( + `s_suppkey` int(11) NOT NULL COMMENT "", + `s_name` varchar(26) NOT NULL COMMENT "", + `s_address` varchar(26) NOT NULL COMMENT "", + `s_city` varchar(11) NOT NULL COMMENT "", + `s_nation` varchar(16) NOT NULL COMMENT "", + `s_region` varchar(13) NOT NULL COMMENT "", + `s_phone` varchar(16) NOT NULL COMMENT "" +) +UNIQUE KEY (`s_suppkey`) +CLUSTER BY (`s_region`, `s_city`, `s_name`) +DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 10 +PROPERTIES ( +"function_column.sequence_type" = 'int', +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/four/load_four_step.groovy b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/four/load_four_step.groovy new file mode 100644 index 0000000000..49e7904191 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/four/load_four_step.groovy @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Most of the cases are copied from https://github.com/trinodb/trino/tree/master +// /testing/trino-product-tests/src/main/resources/sql-tests/testcases +// and modified by Doris. + +suite("load_four_step") { + def tables = ["customer": ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", 3000, "c_custkey", 1500], + "date": ["""d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", 255, "d_datekey", 224], + "supplier": ["""s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy""", 200, "s_suppkey", 100]] + + tables.each { tableName, rows -> + sql """ DROP TABLE IF EXISTS $tableName """ + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_sequence_create.sql""").text + for (j in 0..<2) { + streamLoad { + table tableName + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', rows[0] + set 'function_column.sequence_col', rows[2] + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + sql 'sync' + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[1]) + } + } + sql """ set delete_without_partition = true; """ + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_part_delete.sql""").text + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[3]) + } + streamLoad { + table tableName + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', rows[0] + set 'function_column.sequence_col', rows[2] + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + sql 'sync' + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[1]) + } + } +} diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/one/load_one_step.groovy b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/one/load_one_step.groovy new file mode 100644 index 0000000000..74ff14bf68 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/one/load_one_step.groovy @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("load_one_step") { + def tables = ["customer": ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", 3000], + "date": ["""d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", 255], + "supplier": ["""s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy""", 200]] + + tables.each { tableName, rows -> + sql """ DROP TABLE IF EXISTS $tableName """ + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_create.sql""").text + streamLoad { + table "${tableName}" + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', "${rows[0]}" + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + sql 'sync' + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[1]) + } + } +} diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/three/load_three_step.groovy b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/three/load_three_step.groovy new file mode 100644 index 0000000000..b33f77d9ef --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/three/load_three_step.groovy @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("load_three_step") { + def tables = ["customer": ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", 3000, "c_custkey"], + "date": ["""d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", 255, "d_datekey"], + "supplier": ["""s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy""", 200, "s_suppkey"]] + + tables.each { tableName, rows -> + sql """ DROP TABLE IF EXISTS $tableName """ + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_sequence_create.sql""").text + for (j in 0..<2) { + streamLoad { + table tableName + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', rows[0] + set 'function_column.sequence_col', rows[2] + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + sql 'sync' + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[1]) + } + } + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_delete.sql""").text + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == 0) + } + sql """ ANALYZE TABLE $tableName WITH SYNC """ + } +} diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/two/load_two_step.groovy b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/two/load_two_step.groovy new file mode 100644 index 0000000000..7b9a3ab0b8 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_load_zstd_c/two/load_two_step.groovy @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("load_two_step") { + def tables = ["customer": ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", 3000, "c_custkey"], + "date": ["""d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", 255, "d_datekey"], + "supplier": ["""s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy""", 200, "s_suppkey"]] + + tables.each { tableName, rows -> + sql """ DROP TABLE IF EXISTS $tableName """ + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_sequence_create.sql""").text + streamLoad { + table tableName + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', rows[0] + set 'function_column.sequence_col', rows[2] + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + sql 'sync' + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[1]) + } + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_delete.sql""").text + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == 0) + } + } +} diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/customer_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/customer_create.sql new file mode 100644 index 0000000000..3640400704 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/customer_create.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS `customer` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "", + `c_nation` varchar(16) NOT NULL COMMENT "", + `c_region` varchar(13) NOT NULL COMMENT "", + `c_phone` varchar(16) NOT NULL COMMENT "", + `c_mktsegment` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`c_custkey`) +CLUSTER BY (`c_region`, `c_city`, `c_name`) +DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/customer_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/customer_delete.sql new file mode 100644 index 0000000000..68a98512b2 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/customer_delete.sql @@ -0,0 +1 @@ +drop table if exists customer; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/date_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/date_create.sql new file mode 100644 index 0000000000..6a06553782 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/date_create.sql @@ -0,0 +1,28 @@ +CREATE TABLE IF NOT EXISTS `date` ( + `d_datekey` int(11) NOT NULL COMMENT "", + `d_date` varchar(20) NOT NULL COMMENT "", + `d_dayofweek` varchar(10) NOT NULL COMMENT "", + `d_month` varchar(11) NOT NULL COMMENT "", + `d_year` int(11) NOT NULL COMMENT "", + `d_yearmonthnum` int(11) NOT NULL COMMENT "", + `d_yearmonth` varchar(9) NOT NULL COMMENT "", + `d_daynuminweek` int(11) NOT NULL COMMENT "", + `d_daynuminmonth` int(11) NOT NULL COMMENT "", + `d_daynuminyear` int(11) NOT NULL COMMENT "", + `d_monthnuminyear` int(11) NOT NULL COMMENT "", + `d_weeknuminyear` int(11) NOT NULL COMMENT "", + `d_sellingseason` varchar(14) NOT NULL COMMENT "", + `d_lastdayinweekfl` int(11) NOT NULL COMMENT "", + `d_lastdayinmonthfl` int(11) NOT NULL COMMENT "", + `d_holidayfl` int(11) NOT NULL COMMENT "", + `d_weekdayfl` int(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`d_datekey`) +CLUSTER BY (`d_year`, `d_month`, `d_weeknuminyear`) +DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/date_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/date_delete.sql new file mode 100644 index 0000000000..c6cf155575 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/date_delete.sql @@ -0,0 +1 @@ +drop table if exists `date`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/lineorder_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/lineorder_create.sql new file mode 100644 index 0000000000..d56c8aee33 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/lineorder_create.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS `lineorder` ( + `lo_orderdate` int(11) NOT NULL COMMENT "", + `lo_orderkey` bigint(20) NOT NULL COMMENT "", + `lo_linenumber` bigint(20) NOT NULL COMMENT "", + `lo_custkey` int(11) NOT NULL COMMENT "", + `lo_partkey` int(11) NOT NULL COMMENT "", + `lo_suppkey` int(11) NOT NULL COMMENT "", + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", + `lo_shippriority` int(11) NOT NULL COMMENT "", + `lo_quantity` bigint(20) NOT NULL COMMENT "", + `lo_extendedprice` bigint(20) NOT NULL COMMENT "", + `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "", + `lo_discount` bigint(20) NOT NULL COMMENT "", + `lo_revenue` bigint(20) NOT NULL COMMENT "", + `lo_supplycost` bigint(20) NOT NULL COMMENT "", + `lo_tax` bigint(20) NOT NULL COMMENT "", + `lo_commitdate` bigint(20) NOT NULL COMMENT "", + `lo_shipmode` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`lo_orderdate`,`lo_orderkey`,`lo_linenumber`) +CLUSTER BY (`lo_orderdate`, `lo_orderpriority`) +PARTITION BY RANGE(`lo_orderdate`) +(PARTITION p1992 VALUES [("-2147483648"), ("19930101")), +PARTITION p1993 VALUES [("19930101"), ("19940101")), +PARTITION p1994 VALUES [("19940101"), ("19950101")), +PARTITION p1995 VALUES [("19950101"), ("19960101")), +PARTITION p1996 VALUES [("19960101"), ("19970101")), +PARTITION p1997 VALUES [("19970101"), ("19980101")), +PARTITION p1998 VALUES [("19980101"), ("19990101"))) +DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 48 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/lineorder_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/lineorder_delete.sql new file mode 100644 index 0000000000..d8f94cfe9f --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/lineorder_delete.sql @@ -0,0 +1 @@ +drop table if exists lineorder; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/part_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/part_create.sql new file mode 100644 index 0000000000..34a1555fa5 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/part_create.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS `part` ( + `p_partkey` int(11) NOT NULL COMMENT "", + `p_name` varchar(23) NOT NULL COMMENT "", + `p_mfgr` varchar(7) NOT NULL COMMENT "", + `p_category` varchar(8) NOT NULL COMMENT "", + `p_brand` varchar(10) NOT NULL COMMENT "", + `p_color` varchar(12) NOT NULL COMMENT "", + `p_type` varchar(26) NOT NULL COMMENT "", + `p_size` int(11) NOT NULL COMMENT "", + `p_container` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`p_partkey`) +CLUSTER BY (`p_category`, `p_brand`, `p_size`) +DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/part_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/part_delete.sql new file mode 100644 index 0000000000..4ad502e24d --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/part_delete.sql @@ -0,0 +1 @@ +drop table if exists `part`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/supplier_create.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/supplier_create.sql new file mode 100644 index 0000000000..662aca9847 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/supplier_create.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS `supplier` ( + `s_suppkey` int(11) NOT NULL COMMENT "", + `s_name` varchar(26) NOT NULL COMMENT "", + `s_address` varchar(26) NOT NULL COMMENT "", + `s_city` varchar(11) NOT NULL COMMENT "", + `s_nation` varchar(16) NOT NULL COMMENT "", + `s_region` varchar(13) NOT NULL COMMENT "", + `s_phone` varchar(16) NOT NULL COMMENT "" +) +UNIQUE KEY (`s_suppkey`) +CLUSTER BY (`s_region`, `s_city`, `s_name`) +DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/supplier_delete.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/supplier_delete.sql new file mode 100644 index 0000000000..72e1c39dae --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/ddl/supplier_delete.sql @@ -0,0 +1 @@ +drop table if exists `supplier`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/load.groovy b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/load.groovy new file mode 100644 index 0000000000..1715ac09c6 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/load.groovy @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Most of the cases are copied from https://github.com/trinodb/trino/tree/master +// /testing/trino-product-tests/src/main/resources/sql-tests/testcases +// and modified by Doris. + +// Note: To filter out tables from sql files, use the following one-liner comamnd +// sed -nr 's/.*tables: (.*)$/\1/gp' /path/to/*.sql | sed -nr 's/,/\n/gp' | sort | uniq +suite("load") { + def tables = ["customer", "lineorder", "part", "date", "supplier"] + def columns = ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", + """lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority, + lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount, + lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy""", + """p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy""", + """d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", + """s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy"""] + + for (String table in tables) { + sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text + sql new File("""${context.file.parent}/ddl/${table}_create.sql""").text + } + def i = 0 + for (String tableName in tables) { + streamLoad { + // a default db 'regression_test' is specified in + // ${DORIS_HOME}/conf/regression-conf.groovy + table tableName + + // default label is UUID: + // set 'label' UUID.randomUUID().toString() + + // default column_separator is specify in doris fe config, usually is '\t'. + // this line change to ',' + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', columns[i] + + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + i++ + } +} diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.1.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.1.sql new file mode 100644 index 0000000000..4ef15e93ea --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.1.sql @@ -0,0 +1,24 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_extendedprice*lo_discount) AS +REVENUE +FROM lineorder, date +WHERE lo_orderdate = d_datekey +AND d_year = 1993 +AND lo_discount BETWEEN 1 AND 3 +AND lo_quantity < 25; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.2.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.2.sql new file mode 100644 index 0000000000..1b8442bd93 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.2.sql @@ -0,0 +1,24 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_extendedprice*lo_discount) AS +REVENUE +FROM lineorder, date +WHERE lo_orderdate = d_datekey +AND d_yearmonth = 'Jan1994' +AND lo_discount BETWEEN 4 AND 6 +AND lo_quantity BETWEEN 26 AND 35; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.3.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.3.sql new file mode 100644 index 0000000000..ed6e51b1cf --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q1.3.sql @@ -0,0 +1,25 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_extendedprice*lo_discount) AS +REVENUE +FROM lineorder, date +WHERE lo_orderdate = d_datekey +AND d_weeknuminyear= 6 +AND d_year = 1994 +AND lo_discount BETWEEN 5 AND 7 +AND lo_quantity BETWEEN 26 AND 35; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.1.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.1.sql new file mode 100644 index 0000000000..e1a1f52d18 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.1.sql @@ -0,0 +1,26 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_revenue), d_year, p_brand +FROM lineorder, date, part, supplier +WHERE lo_orderdate = d_datekey +AND lo_partkey = p_partkey +AND lo_suppkey = s_suppkey +AND p_category = 'MFGR#12' +AND s_region = 'AMERICA' +GROUP BY d_year, p_brand +ORDER BY d_year, p_brand; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.2.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.2.sql new file mode 100644 index 0000000000..3db6170119 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.2.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_revenue), d_year, p_brand +FROM lineorder, date, part, supplier +WHERE lo_orderdate = d_datekey +AND lo_partkey = p_partkey +AND lo_suppkey = s_suppkey +AND p_brand BETWEEN 'MFGR#2221' +AND 'MFGR#2228' +AND s_region = 'ASIA' +GROUP BY d_year, p_brand +ORDER BY d_year, p_brand; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.3.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.3.sql new file mode 100644 index 0000000000..b70ca90666 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q2.3.sql @@ -0,0 +1,26 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_revenue), d_year, p_brand +FROM lineorder, date, part, supplier +WHERE lo_orderdate = d_datekey +AND lo_partkey = p_partkey +AND lo_suppkey = s_suppkey +AND p_brand = 'MFGR#2239' +AND s_region = 'EUROPE' +GROUP BY d_year, p_brand +ORDER BY d_year, p_brand; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.1.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.1.sql new file mode 100644 index 0000000000..70f17d789b --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.1.sql @@ -0,0 +1,28 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT c_nation, s_nation, d_year, +SUM(lo_revenue) AS REVENUE +FROM customer, lineorder, supplier, date +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_orderdate = d_datekey +AND c_region = 'ASIA' +AND s_region = 'ASIA' +AND d_year >= 1992 AND d_year <= 1997 +GROUP BY c_nation, s_nation, d_year +ORDER BY d_year ASC, REVENUE DESC; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.2.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.2.sql new file mode 100644 index 0000000000..a416fbea8b --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.2.sql @@ -0,0 +1,28 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT c_city, s_city, d_year, sum(lo_revenue) +AS REVENUE +FROM customer, lineorder, supplier, date +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_orderdate = d_datekey +AND c_nation = 'UNITED STATES' +AND s_nation = 'UNITED STATES' +AND d_year >= 1992 AND d_year <= 1997 +GROUP BY c_city, s_city, d_year +ORDER BY d_year ASC, REVENUE DESC; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.3.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.3.sql new file mode 100644 index 0000000000..98e29b72e7 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.3.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT c_city, s_city, d_year, SUM(lo_revenue) +AS REVENUE +FROM customer, lineorder, supplier, date +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_orderdate = d_datekey +AND (c_city='UNITED KI1' +OR c_city='UNITED KI5') +AND (s_city='UNITED KI1' +OR s_city='UNITED KI5') +AND d_year >= 1992 AND d_year <= 1997 +GROUP BY c_city, s_city, d_year +ORDER BY d_year ASC, REVENUE DESC; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.4.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.4.sql new file mode 100644 index 0000000000..65fe992ca4 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q3.4.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT c_city, s_city, d_year, SUM(lo_revenue) +AS REVENUE +FROM customer, lineorder, supplier, date +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_orderdate = d_datekey +AND (c_city='UNITED KI1' +OR c_city='UNITED KI5') +AND (s_city='UNITED KI1' +OR s_city='UNITED KI5') +AND d_yearmonth = 'Dec1997' +GROUP BY c_city, s_city, d_year +ORDER BY d_year ASC, REVENUE DESC; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.1.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.1.sql new file mode 100644 index 0000000000..bdcd730bf9 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.1.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT d_year, c_nation, +SUM(lo_revenue - lo_supplycost) AS PROFIT +FROM date, customer, supplier, part, lineorder +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_partkey = p_partkey +AND lo_orderdate = d_datekey +AND c_region = 'AMERICA' +AND s_region = 'AMERICA' +AND (p_mfgr = 'MFGR#1' +OR p_mfgr = 'MFGR#2') +GROUP BY d_year, c_nation +ORDER BY d_year, c_nation; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.2.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.2.sql new file mode 100644 index 0000000000..24c82cf682 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.2.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT d_year, s_nation, p_category, +SUM(lo_revenue - lo_supplycost) AS PROFIT +FROM date, customer, supplier, part, lineorder +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_partkey = p_partkey +AND lo_orderdate = d_datekey +AND c_region = 'AMERICA' +AND s_region = 'AMERICA' +AND (d_year = 1997 OR d_year = 1998) +AND (p_mfgr = 'MFGR#1' +OR p_mfgr = 'MFGR#2') +GROUP BY d_year, s_nation, p_category +ORDER BY d_year, s_nation, p_category; diff --git a/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.3.sql b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.3.sql new file mode 100644 index 0000000000..0dcc08bd26 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/ssb_unique_sql_zstd_cluster/sql/q4.3.sql @@ -0,0 +1,29 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT d_year, s_city, p_brand, +SUM(lo_revenue - lo_supplycost) AS PROFIT +FROM date, customer, supplier, part, lineorder +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_partkey = p_partkey +AND lo_orderdate = d_datekey +AND s_nation = 'UNITED STATES' +AND (d_year = 1997 OR d_year = 1998) +AND p_category = 'MFGR#14' +GROUP BY d_year, s_city, p_brand +ORDER BY d_year, s_city, p_brand; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_create.sql new file mode 100644 index 0000000000..0bf16f3911 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_create.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS `customer` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "", + `c_nation` varchar(16) NOT NULL COMMENT "", + `c_region` varchar(13) NOT NULL COMMENT "", + `c_phone` varchar(16) NOT NULL COMMENT "", + `c_mktsegment` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`c_custkey`) +CLUSTER BY (`c_region`, `c_phone`, `c_city`) +DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_delete.sql new file mode 100644 index 0000000000..fe22a226fe --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_delete.sql @@ -0,0 +1 @@ +truncate table customer; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_part_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_part_delete.sql new file mode 100644 index 0000000000..a9d1b34d68 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_part_delete.sql @@ -0,0 +1 @@ +delete from customer where c_custkey > 1500 ; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_sequence_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_sequence_create.sql new file mode 100644 index 0000000000..892384684b --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/customer_sequence_create.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS `customer` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "", + `c_nation` varchar(16) NOT NULL COMMENT "", + `c_region` varchar(13) NOT NULL COMMENT "", + `c_phone` varchar(16) NOT NULL COMMENT "", + `c_mktsegment` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`c_custkey`) +CLUSTER BY (`c_mktsegment`, `c_city`, `c_region`, `c_nation`) +DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10 +PROPERTIES ( +"function_column.sequence_type" = 'int', +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_create.sql new file mode 100644 index 0000000000..32b4e24f6c --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_create.sql @@ -0,0 +1,28 @@ +CREATE TABLE IF NOT EXISTS `date` ( + `d_datekey` int(11) NOT NULL COMMENT "", + `d_date` varchar(20) NOT NULL COMMENT "", + `d_dayofweek` varchar(10) NOT NULL COMMENT "", + `d_month` varchar(11) NOT NULL COMMENT "", + `d_year` int(11) NOT NULL COMMENT "", + `d_yearmonthnum` int(11) NOT NULL COMMENT "", + `d_yearmonth` varchar(9) NOT NULL COMMENT "", + `d_daynuminweek` int(11) NOT NULL COMMENT "", + `d_daynuminmonth` int(11) NOT NULL COMMENT "", + `d_daynuminyear` int(11) NOT NULL COMMENT "", + `d_monthnuminyear` int(11) NOT NULL COMMENT "", + `d_weeknuminyear` int(11) NOT NULL COMMENT "", + `d_sellingseason` varchar(14) NOT NULL COMMENT "", + `d_lastdayinweekfl` int(11) NOT NULL COMMENT "", + `d_lastdayinmonthfl` int(11) NOT NULL COMMENT "", + `d_holidayfl` int(11) NOT NULL COMMENT "", + `d_weekdayfl` int(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`d_datekey`) +CLUSTER BY (`d_sellingseason`, `d_holidayfl`) +DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_delete.sql new file mode 100644 index 0000000000..12933cbbad --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_delete.sql @@ -0,0 +1 @@ +truncate table `date`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_part_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_part_delete.sql new file mode 100644 index 0000000000..0c21b27cc4 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_part_delete.sql @@ -0,0 +1 @@ +delete from `date` where d_datekey >= '19920701' and d_datekey <= '19920731'; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_sequence_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_sequence_create.sql new file mode 100644 index 0000000000..9ec46190c7 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/date_sequence_create.sql @@ -0,0 +1,29 @@ +CREATE TABLE IF NOT EXISTS `date` ( + `d_datekey` int(11) NOT NULL COMMENT "", + `d_date` varchar(20) NOT NULL COMMENT "", + `d_dayofweek` varchar(10) NOT NULL COMMENT "", + `d_month` varchar(11) NOT NULL COMMENT "", + `d_year` int(11) NOT NULL COMMENT "", + `d_yearmonthnum` int(11) NOT NULL COMMENT "", + `d_yearmonth` varchar(9) NOT NULL COMMENT "", + `d_daynuminweek` int(11) NOT NULL COMMENT "", + `d_daynuminmonth` int(11) NOT NULL COMMENT "", + `d_daynuminyear` int(11) NOT NULL COMMENT "", + `d_monthnuminyear` int(11) NOT NULL COMMENT "", + `d_weeknuminyear` int(11) NOT NULL COMMENT "", + `d_sellingseason` varchar(14) NOT NULL COMMENT "", + `d_lastdayinweekfl` int(11) NOT NULL COMMENT "", + `d_lastdayinmonthfl` int(11) NOT NULL COMMENT "", + `d_holidayfl` int(11) NOT NULL COMMENT "", + `d_weekdayfl` int(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`d_datekey`) +CLUSTER BY (`d_sellingseason`, `d_lastdayinweekfl`) +DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1 +PROPERTIES ( +"function_column.sequence_type" = 'int', +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_create.sql new file mode 100644 index 0000000000..8cb2ae7309 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_create.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS `lineorder` ( + `lo_orderdate` int(11) NOT NULL COMMENT "", + `lo_orderkey` bigint(20) NOT NULL COMMENT "", + `lo_linenumber` bigint(20) NOT NULL COMMENT "", + `lo_custkey` int(11) NOT NULL COMMENT "", + `lo_partkey` int(11) NOT NULL COMMENT "", + `lo_suppkey` int(11) NOT NULL COMMENT "", + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", + `lo_shippriority` int(11) NOT NULL COMMENT "", + `lo_quantity` bigint(20) NOT NULL COMMENT "", + `lo_extendedprice` bigint(20) NOT NULL COMMENT "", + `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "", + `lo_discount` bigint(20) NOT NULL COMMENT "", + `lo_revenue` bigint(20) NOT NULL COMMENT "", + `lo_supplycost` bigint(20) NOT NULL COMMENT "", + `lo_tax` bigint(20) NOT NULL COMMENT "", + `lo_commitdate` bigint(20) NOT NULL COMMENT "", + `lo_shipmode` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`lo_orderdate`,`lo_orderkey`,`lo_linenumber`) +CLUSTER BY (`lo_orderdate`, `lo_orderkey`, `lo_shipmode`) +PARTITION BY RANGE(`lo_orderdate`) +(PARTITION p1992 VALUES [("-2147483648"), ("19930101")), +PARTITION p1993 VALUES [("19930101"), ("19940101")), +PARTITION p1994 VALUES [("19940101"), ("19950101")), +PARTITION p1995 VALUES [("19950101"), ("19960101")), +PARTITION p1996 VALUES [("19960101"), ("19970101")), +PARTITION p1997 VALUES [("19970101"), ("19980101")), +PARTITION p1998 VALUES [("19980101"), ("19990101"))) +DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 48 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_delete.sql new file mode 100644 index 0000000000..329e040060 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_delete.sql @@ -0,0 +1 @@ +truncate table lineorder; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_part_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_part_delete.sql new file mode 100644 index 0000000000..abb7ded433 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_part_delete.sql @@ -0,0 +1 @@ +delete from lineorder where lo_orderkey >= 240001 and lo_orderkey <= 360000; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_sequence_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_sequence_create.sql new file mode 100644 index 0000000000..a9b1d41156 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/lineorder_sequence_create.sql @@ -0,0 +1,37 @@ +CREATE TABLE IF NOT EXISTS `lineorder` ( + `lo_orderdate` int(11) NOT NULL COMMENT "", + `lo_orderkey` bigint(20) NOT NULL COMMENT "", + `lo_linenumber` bigint(20) NOT NULL COMMENT "", + `lo_custkey` int(11) NOT NULL COMMENT "", + `lo_partkey` int(11) NOT NULL COMMENT "", + `lo_suppkey` int(11) NOT NULL COMMENT "", + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", + `lo_shippriority` int(11) NOT NULL COMMENT "", + `lo_quantity` bigint(20) NOT NULL COMMENT "", + `lo_extendedprice` bigint(20) NOT NULL COMMENT "", + `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "", + `lo_discount` bigint(20) NOT NULL COMMENT "", + `lo_revenue` bigint(20) NOT NULL COMMENT "", + `lo_supplycost` bigint(20) NOT NULL COMMENT "", + `lo_tax` bigint(20) NOT NULL COMMENT "", + `lo_commitdate` bigint(20) NOT NULL COMMENT "", + `lo_shipmode` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`lo_orderdate`,`lo_orderkey`,`lo_linenumber`) +CLUSTER BY (`lo_supplycost`, `lo_orderdate`, `lo_orderkey`) +PARTITION BY RANGE(`lo_orderdate`) +(PARTITION p1992 VALUES [("-2147483648"), ("19930101")), +PARTITION p1993 VALUES [("19930101"), ("19940101")), +PARTITION p1994 VALUES [("19940101"), ("19950101")), +PARTITION p1995 VALUES [("19950101"), ("19960101")), +PARTITION p1996 VALUES [("19960101"), ("19970101")), +PARTITION p1997 VALUES [("19970101"), ("19980101")), +PARTITION p1998 VALUES [("19980101"), ("19990101"))) +DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 48 +PROPERTIES ( +"function_column.sequence_type" = 'int', +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_create.sql new file mode 100644 index 0000000000..722b7eba1a --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_create.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS `part` ( + `p_partkey` int(11) NOT NULL COMMENT "", + `p_name` varchar(23) NOT NULL COMMENT "", + `p_mfgr` varchar(7) NOT NULL COMMENT "", + `p_category` varchar(8) NOT NULL COMMENT "", + `p_brand` varchar(10) NOT NULL COMMENT "", + `p_color` varchar(12) NOT NULL COMMENT "", + `p_type` varchar(26) NOT NULL COMMENT "", + `p_size` int(11) NOT NULL COMMENT "", + `p_container` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`p_partkey`) +CLUSTER BY (`p_color`, `p_name`, `p_category`) +DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_delete.sql new file mode 100644 index 0000000000..02c6abd253 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_delete.sql @@ -0,0 +1 @@ +truncate table `part`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_part_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_part_delete.sql new file mode 100644 index 0000000000..32ec2aa18b --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_part_delete.sql @@ -0,0 +1 @@ +delete from `part` where p_partkey > 10000; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_sequence_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_sequence_create.sql new file mode 100644 index 0000000000..5ba4038e12 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/part_sequence_create.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS `part` ( + `p_partkey` int(11) NOT NULL COMMENT "", + `p_name` varchar(23) NOT NULL COMMENT "", + `p_mfgr` varchar(7) NOT NULL COMMENT "", + `p_category` varchar(8) NOT NULL COMMENT "", + `p_brand` varchar(10) NOT NULL COMMENT "", + `p_color` varchar(12) NOT NULL COMMENT "", + `p_type` varchar(26) NOT NULL COMMENT "", + `p_size` int(11) NOT NULL COMMENT "", + `p_container` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`p_partkey`) +CLUSTER BY (`p_size`, `p_type`, `p_partkey`) +DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 10 +PROPERTIES ( +"function_column.sequence_type" = 'int', +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_create.sql new file mode 100644 index 0000000000..aa798357e8 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_create.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS `supplier` ( + `s_suppkey` int(11) NOT NULL COMMENT "", + `s_name` varchar(26) NOT NULL COMMENT "", + `s_address` varchar(26) NOT NULL COMMENT "", + `s_city` varchar(11) NOT NULL COMMENT "", + `s_nation` varchar(16) NOT NULL COMMENT "", + `s_region` varchar(13) NOT NULL COMMENT "", + `s_phone` varchar(16) NOT NULL COMMENT "" +) +UNIQUE KEY (`s_suppkey`) +CLUSTER BY (`s_address`) +DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_delete.sql new file mode 100644 index 0000000000..39e663134c --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_delete.sql @@ -0,0 +1 @@ +truncate table `supplier`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_part_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_part_delete.sql new file mode 100644 index 0000000000..ac6a7030fd --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_part_delete.sql @@ -0,0 +1 @@ +delete from `supplier` where s_suppkey > 100; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_sequence_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_sequence_create.sql new file mode 100644 index 0000000000..fd109360fd --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/ddl/supplier_sequence_create.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS `supplier` ( + `s_suppkey` int(11) NOT NULL COMMENT "", + `s_name` varchar(26) NOT NULL COMMENT "", + `s_address` varchar(26) NOT NULL COMMENT "", + `s_city` varchar(11) NOT NULL COMMENT "", + `s_nation` varchar(16) NOT NULL COMMENT "", + `s_region` varchar(13) NOT NULL COMMENT "", + `s_phone` varchar(16) NOT NULL COMMENT "" +) +UNIQUE KEY (`s_suppkey`) +CLUSTER BY (`s_nation`, `s_region`, `s_city`, `s_name`) +DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 10 +PROPERTIES ( +"function_column.sequence_type" = 'int', +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/four/load_four_step.groovy b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/four/load_four_step.groovy new file mode 100644 index 0000000000..81702248a8 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/four/load_four_step.groovy @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Most of the cases are copied from https://github.com/trinodb/trino/tree/master +// /testing/trino-product-tests/src/main/resources/sql-tests/testcases +// and modified by Doris. + +suite("load_four_step") { + def tables = ["customer": ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", 3000, "c_custkey", 1500], + "lineorder": ["""lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority, + lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount, + lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy""", 600572, "lo_orderkey", 481137], + "part": ["""p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy""", 20000, "p_partkey", 10000], + "date": ["""d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", 255, "d_datekey", 224], + "supplier": ["""s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy""", 200, "s_suppkey", 100]] + + tables.each { tableName, rows -> + sql """ DROP TABLE IF EXISTS $tableName """ + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_sequence_create.sql""").text + for (j in 0..<2) { + streamLoad { + table tableName + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', rows[0] + set 'function_column.sequence_col', rows[2] + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + sql 'sync' + int flag = 1 + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[1]) + } + } + sql """ set delete_without_partition = true; """ + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_part_delete.sql""").text + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[3]) + } + streamLoad { + table tableName + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', rows[0] + set 'function_column.sequence_col', rows[2] + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + sql 'sync' + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[1]) + } + } +} diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/one/load_one_step.groovy b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/one/load_one_step.groovy new file mode 100644 index 0000000000..31a8344d73 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/one/load_one_step.groovy @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("load_one_step") { + def tables = ["customer": ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", 3000], + "lineorder": ["""lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority, + lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount, + lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy""", 600572], + "part": ["""p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy""", 20000], + "date": ["""d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", 255], + "supplier": ["""s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy""", 200]] + + tables.each { tableName, rows -> + sql """ DROP TABLE IF EXISTS $tableName """ + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_create.sql""").text + streamLoad { + table "${tableName}" + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', "${rows[0]}" + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + sql 'sync' + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[1]) + } + } +} diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/three/load_three_step.groovy b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/three/load_three_step.groovy new file mode 100644 index 0000000000..aae1b16426 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/three/load_three_step.groovy @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("load_three_step") { + def tables = ["customer": ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", 3000, "c_custkey"], + "lineorder": ["""lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority, + lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount, + lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy""", 600572, "lo_orderkey"], + "part": ["""p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy""", 20000, "p_partkey"], + "date": ["""d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", 255, "d_datekey"], + "supplier": ["""s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy""", 200, "s_suppkey"]] + + tables.each { tableName, rows -> + sql """ DROP TABLE IF EXISTS $tableName """ + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_sequence_create.sql""").text + for (j in 0..<2) { + streamLoad { + table tableName + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', rows[0] + set 'function_column.sequence_col', rows[2] + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + sql 'sync' + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[1]) + } + } + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_delete.sql""").text + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == 0) + } + } +} diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/two/load_two_step.groovy b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/two/load_two_step.groovy new file mode 100644 index 0000000000..f309aaeba3 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_load_zstd_c/two/load_two_step.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("load_two_step") { + def tables = ["customer": ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", 3000, "c_custkey"], + "lineorder": ["""lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority, + lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount, + lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy""", 600572, "lo_orderkey"], + "part": ["""p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy""", 20000, "p_partkey"], + "date": ["""d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", 255, "d_datekey"], + "supplier": ["""s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy""", 200, "s_suppkey"]] + + tables.each { tableName, rows -> + sql """ DROP TABLE IF EXISTS $tableName """ + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_sequence_create.sql""").text + streamLoad { + table tableName + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', rows[0] + set 'function_column.sequence_col', rows[2] + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + sql 'sync' + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == rows[1]) + } + sql new File("""${context.file.parentFile.parent}/ddl/${tableName}_delete.sql""").text + for (int i = 1; i <= 5; i++) { + def loadRowCount = sql "select count(1) from ${tableName}" + logger.info("select ${tableName} numbers: ${loadRowCount[0][0]}".toString()) + assertTrue(loadRowCount[0][0] == 0) + } + } +} diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/customer_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/customer_create.sql new file mode 100644 index 0000000000..8240bd709c --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/customer_create.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS `customer` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "", + `c_nation` varchar(16) NOT NULL COMMENT "", + `c_region` varchar(13) NOT NULL COMMENT "", + `c_phone` varchar(16) NOT NULL COMMENT "", + `c_mktsegment` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`c_custkey`) +CLUSTER BY (`c_region`, `c_address`, `c_city`, `c_name`) +DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/customer_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/customer_delete.sql new file mode 100644 index 0000000000..68a98512b2 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/customer_delete.sql @@ -0,0 +1 @@ +drop table if exists customer; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/date_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/date_create.sql new file mode 100644 index 0000000000..1ff610fd69 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/date_create.sql @@ -0,0 +1,28 @@ +CREATE TABLE IF NOT EXISTS `date` ( + `d_datekey` int(11) NOT NULL COMMENT "", + `d_date` varchar(20) NOT NULL COMMENT "", + `d_dayofweek` varchar(10) NOT NULL COMMENT "", + `d_month` varchar(11) NOT NULL COMMENT "", + `d_year` int(11) NOT NULL COMMENT "", + `d_yearmonthnum` int(11) NOT NULL COMMENT "", + `d_yearmonth` varchar(9) NOT NULL COMMENT "", + `d_daynuminweek` int(11) NOT NULL COMMENT "", + `d_daynuminmonth` int(11) NOT NULL COMMENT "", + `d_daynuminyear` int(11) NOT NULL COMMENT "", + `d_monthnuminyear` int(11) NOT NULL COMMENT "", + `d_weeknuminyear` int(11) NOT NULL COMMENT "", + `d_sellingseason` varchar(14) NOT NULL COMMENT "", + `d_lastdayinweekfl` int(11) NOT NULL COMMENT "", + `d_lastdayinmonthfl` int(11) NOT NULL COMMENT "", + `d_holidayfl` int(11) NOT NULL COMMENT "", + `d_weekdayfl` int(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`d_datekey`) +CLUSTER BY (`d_weeknuminyear`, `d_month`) +DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/date_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/date_delete.sql new file mode 100644 index 0000000000..c6cf155575 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/date_delete.sql @@ -0,0 +1 @@ +drop table if exists `date`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/lineorder_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/lineorder_create.sql new file mode 100644 index 0000000000..829b8d65bd --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/lineorder_create.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS `lineorder` ( + `lo_orderdate` int(11) NOT NULL COMMENT "", + `lo_orderkey` bigint(20) NOT NULL COMMENT "", + `lo_linenumber` bigint(20) NOT NULL COMMENT "", + `lo_custkey` int(11) NOT NULL COMMENT "", + `lo_partkey` int(11) NOT NULL COMMENT "", + `lo_suppkey` int(11) NOT NULL COMMENT "", + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", + `lo_shippriority` int(11) NOT NULL COMMENT "", + `lo_quantity` bigint(20) NOT NULL COMMENT "", + `lo_extendedprice` bigint(20) NOT NULL COMMENT "", + `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "", + `lo_discount` bigint(20) NOT NULL COMMENT "", + `lo_revenue` bigint(20) NOT NULL COMMENT "", + `lo_supplycost` bigint(20) NOT NULL COMMENT "", + `lo_tax` bigint(20) NOT NULL COMMENT "", + `lo_commitdate` bigint(20) NOT NULL COMMENT "", + `lo_shipmode` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`lo_orderdate`,`lo_orderkey`,`lo_linenumber`) +CLUSTER BY (`lo_revenue`, `lo_orderdate`, `lo_orderpriority`) +PARTITION BY RANGE(`lo_orderdate`) +(PARTITION p1992 VALUES [("-2147483648"), ("19930101")), +PARTITION p1993 VALUES [("19930101"), ("19940101")), +PARTITION p1994 VALUES [("19940101"), ("19950101")), +PARTITION p1995 VALUES [("19950101"), ("19960101")), +PARTITION p1996 VALUES [("19960101"), ("19970101")), +PARTITION p1997 VALUES [("19970101"), ("19980101")), +PARTITION p1998 VALUES [("19980101"), ("19990101"))) +DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 48 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/lineorder_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/lineorder_delete.sql new file mode 100644 index 0000000000..d8f94cfe9f --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/lineorder_delete.sql @@ -0,0 +1 @@ +drop table if exists lineorder; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/part_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/part_create.sql new file mode 100644 index 0000000000..9dda02c7b7 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/part_create.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS `part` ( + `p_partkey` int(11) NOT NULL COMMENT "", + `p_name` varchar(23) NOT NULL COMMENT "", + `p_mfgr` varchar(7) NOT NULL COMMENT "", + `p_category` varchar(8) NOT NULL COMMENT "", + `p_brand` varchar(10) NOT NULL COMMENT "", + `p_color` varchar(12) NOT NULL COMMENT "", + `p_type` varchar(26) NOT NULL COMMENT "", + `p_size` int(11) NOT NULL COMMENT "", + `p_container` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`p_partkey`) +CLUSTER BY (`p_color`, `p_container`) +DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/part_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/part_delete.sql new file mode 100644 index 0000000000..4ad502e24d --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/part_delete.sql @@ -0,0 +1 @@ +drop table if exists `part`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/supplier_create.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/supplier_create.sql new file mode 100644 index 0000000000..b827e9b6db --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/supplier_create.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS `supplier` ( + `s_suppkey` int(11) NOT NULL COMMENT "", + `s_name` varchar(26) NOT NULL COMMENT "", + `s_address` varchar(26) NOT NULL COMMENT "", + `s_city` varchar(11) NOT NULL COMMENT "", + `s_nation` varchar(16) NOT NULL COMMENT "", + `s_region` varchar(13) NOT NULL COMMENT "", + `s_phone` varchar(16) NOT NULL COMMENT "" +) +UNIQUE KEY (`s_suppkey`) +CLUSTER BY (`s_address`, `s_name`) +DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 10 +PROPERTIES ( +"compression"="zstd", +"replication_num" = "1", +"disable_auto_compaction" = "true", +"enable_unique_key_merge_on_write" = "true" +); diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/supplier_delete.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/supplier_delete.sql new file mode 100644 index 0000000000..72e1c39dae --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/ddl/supplier_delete.sql @@ -0,0 +1 @@ +drop table if exists `supplier`; \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/load.groovy b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/load.groovy new file mode 100644 index 0000000000..f348280242 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/load.groovy @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Most of the cases are copied from https://github.com/trinodb/trino/tree/master +// /testing/trino-product-tests/src/main/resources/sql-tests/testcases +// and modified by Doris. + +// Note: To filter out tables from sql files, use the following one-liner comamnd +// sed -nr 's/.*tables: (.*)$/\1/gp' /path/to/*.sql | sed -nr 's/,/\n/gp' | sort | uniq +suite("load") { + def tables = ["customer", "lineorder", "part", "date", "supplier"] + def columns = ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", + """lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority, + lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount, + lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy""", + """p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy""", + """d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", + """s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy"""] + + for (String table in tables) { + sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text + sql new File("""${context.file.parent}/ddl/${table}_create.sql""").text + } + for (int j = 0; j < 10; j++) { + def i = 0 + for (String tableName in tables) { + streamLoad { + // a default db 'regression_test' is specified in + // ${DORIS_HOME}/conf/regression-conf.groovy + table tableName + + // default label is UUID: + // set 'label' UUID.randomUUID().toString() + + // default column_separator is specify in doris fe config, usually is '\t'. + // this line change to ',' + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', columns[i] + + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf0.1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + i++ + } + } +} diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.1.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.1.sql new file mode 100644 index 0000000000..4ef15e93ea --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.1.sql @@ -0,0 +1,24 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_extendedprice*lo_discount) AS +REVENUE +FROM lineorder, date +WHERE lo_orderdate = d_datekey +AND d_year = 1993 +AND lo_discount BETWEEN 1 AND 3 +AND lo_quantity < 25; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.2.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.2.sql new file mode 100644 index 0000000000..1b8442bd93 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.2.sql @@ -0,0 +1,24 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_extendedprice*lo_discount) AS +REVENUE +FROM lineorder, date +WHERE lo_orderdate = d_datekey +AND d_yearmonth = 'Jan1994' +AND lo_discount BETWEEN 4 AND 6 +AND lo_quantity BETWEEN 26 AND 35; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.3.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.3.sql new file mode 100644 index 0000000000..ed6e51b1cf --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q1.3.sql @@ -0,0 +1,25 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_extendedprice*lo_discount) AS +REVENUE +FROM lineorder, date +WHERE lo_orderdate = d_datekey +AND d_weeknuminyear= 6 +AND d_year = 1994 +AND lo_discount BETWEEN 5 AND 7 +AND lo_quantity BETWEEN 26 AND 35; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.1.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.1.sql new file mode 100644 index 0000000000..e1a1f52d18 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.1.sql @@ -0,0 +1,26 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_revenue), d_year, p_brand +FROM lineorder, date, part, supplier +WHERE lo_orderdate = d_datekey +AND lo_partkey = p_partkey +AND lo_suppkey = s_suppkey +AND p_category = 'MFGR#12' +AND s_region = 'AMERICA' +GROUP BY d_year, p_brand +ORDER BY d_year, p_brand; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.2.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.2.sql new file mode 100644 index 0000000000..3db6170119 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.2.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_revenue), d_year, p_brand +FROM lineorder, date, part, supplier +WHERE lo_orderdate = d_datekey +AND lo_partkey = p_partkey +AND lo_suppkey = s_suppkey +AND p_brand BETWEEN 'MFGR#2221' +AND 'MFGR#2228' +AND s_region = 'ASIA' +GROUP BY d_year, p_brand +ORDER BY d_year, p_brand; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.3.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.3.sql new file mode 100644 index 0000000000..b70ca90666 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q2.3.sql @@ -0,0 +1,26 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT SUM(lo_revenue), d_year, p_brand +FROM lineorder, date, part, supplier +WHERE lo_orderdate = d_datekey +AND lo_partkey = p_partkey +AND lo_suppkey = s_suppkey +AND p_brand = 'MFGR#2239' +AND s_region = 'EUROPE' +GROUP BY d_year, p_brand +ORDER BY d_year, p_brand; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.1.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.1.sql new file mode 100644 index 0000000000..70f17d789b --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.1.sql @@ -0,0 +1,28 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT c_nation, s_nation, d_year, +SUM(lo_revenue) AS REVENUE +FROM customer, lineorder, supplier, date +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_orderdate = d_datekey +AND c_region = 'ASIA' +AND s_region = 'ASIA' +AND d_year >= 1992 AND d_year <= 1997 +GROUP BY c_nation, s_nation, d_year +ORDER BY d_year ASC, REVENUE DESC; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.2.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.2.sql new file mode 100644 index 0000000000..a416fbea8b --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.2.sql @@ -0,0 +1,28 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT c_city, s_city, d_year, sum(lo_revenue) +AS REVENUE +FROM customer, lineorder, supplier, date +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_orderdate = d_datekey +AND c_nation = 'UNITED STATES' +AND s_nation = 'UNITED STATES' +AND d_year >= 1992 AND d_year <= 1997 +GROUP BY c_city, s_city, d_year +ORDER BY d_year ASC, REVENUE DESC; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.3.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.3.sql new file mode 100644 index 0000000000..98e29b72e7 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.3.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT c_city, s_city, d_year, SUM(lo_revenue) +AS REVENUE +FROM customer, lineorder, supplier, date +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_orderdate = d_datekey +AND (c_city='UNITED KI1' +OR c_city='UNITED KI5') +AND (s_city='UNITED KI1' +OR s_city='UNITED KI5') +AND d_year >= 1992 AND d_year <= 1997 +GROUP BY c_city, s_city, d_year +ORDER BY d_year ASC, REVENUE DESC; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.4.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.4.sql new file mode 100644 index 0000000000..65fe992ca4 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q3.4.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT c_city, s_city, d_year, SUM(lo_revenue) +AS REVENUE +FROM customer, lineorder, supplier, date +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_orderdate = d_datekey +AND (c_city='UNITED KI1' +OR c_city='UNITED KI5') +AND (s_city='UNITED KI1' +OR s_city='UNITED KI5') +AND d_yearmonth = 'Dec1997' +GROUP BY c_city, s_city, d_year +ORDER BY d_year ASC, REVENUE DESC; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.1.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.1.sql new file mode 100644 index 0000000000..bdcd730bf9 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.1.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT d_year, c_nation, +SUM(lo_revenue - lo_supplycost) AS PROFIT +FROM date, customer, supplier, part, lineorder +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_partkey = p_partkey +AND lo_orderdate = d_datekey +AND c_region = 'AMERICA' +AND s_region = 'AMERICA' +AND (p_mfgr = 'MFGR#1' +OR p_mfgr = 'MFGR#2') +GROUP BY d_year, c_nation +ORDER BY d_year, c_nation; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.2.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.2.sql new file mode 100644 index 0000000000..24c82cf682 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.2.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT d_year, s_nation, p_category, +SUM(lo_revenue - lo_supplycost) AS PROFIT +FROM date, customer, supplier, part, lineorder +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_partkey = p_partkey +AND lo_orderdate = d_datekey +AND c_region = 'AMERICA' +AND s_region = 'AMERICA' +AND (d_year = 1997 OR d_year = 1998) +AND (p_mfgr = 'MFGR#1' +OR p_mfgr = 'MFGR#2') +GROUP BY d_year, s_nation, p_category +ORDER BY d_year, s_nation, p_category; diff --git a/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.3.sql b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.3.sql new file mode 100644 index 0000000000..0dcc08bd26 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/ssb_unique_sql_zstd_cluster/sql/q4.3.sql @@ -0,0 +1,29 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +SELECT d_year, s_city, p_brand, +SUM(lo_revenue - lo_supplycost) AS PROFIT +FROM date, customer, supplier, part, lineorder +WHERE lo_custkey = c_custkey +AND lo_suppkey = s_suppkey +AND lo_partkey = p_partkey +AND lo_orderdate = d_datekey +AND s_nation = 'UNITED STATES' +AND (d_year = 1997 OR d_year = 1998) +AND p_category = 'MFGR#14' +GROUP BY d_year, s_city, p_brand +ORDER BY d_year, s_city, p_brand; diff --git a/regression-test/suites/unique_with_mow_p2/test_pk_uk_case_cluster.groovy b/regression-test/suites/unique_with_mow_p2/test_pk_uk_case_cluster.groovy new file mode 100644 index 0000000000..ec12413276 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p2/test_pk_uk_case_cluster.groovy @@ -0,0 +1,258 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods; +import java.util.Random; +import org.apache.commons.lang.RandomStringUtils; +import java.util.Date; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.Map; +import java.util.UUID; +import java.time.format.DateTimeFormatter; + +suite("test_pk_uk_case_cluster") { + def tableNamePk = "primary_key_pk_uk_cluster" + def tableNameUk = "unique_key_pk_uk_cluster" + + onFinish { + try_sql("DROP TABLE IF EXISTS ${tableNamePk}") + try_sql("DROP TABLE IF EXISTS ${tableNameUk}") + } + + sql """ DROP TABLE IF EXISTS ${tableNamePk} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableNamePk} ( + L_ORDERKEY INTEGER NOT NULL, + L_PARTKEY INTEGER NOT NULL, + L_SUPPKEY INTEGER NOT NULL, + L_LINENUMBER INTEGER NOT NULL, + L_QUANTITY DECIMAL(15,2) NOT NULL, + L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL, + L_DISCOUNT DECIMAL(15,2) NOT NULL, + L_TAX DECIMAL(15,2) NOT NULL, + L_RETURNFLAG CHAR(1) NOT NULL, + L_LINESTATUS CHAR(1) NOT NULL, + L_SHIPDATE DATE NOT NULL, + L_COMMITDATE DATE NOT NULL, + L_RECEIPTDATE DATE NOT NULL, + L_SHIPINSTRUCT CHAR(60) NOT NULL, + L_SHIPMODE CHAR(60) NOT NULL, + L_COMMENT VARCHAR(60) NOT NULL + ) + UNIQUE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER) + CLUSTER BY (L_PARTKEY, L_SUPPKEY, L_SHIPDATE) + DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + + sql """ DROP TABLE IF EXISTS ${tableNameUk} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableNameUk} ( + L_ORDERKEY INTEGER NOT NULL, + L_PARTKEY INTEGER NOT NULL, + L_SUPPKEY INTEGER NOT NULL, + L_LINENUMBER INTEGER NOT NULL, + L_QUANTITY DECIMAL(15,2) NOT NULL, + L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL, + L_DISCOUNT DECIMAL(15,2) NOT NULL, + L_TAX DECIMAL(15,2) NOT NULL, + L_RETURNFLAG CHAR(1) NOT NULL, + L_LINESTATUS CHAR(1) NOT NULL, + L_SHIPDATE DATE NOT NULL, + L_COMMITDATE DATE NOT NULL, + L_RECEIPTDATE DATE NOT NULL, + L_SHIPINSTRUCT CHAR(60) NOT NULL, + L_SHIPMODE CHAR(60) NOT NULL, + L_COMMENT VARCHAR(60) NOT NULL + ) + UNIQUE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER) + DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false" + ) + """ + + Random rd = new Random() + def order_key = rd.nextInt(1000) + def part_key = rd.nextInt(1000) + def sub_key = 13 + def line_num = 29 + def decimal = rd.nextInt(1000) + 0.11 + def city = RandomStringUtils.randomAlphabetic(10) + def name = UUID.randomUUID().toString() + def date = DateTimeFormatter.ofPattern("yyyy-MM-dd").format(LocalDateTime.now()) + for (int idx = 0; idx < 500; idx++) { + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, + $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + // insert batch key + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + city = RandomStringUtils.randomAlphabetic(10) + name = UUID.randomUUID().toString() + sql """ INSERT INTO ${tableNamePk} VALUES + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + sql """ INSERT INTO ${tableNameUk} VALUES + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city'), + ($order_key, $part_key, $sub_key, $line_num, $decimal, $decimal, $decimal, $decimal, '1', '1', '$date', '$date', '$date', '$name', '$name', '$city') + """ + + sql "sync" + + // count(*) + def result0 = sql """ SELECT count(*) FROM ${tableNamePk}; """ + def result1 = sql """ SELECT count(*) FROM ${tableNameUk}; """ + logger.info("result:" + result0[0][0] + "|" + result1[0][0]) + assertTrue(result0[0]==result1[0]) + if (result0[0][0]!=result1[0][0]) { + logger.info("result:" + result0[0][0] + "|" + result1[0][0]) + } + + result0 = sql """ SELECT + l_returnflag, + l_linestatus, + sum(l_quantity) AS sum_qty, + sum(l_extendedprice) AS sum_base_price, + sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + avg(l_quantity) AS avg_qty, + avg(l_extendedprice) AS avg_price, + avg(l_discount) AS avg_disc, + count(*) AS count_order + FROM + ${tableNamePk} + GROUP BY + l_returnflag, + l_linestatus + ORDER BY + l_returnflag, + l_linestatus + """ + result1 = sql """ SELECT + l_returnflag, + l_linestatus, + sum(l_quantity) AS sum_qty, + sum(l_extendedprice) AS sum_base_price, + sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, + avg(l_quantity) AS avg_qty, + avg(l_extendedprice) AS avg_price, + avg(l_discount) AS avg_disc, + count(*) AS count_order + FROM + ${tableNameUk} + GROUP BY + l_returnflag, + l_linestatus + ORDER BY + l_returnflag, + l_linestatus + """ + assertTrue(result0.size()==result1.size()) + for (int i = 0; i < result0.size(); ++i) { + for (j = 0; j < result0[0].size(); j++) { + logger.info("result: " + result0[i][j] + "|" + result1[i][j]) + assertTrue(result0[i][j]==result1[i][j]) + } + } + + // delete + if (idx % 10 == 0) { + order_key = rd.nextInt(10) + part_key = rd.nextInt(10) + result0 = sql """ SELECT count(*) FROM ${tableNamePk} where L_ORDERKEY < $order_key and L_PARTKEY < $part_key; """ + result1 = sql """ SELECT count(*) FROM ${tableNameUk} where L_ORDERKEY < $order_key and L_PARTKEY < $part_key""" + logger.info("result:" + result0[0][0] + "|" + result1[0][0]) + sql "DELETE FROM ${tableNamePk} where L_ORDERKEY < $order_key and L_PARTKEY < $part_key" + sql "DELETE FROM ${tableNameUk} where L_ORDERKEY < $order_key and L_PARTKEY < $part_key" + } + } +}