From ec5ecd1604c3336cd6d00d28ecc8836f6fda029e Mon Sep 17 00:00:00 2001 From: zuochunwei Date: Wed, 26 Jan 2022 16:33:37 +0800 Subject: [PATCH] handle conflict (#7836) Co-authored-by: zuochunwei --- .../rowset/segment_v2/binary_dict_page.cpp | 7 +---- .../olap/rowset/segment_v2/bitshuffle_page.h | 9 ++---- be/src/vec/columns/column.h | 4 +++ be/src/vec/columns/column_nullable.h | 5 ++++ be/src/vec/columns/column_vector.h | 29 +++++++++++++++++++ 5 files changed, 41 insertions(+), 13 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp index 47e26b3ba0..f986fce598 100644 --- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp +++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp @@ -263,12 +263,7 @@ Status BinaryDictPageDecoder::next_batch(size_t* n, vectorized::MutableColumnPtr if (dst->is_nullable()) { auto nullable_column = assert_cast(dst.get()); dst_col_ptr = nullable_column->get_nested_column_ptr().get(); - - // fill null bitmap here, not null; - // todo(wb) using SIMD speed up here - for (int i = 0; i < max_fetch; i++) { - nullable_column->get_null_map_data().push_back(0); - } + nullable_column->get_null_map_column().insert_zeroed_elements(max_fetch); } if (dst_col_ptr->is_predicate_column()) { diff --git a/be/src/olap/rowset/segment_v2/bitshuffle_page.h b/be/src/olap/rowset/segment_v2/bitshuffle_page.h index 83eebae27b..fcaaab1fed 100644 --- a/be/src/olap/rowset/segment_v2/bitshuffle_page.h +++ b/be/src/olap/rowset/segment_v2/bitshuffle_page.h @@ -368,9 +368,7 @@ public: dst_col_ptr = nullable_column->get_nested_column_ptr().get(); // fill null bitmap here, not null; - for (int j = begin; j < end; j++) { - nullable_column->get_null_map_data().push_back(0); - } + nullable_column->get_null_map_column().insert_zeroed_elements(end - begin); } // todo(wb) Try to eliminate type judgment in pagedecoder @@ -403,10 +401,7 @@ public: dst_col_ptr->insert_data(reinterpret_cast(&date), 0); } } else { - // todo(wb) batch insert here - for (; begin < end; begin++) { - dst_col_ptr->insert_data((const char*)&_chunk.data[begin * SIZE_OF_TYPE], 0); - } + dst_col_ptr->insert_elements(&_chunk.data[begin * SIZE_OF_TYPE], (end - begin)); } *n = max_fetch; diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h index 71b86ae7e4..c730168d1c 100644 --- a/be/src/vec/columns/column.h +++ b/be/src/vec/columns/column.h @@ -181,6 +181,10 @@ public: for (size_t i = 0; i < length; ++i) insert_default(); } + virtual void insert_elements(void* elements, size_t num) { + LOG(FATAL) << "Method insert_elements is not supported for " << get_name(); + } + /** Removes last n elements. * Is used to support exception-safety of several operations. * For example, sometimes insertion should be reverted if we catch an exception during operation processing. diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h index cf5efffa59..a369c02abd 100644 --- a/be/src/vec/columns/column_nullable.h +++ b/be/src/vec/columns/column_nullable.h @@ -101,6 +101,11 @@ public: get_null_map_data().resize_fill(get_null_map_data().size() + length, 1); } + void insert_null_elements(int num) { + get_nested_column().insert_many_defaults(num); + get_null_map_column().insert_elements(1, num); + } + void pop_back(size_t n) override; ColumnPtr filter(const Filter& filt, ssize_t result_size_hint) const override; ColumnPtr filter_by_selector(const uint16_t* sel, size_t sel_size, diff --git a/be/src/vec/columns/column_vector.h b/be/src/vec/columns/column_vector.h index d909ee5a3c..69d98168d3 100644 --- a/be/src/vec/columns/column_vector.h +++ b/be/src/vec/columns/column_vector.h @@ -208,6 +208,35 @@ public: void insert_indices_from(const IColumn& src, const int* indices_begin, const int* indices_end) override; + void insert_elements(void* elements, size_t num) { + auto old_size = data.size(); + auto new_size = old_size + num; + data.resize(new_size); + memcpy(&data[old_size], elements, sizeof(value_type) * num); + } + + void insert_elements(const value_type& element, size_t num) { + auto old_size = data.size(); + auto new_size = old_size + num; + data.resize(new_size); + if constexpr (std::is_same_v) { + memset(&data[old_size], element, sizeof(value_type) * num); + } else if constexpr (std::is_same_v) { + memset(&data[old_size], element, sizeof(value_type) * num); + } else { + for (size_t i = 0; i < num; ++i) { + data[old_size + i] = element; + } + } + } + + void insert_zeroed_elements(size_t num) { + auto old_size = data.size(); + auto new_size = old_size + num; + data.resize(new_size); + memset(&data[old_size], 0, sizeof(value_type) * num); + } + ColumnPtr filter(const IColumn::Filter& filt, ssize_t result_size_hint) const override; // note(wb) this method is only used in storage layer now