diff --git a/be/src/common/config.h b/be/src/common/config.h index 648643b6a9..8b63ccf00c 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -295,12 +295,6 @@ CONF_mInt64(min_compaction_failure_interval_sec, "5"); // 5 seconds CONF_mInt32(max_base_compaction_threads, "4"); CONF_mInt32(max_cumu_compaction_threads, "10"); -// This config can be set to limit thread number in convert rowset thread pool. -CONF_mInt32(convert_rowset_thread_num, "0"); - -// initial sleep interval in seconds of scan alpha rowset -CONF_mInt32(scan_alpha_rowset_min_interval_sec, "3"); - // This config can be set to limit thread number in smallcompaction thread pool. CONF_mInt32(quick_compaction_max_threads, "10"); diff --git a/be/src/exprs/expr.cpp b/be/src/exprs/expr.cpp index 37f8bc4c31..695bc5a44e 100644 --- a/be/src/exprs/expr.cpp +++ b/be/src/exprs/expr.cpp @@ -90,8 +90,7 @@ Expr::Expr(const Expr& expr) _output_column(expr._output_column), _fn(expr._fn), _fn_context_index(expr._fn_context_index), - _constant_val(expr._constant_val), - _vector_compute_fn(expr._vector_compute_fn) {} + _constant_val(expr._constant_val) {} Expr::Expr(const TypeDescriptor& type) : _opcode(TExprOpcode::INVALID_OPCODE), diff --git a/be/src/exprs/expr.h b/be/src/exprs/expr.h index 56ff05e8a3..2fe979cc17 100644 --- a/be/src/exprs/expr.h +++ b/be/src/exprs/expr.h @@ -51,7 +51,6 @@ class TColumnValue; class TExpr; class TExprNode; class TupleIsNullPredicate; -class VectorizedRowBatch; class Literal; class MemTracker; struct UserFunctionCacheEntry; @@ -62,9 +61,6 @@ public: // typedef for compute functions. typedef void* (*ComputeFn)(Expr*, TupleRow*); - // typdef for vectorize compute functions. - typedef bool (*VectorComputeFn)(Expr*, VectorizedRowBatch*); - // Empty virtual destructor virtual ~Expr(); @@ -77,10 +73,6 @@ public: // TODO: stop having the result cached in this Expr object void* get_value(TupleRow* row) { return nullptr; } - // Vectorize Evalute expr and return result column index. - // Result cached in batch and valid as long as batch. - bool evaluate(VectorizedRowBatch* batch); - bool is_null_scalar_function(std::string& str) { // name and function_name both are required if (_fn.name.function_name.compare("is_null_pred") == 0) { @@ -374,9 +366,6 @@ protected: // get_const_val(). std::shared_ptr _constant_val; - // function to evaluate vectorize expr; typically set in prepare() - VectorComputeFn _vector_compute_fn; - /// Simple debug string that provides no expr subclass-specific information std::string debug_string(const std::string& expr_name) const { std::stringstream out; @@ -458,17 +447,6 @@ private: int _fn_ctx_idx_end = 0; }; -inline bool Expr::evaluate(VectorizedRowBatch* batch) { - DCHECK(_type.type != INVALID_TYPE); - - if (_is_slotref) { - // return SlotRef::vector_compute_fn(this, batch); - return false; - } else { - return _vector_compute_fn(this, batch); - } -} - template Status create_texpr_literal_node(const void* data, TExprNode* node) { if constexpr (std::is_same_v) { diff --git a/be/src/exprs/slot_ref.h b/be/src/exprs/slot_ref.h index 35ba68d66c..3e63bda69c 100644 --- a/be/src/exprs/slot_ref.h +++ b/be/src/exprs/slot_ref.h @@ -50,7 +50,6 @@ public: void* get_slot(TupleRow* row); Tuple* get_tuple(TupleRow* row); bool is_null_bit_set(TupleRow* row); - static bool vector_compute_fn(Expr* expr, VectorizedRowBatch* batch); static bool is_nullable(Expr* expr); virtual std::string debug_string() const override; virtual bool is_constant() const override { return false; } @@ -83,10 +82,6 @@ private: bool _is_nullable; }; -inline bool SlotRef::vector_compute_fn(Expr* expr, VectorizedRowBatch* /* batch */) { - return true; -} - inline void* SlotRef::get_value(Expr* expr, TupleRow* row) { SlotRef* ref = (SlotRef*)expr; Tuple* t = row->get_tuple(ref->_tuple_idx); diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt index 805cc9458f..1853c62523 100644 --- a/be/src/olap/CMakeLists.txt +++ b/be/src/olap/CMakeLists.txt @@ -29,25 +29,20 @@ add_library(Olap STATIC base_compaction.cpp base_tablet.cpp bloom_filter.hpp - bloom_filter_reader.cpp - bloom_filter_writer.cpp block_column_predicate.cpp byte_buffer.cpp collect_iterator.cpp compaction.cpp compaction_permit_limiter.cpp compress.cpp - convert_rowset.cpp cumulative_compaction.cpp cumulative_compaction_policy.cpp delete_handler.cpp delta_writer.cpp file_helper.cpp - file_stream.cpp generic_iterators.cpp hll.cpp bloom_filter_predicate.cpp - in_stream.cpp key_coder.cpp lru_cache.cpp memtable.cpp @@ -55,11 +50,9 @@ add_library(Olap STATIC merger.cpp null_predicate.cpp olap_cond.cpp - olap_index.cpp olap_meta.cpp olap_server.cpp options.cpp - out_stream.cpp page_cache.cpp push_handler.cpp reader.cpp @@ -71,14 +64,11 @@ add_library(Olap STATIC schema.cpp schema_change.cpp storage_migration_v2.cpp - serialize.cpp storage_engine.cpp data_dir.cpp short_key_index.cpp snapshot_manager.cpp stream_index_common.cpp - stream_index_reader.cpp - stream_index_writer.cpp stream_name.cpp tablet.cpp tablet_manager.cpp diff --git a/be/src/olap/bloom_filter_predicate.h b/be/src/olap/bloom_filter_predicate.h index c25258f55a..b4ed077709 100644 --- a/be/src/olap/bloom_filter_predicate.h +++ b/be/src/olap/bloom_filter_predicate.h @@ -23,7 +23,6 @@ #include "exprs/bloomfilter_predicate.h" #include "olap/column_predicate.h" -#include "runtime/vectorized_row_batch.h" #include "vec/columns/column_dictionary.h" #include "vec/columns/column_nullable.h" #include "vec/columns/column_vector.h" @@ -31,8 +30,6 @@ namespace doris { -class VectorizedRowBatch; - // only use in runtime filter and segment v2 template class BloomFilterColumnPredicate : public ColumnPredicate { @@ -48,8 +45,6 @@ public: PredicateType type() const override { return PredicateType::BF; } - void evaluate(VectorizedRowBatch* batch) const override; - void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const override; void evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, @@ -136,18 +131,6 @@ private: mutable bool _enable_pred = true; }; -// bloom filter column predicate do not support in segment v1 -template -void BloomFilterColumnPredicate::evaluate(VectorizedRowBatch* batch) const { - uint16_t n = batch->size(); - uint16_t* sel = batch->selected(); - if (!batch->selected_in_use()) { - for (uint16_t i = 0; i != n; ++i) { - sel[i] = i; - } - } -} - template void BloomFilterColumnPredicate::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const { diff --git a/be/src/olap/bloom_filter_reader.cpp b/be/src/olap/bloom_filter_reader.cpp deleted file mode 100644 index 1c3bfc2431..0000000000 --- a/be/src/olap/bloom_filter_reader.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/bloom_filter_reader.h" - -namespace doris { - -BloomFilterIndexReader::~BloomFilterIndexReader() { - _entry.reset(); - - if (!_is_using_cache) { - SAFE_DELETE_ARRAY(_buffer); - } -} - -Status BloomFilterIndexReader::init(char* buffer, size_t buffer_size, bool is_using_cache, - uint32_t hash_function_num, uint32_t bit_num) { - Status res = Status::OK(); - - _buffer = buffer; - _buffer_size = buffer_size; - _is_using_cache = is_using_cache; - - BloomFilterIndexHeader* header = reinterpret_cast(_buffer); - _step_size = bit_num >> 3; - _entry_count = header->block_count; - _hash_function_num = hash_function_num; - _start_offset = sizeof(BloomFilterIndexHeader); - if (_step_size * _entry_count + _start_offset > _buffer_size) { - LOG(WARNING) << "invalid param found.[buffer_size=" << buffer_size << " bit_num=" << bit_num - << " block_count=" << _entry_count << " header_size=" << _start_offset << "]"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - return res; -} - -const BloomFilter& BloomFilterIndexReader::entry(uint64_t entry_id) { - _entry.init((uint64_t*)(_buffer + _start_offset + _step_size * entry_id), - _step_size / sizeof(uint64_t), _hash_function_num); - return _entry; -} - -size_t BloomFilterIndexReader::entry_count() { - return _entry_count; -} - -} // namespace doris diff --git a/be/src/olap/bloom_filter_reader.h b/be/src/olap/bloom_filter_reader.h deleted file mode 100644 index 06408a0ada..0000000000 --- a/be/src/olap/bloom_filter_reader.h +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "olap/bloom_filter.hpp" -#include "olap/bloom_filter_writer.h" - -namespace doris { - -// Each bloom filter index contains multiple bloom filter entries, -// each of which is related to a data block. -// BloomFilterIndexReader allow caller to get specified bloom filter entry -// by parsing bloom filter index buffer. -class BloomFilterIndexReader { -public: - BloomFilterIndexReader() {} - ~BloomFilterIndexReader(); - - // Init BloomFilterIndexReader with given bloom filter index buffer - Status init(char* buffer, size_t buffer_size, bool is_using_cache, uint32_t hash_function_num, - uint32_t bit_num); - - // Get specified bloom filter entry - const BloomFilter& entry(uint64_t entry_id); - - // Get total number of bloom filter entries in current bloom filter index buffer - size_t entry_count(); - -private: - // Bloom filter index buffer and length - char* _buffer; - size_t _buffer_size; - - // Total bloom filter entries count, start offset, each bloom filter entry size - size_t _entry_count; - size_t _start_offset; - size_t _step_size; - - // Bloom filter param - uint32_t _hash_function_num; - - // BloomFilterIndexReader will not release bloom filter index buffer in destructor - // when it is cached in memory - bool _is_using_cache; - - BloomFilter _entry; -}; - -} // namespace doris diff --git a/be/src/olap/bloom_filter_writer.cpp b/be/src/olap/bloom_filter_writer.cpp deleted file mode 100644 index 28517be782..0000000000 --- a/be/src/olap/bloom_filter_writer.cpp +++ /dev/null @@ -1,109 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/bloom_filter_writer.h" - -#include - -namespace doris { - -BloomFilterIndexWriter::~BloomFilterIndexWriter() { - for (std::vector::iterator it = _bloom_filters.begin(); - it != _bloom_filters.end(); ++it) { - SAFE_DELETE(*it); - } -} - -Status BloomFilterIndexWriter::add_bloom_filter(BloomFilter* bf) { - try { - _bloom_filters.push_back(bf); - } catch (...) { - LOG(WARNING) << "add bloom filter to vector fail"; - return Status::OLAPInternalError(OLAP_ERR_STL_ERROR); - } - - return Status::OK(); -} - -uint64_t BloomFilterIndexWriter::estimate_buffered_memory() { - uint64_t buffered_size = sizeof(_header); - if (_bloom_filters.size() > 0) { - buffered_size += - _bloom_filters.size() * _bloom_filters[0]->bit_set_data_len() * sizeof(uint64_t); - } - return buffered_size; -} - -Status BloomFilterIndexWriter::write_to_buffer(OutStream* out_stream) { - Status res = Status::OK(); - if (nullptr == out_stream) { - LOG(WARNING) << "out stream is null"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - // write header - _header.block_count = _bloom_filters.size(); - res = out_stream->write(reinterpret_cast(&_header), sizeof(_header)); - if (!res.ok()) { - LOG(WARNING) << "write bloom filter index header fail"; - return res; - } - - // write bloom filters - for (size_t i = 0; i < _bloom_filters.size(); ++i) { - uint64_t* data = _bloom_filters[i]->bit_set_data(); - uint32_t data_len = _bloom_filters[i]->bit_set_data_len(); - res = out_stream->write(reinterpret_cast(data), sizeof(uint64_t) * data_len); - if (!res.ok()) { - LOG(WARNING) << "write bloom filter index fail, i=" << i; - return res; - } - } - - return res; -} - -Status BloomFilterIndexWriter::write_to_buffer(char* buffer, size_t buffer_size) { - Status res = Status::OK(); - if (nullptr == buffer) { - LOG(WARNING) << "out stream is nullptr."; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - if (estimate_buffered_memory() > buffer_size) { - LOG(WARNING) << "need more buffer. [scr_size=" << estimate_buffered_memory() - << " buffer_size=" << buffer_size << "]"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - // write header - _header.block_count = _bloom_filters.size(); - memcpy(buffer, reinterpret_cast(&_header), sizeof(_header)); - buffer += sizeof(_header); - - // write bloom filters - for (size_t i = 0; i < _bloom_filters.size(); ++i) { - uint64_t* data = _bloom_filters[i]->bit_set_data(); - uint32_t data_len = _bloom_filters[i]->bit_set_data_len(); - memcpy(buffer, reinterpret_cast(data), sizeof(uint64_t) * data_len); - buffer += sizeof(uint64_t) * data_len; - } - - return res; -} - -} // namespace doris diff --git a/be/src/olap/bloom_filter_writer.h b/be/src/olap/bloom_filter_writer.h deleted file mode 100644 index b2c8224c3f..0000000000 --- a/be/src/olap/bloom_filter_writer.h +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "olap/bloom_filter.hpp" -#include "olap/out_stream.h" - -namespace doris { - -class BloomFilterIndexWriter { -public: - BloomFilterIndexWriter() {} - ~BloomFilterIndexWriter(); - - Status add_bloom_filter(BloomFilter* bf); - uint64_t estimate_buffered_memory(); - Status write_to_buffer(OutStream* out_stream); - Status write_to_buffer(char* buffer, size_t buffer_size); - -private: - std::vector _bloom_filters; - BloomFilterIndexHeader _header; -}; - -} // namespace doris diff --git a/be/src/olap/column_predicate.h b/be/src/olap/column_predicate.h index b02cdc0146..be7a1bdc59 100644 --- a/be/src/olap/column_predicate.h +++ b/be/src/olap/column_predicate.h @@ -28,7 +28,6 @@ using namespace doris::segment_v2; namespace doris { -class VectorizedRowBatch; class Schema; class RowBlockV2; @@ -64,9 +63,6 @@ public: virtual PredicateType type() const = 0; - //evaluate predicate on VectorizedRowBatch - virtual void evaluate(VectorizedRowBatch* batch) const = 0; - // evaluate predicate on ColumnBlock virtual void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const = 0; virtual void evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, diff --git a/be/src/olap/comparison_predicate.h b/be/src/olap/comparison_predicate.h index d1fdfcbb81..f0401aa4cc 100644 --- a/be/src/olap/comparison_predicate.h +++ b/be/src/olap/comparison_predicate.h @@ -20,7 +20,6 @@ #include #include "olap/column_predicate.h" -#include "runtime/vectorized_row_batch.h" #include "vec/columns/column_dictionary.h" namespace doris { @@ -40,54 +39,6 @@ public: PredicateType type() const override { return PT; } - void evaluate(VectorizedRowBatch* batch) const override { - uint16_t n = batch->size(); - if (n == 0) { - return; - } - uint16_t* sel = batch->selected(); - const T* col_vector = reinterpret_cast(batch->column(_column_id)->col_data()); - uint16_t new_size = 0; - if (batch->column(_column_id)->no_nulls()) { - if (batch->selected_in_use()) { - for (uint16_t j = 0; j != n; ++j) { - uint16_t i = sel[j]; - sel[new_size] = i; - new_size += _operator(col_vector[i], _value); - } - batch->set_size(new_size); - } else { - for (uint16_t i = 0; i != n; ++i) { - sel[new_size] = i; - new_size += _operator(col_vector[i], _value); - } - if (new_size < n) { - batch->set_size(new_size); - batch->set_selected_in_use(true); - } - } - } else { - bool* is_null = batch->column(_column_id)->is_null(); - if (batch->selected_in_use()) { - for (uint16_t j = 0; j != n; ++j) { - uint16_t i = sel[j]; - sel[new_size] = i; - new_size += (!is_null[i] && _operator(col_vector[i], _value)); - } - batch->set_size(new_size); - } else { - for (uint16_t i = 0; i != n; ++i) { - sel[new_size] = i; - new_size += (!is_null[i] && _operator(col_vector[i], _value)); - } - if (new_size < n) { - batch->set_size(new_size); - batch->set_selected_in_use(true); - } - } - } - } - void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const override { uint16_t new_size = 0; if (block->is_nullable()) { diff --git a/be/src/olap/convert_rowset.cpp b/be/src/olap/convert_rowset.cpp deleted file mode 100644 index c71e66c246..0000000000 --- a/be/src/olap/convert_rowset.cpp +++ /dev/null @@ -1,140 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/convert_rowset.h" - -namespace doris { - -Status ConvertRowset::do_convert() { - if (!_tablet->init_succeeded()) { - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - std::unique_lock base_compaction_lock(_tablet->get_base_compaction_lock(), - std::try_to_lock); - std::unique_lock cumulative_compaction_lock( - _tablet->get_cumulative_compaction_lock(), std::try_to_lock); - if (!base_compaction_lock.owns_lock() || !cumulative_compaction_lock.owns_lock()) { - LOG(INFO) << "The tablet is under compaction. tablet=" << _tablet->full_name(); - return Status::OLAPInternalError(OLAP_ERR_CE_TRY_CE_LOCK_ERROR); - } - - std::vector alpah_rowsets; - _tablet->find_alpha_rowsets(&alpah_rowsets); - - Merger::Statistics stats; - Status res; - const size_t max_convert_row_count = 20000000; - size_t row_count = 0; - for (size_t i = 0; i < alpah_rowsets.size(); ++i) { - Version output_version = - Version(alpah_rowsets[i]->start_version(), alpah_rowsets[i]->end_version()); - - RowsetReaderSharedPtr input_rs_reader; - RETURN_NOT_OK(alpah_rowsets[i]->create_reader(&input_rs_reader)); - - std::unique_ptr output_rs_writer; - RETURN_NOT_OK(_tablet->create_rowset_writer(output_version, VISIBLE, NONOVERLAPPING, - &output_rs_writer)); - res = Merger::merge_rowsets(_tablet, ReaderType::READER_BASE_COMPACTION, {input_rs_reader}, - output_rs_writer.get(), &stats); - - if (!res.ok()) { - LOG(WARNING) << "fail to convert rowset. res=" << res - << ", tablet=" << _tablet->full_name(); - return res; - } else { - auto output_rowset = output_rs_writer->build(); - if (output_rowset == nullptr) { - LOG(WARNING) << "rowset writer build failed" - << ", tablet=" << _tablet->full_name(); - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - RETURN_NOT_OK(check_correctness(alpah_rowsets[i], output_rowset, stats)); - - row_count += alpah_rowsets[i]->num_rows(); - - RETURN_NOT_OK(_modify_rowsets(alpah_rowsets[i], output_rowset)); - - LOG(INFO) << "succeed to convert rowset" - << ". tablet=" << _tablet->full_name() - << ", output_version=" << output_version - << ", disk=" << _tablet->data_dir()->path(); - - if (row_count >= max_convert_row_count) { - break; - } - } - } - return Status::OK(); -} - -Status ConvertRowset::check_correctness(RowsetSharedPtr input_rowset, RowsetSharedPtr output_rowset, - const Merger::Statistics& stats) { - // 1. check row number - if (input_rowset->num_rows() != - output_rowset->num_rows() + stats.merged_rows + stats.filtered_rows) { - LOG(WARNING) << "row_num does not match between input and output! " - << "input_row_num=" << input_rowset->num_rows() - << ", merged_row_num=" << stats.merged_rows - << ", filtered_row_num=" << stats.filtered_rows - << ", output_row_num=" << output_rowset->num_rows(); - - // ATTN(cmy): We found that the num_rows in some rowset meta may be set to the wrong value, - // but it is not known which version of the code has the problem. So when the convert - // result is inconsistent, we then try to verify by num_rows recorded in segment_groups. - // If the check passes, ignore the error and set the correct value in the output rowset meta - // to fix this problem. - // Only handle alpha rowset because we only find this bug in alpha rowset - int64_t num_rows = _get_input_num_rows_from_seg_grps(input_rowset); - if (num_rows == -1) { - return Status::OLAPInternalError(OLAP_ERR_CHECK_LINES_ERROR); - } - if (num_rows != output_rowset->num_rows() + stats.merged_rows + stats.filtered_rows) { - // If it is still incorrect, it may be another problem - LOG(WARNING) << "row_num got from seg groups does not match between cumulative input " - "and output! " - << "input_row_num=" << num_rows << ", merged_row_num=" << stats.merged_rows - << ", filtered_row_num=" << stats.filtered_rows - << ", output_row_num=" << output_rowset->num_rows(); - - return Status::OLAPInternalError(OLAP_ERR_CHECK_LINES_ERROR); - } - } - return Status::OK(); -} - -int64_t ConvertRowset::_get_input_num_rows_from_seg_grps(RowsetSharedPtr rowset) { - int64_t num_rows = 0; - for (auto& seg_grp : rowset->rowset_meta()->alpha_rowset_extra_meta_pb().segment_groups()) { - num_rows += seg_grp.num_rows(); - } - return num_rows; -} -Status ConvertRowset::_modify_rowsets(RowsetSharedPtr input_rowset, RowsetSharedPtr output_rowset) { - std::vector input_rowsets; - input_rowsets.push_back(input_rowset); - - std::vector output_rowsets; - output_rowsets.push_back(output_rowset); - - std::lock_guard wrlock(_tablet->get_header_lock()); - RETURN_NOT_OK(_tablet->modify_rowsets(output_rowsets, input_rowsets, true)); - _tablet->save_meta(); - return Status::OK(); -} -} // namespace doris \ No newline at end of file diff --git a/be/src/olap/convert_rowset.h b/be/src/olap/convert_rowset.h deleted file mode 100644 index a691d38624..0000000000 --- a/be/src/olap/convert_rowset.h +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "olap/merger.h" -#include "olap/storage_engine.h" -#include "olap/tablet.h" - -namespace doris { -class DataDir; -class ConvertRowset { -public: - ConvertRowset(TabletSharedPtr tablet) : _tablet(tablet) {} - Status do_convert(); - -private: - Status check_correctness(RowsetSharedPtr input_rowset, RowsetSharedPtr output_rowset, - const Merger::Statistics& stats); - int64_t _get_input_num_rows_from_seg_grps(RowsetSharedPtr rowset); - Status _modify_rowsets(RowsetSharedPtr input_rowset, RowsetSharedPtr output_rowset); - -private: - TabletSharedPtr _tablet; - - DISALLOW_COPY_AND_ASSIGN(ConvertRowset); -}; -} // namespace doris \ No newline at end of file diff --git a/be/src/olap/file_stream.cpp b/be/src/olap/file_stream.cpp deleted file mode 100644 index 54d76d9bac..0000000000 --- a/be/src/olap/file_stream.cpp +++ /dev/null @@ -1,187 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/file_stream.h" - -#include "olap/byte_buffer.h" -#include "olap/out_stream.h" - -namespace doris { - -ReadOnlyFileStream::ReadOnlyFileStream(FileHandler* handler, StorageByteBuffer** shared_buffer, - Decompressor decompressor, uint32_t compress_buffer_size, - OlapReaderStatistics* stats) - : _file_cursor(handler, 0, 0), - _compressed_helper(nullptr), - _uncompressed(nullptr), - _shared_buffer(shared_buffer), - _decompressor(decompressor), - _compress_buffer_size(compress_buffer_size + sizeof(StreamHead)), - _current_compress_position(std::numeric_limits::max()), - _stats(stats) {} - -ReadOnlyFileStream::ReadOnlyFileStream(FileHandler* handler, StorageByteBuffer** shared_buffer, - uint64_t offset, uint64_t length, Decompressor decompressor, - uint32_t compress_buffer_size, OlapReaderStatistics* stats) - : _file_cursor(handler, offset, length), - _compressed_helper(nullptr), - _uncompressed(nullptr), - _shared_buffer(shared_buffer), - _decompressor(decompressor), - _compress_buffer_size(compress_buffer_size + sizeof(StreamHead)), - _current_compress_position(std::numeric_limits::max()), - _stats(stats) {} - -Status ReadOnlyFileStream::_assure_data() { - // if still has data in uncompressed - if (OLAP_LIKELY(_uncompressed != nullptr && _uncompressed->remaining() > 0)) { - return Status::OK(); - } else if (_file_cursor.eof()) { - VLOG_TRACE << "STREAM EOF. length=" << _file_cursor.length() - << ", used=" << _file_cursor.position(); - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_EOF); - } - - StreamHead header; - size_t file_cursor_used = _file_cursor.position(); - Status res = Status::OK(); - { - SCOPED_RAW_TIMER(&_stats->io_ns); - res = _file_cursor.read(reinterpret_cast(&header), sizeof(header)); - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "read header fail"; - return res; - } - res = _fill_compressed(header.length); - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "read header fail"; - return res; - } - _stats->compressed_bytes_read += sizeof(header) + header.length; - } - - if (header.type == StreamHead::UNCOMPRESSED) { - StorageByteBuffer* tmp = _compressed_helper; - _compressed_helper = *_shared_buffer; - *_shared_buffer = tmp; - } else { - _compressed_helper->set_position(0); - _compressed_helper->set_limit(_compress_buffer_size); - { - SCOPED_RAW_TIMER(&_stats->decompress_ns); - res = _decompressor(*_shared_buffer, _compressed_helper); - if (!res.ok()) { - LOG(WARNING) << "fail to decompress err=" << res; - return res; - } - } - } - _stats->uncompressed_bytes_read += _compressed_helper->limit(); - - _uncompressed = _compressed_helper; - _current_compress_position = file_cursor_used; - return res; -} - -// 设置读取的位置 -Status ReadOnlyFileStream::seek(PositionProvider* position) { - Status res = Status::OK(); - // 先seek到解压前的位置，也就是writer中写入的spilled byte - int64_t compressed_position = position->get_next(); - int64_t uncompressed_bytes = position->get_next(); - if (_current_compress_position == compressed_position && nullptr != _uncompressed) { - /* - * 多数情况下不会出现_uncompressed为NULL的情况， - * 但varchar类型的数据可能会导致查询中出现_uncompressed == nullptr 。 - * 假设查询恰好命中A压缩块的最后一行, 而相临下一个 - * 中压缩块varchar全是空串，会导致_uncompressed == nullptr。 - * 如果后面的segmentreader中还需要再次遍历A压缩块，会出现空指针。 - */ - } else { - _file_cursor.seek(compressed_position); - _uncompressed = nullptr; - - res = _assure_data(); - if (OLAP_LIKELY(res.ok())) { - // assure data will be successful in most case - } else if (res.precise_code() == OLAP_ERR_COLUMN_STREAM_EOF) { - VLOG_TRACE << "file stream eof."; - return res; - } else { - LOG(WARNING) << "fail to assure data after seek"; - return res; - } - } - - res = _uncompressed->set_position(uncompressed_bytes); - if (!res.ok()) { - LOG(WARNING) << "fail to set position. res= " << res << ", position=" << uncompressed_bytes; - return res; - } - - return Status::OK(); -} - -// 跳过指定的size的流 -Status ReadOnlyFileStream::skip(uint64_t skip_length) { - Status res = _assure_data(); - - if (!res.ok()) { - return res; - } - - uint64_t skip_byte = 0; - uint64_t byte_to_skip = skip_length; - - // 如果不够跳，则先尝试跳过整个数据块，直到当前的数据 - // 剩下的字节数足够跳过 又或者时EOF - do { - skip_byte = std::min(_uncompressed->remaining(), byte_to_skip); - _uncompressed->set_position(_uncompressed->position() + skip_byte); - byte_to_skip -= skip_byte; - // 如果跳到当前的块尽头，那么assure可以换到下一个块 - // 如果当前块就可以满足skip_length,那么_assure_data没任何作用。 - res = _assure_data(); - // while放下面，通常会少判断一次 - } while (byte_to_skip != 0 && res.ok()); - - return res; -} - -Status ReadOnlyFileStream::_fill_compressed(size_t length) { - if (length > _compress_buffer_size) { - LOG(WARNING) << "overflow when fill compressed." - << ", length=" << length << ", compress_size" << _compress_buffer_size; - return Status::OLAPInternalError(OLAP_ERR_OUT_OF_BOUND); - } - - Status res = _file_cursor.read((*_shared_buffer)->array(), length); - if (!res.ok()) { - LOG(WARNING) << "fail to fill compressed buffer."; - return res; - } - - (*_shared_buffer)->set_position(0); - (*_shared_buffer)->set_limit(length); - return res; -} - -uint64_t ReadOnlyFileStream::available() { - return _file_cursor.remain(); -} - -} // namespace doris diff --git a/be/src/olap/file_stream.h b/be/src/olap/file_stream.h deleted file mode 100644 index 176a3cd22c..0000000000 --- a/be/src/olap/file_stream.h +++ /dev/null @@ -1,276 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include -#include -#include -#include - -#include "olap/byte_buffer.h" -#include "olap/compress.h" -#include "olap/file_helper.h" -#include "olap/olap_common.h" -#include "olap/stream_index_reader.h" -#include "util/runtime_profile.h" - -namespace doris { - -// Define the input data stream interface. -class ReadOnlyFileStream { -public: - // Construct method, use a group of ByteBuffer to create an InStream The position of the input - // ByteBuffer in the stream can be discontinuous, for example, certain data is not required to be - // determined by Index. After reading, this part of the data is not read. However, InStream - // encapsulates the fact that the ByteBuffer is discontinuous. From the perspective of the - // upper-layer user, it is still accessing a continuous stream. The upper-layer user should - // ensure that the StorageByteBuffer is not read. Void locations with no data in between. - // - // When mmap is used, it will degenerate to only one ByteBuffer, whether to use mmap depends on - // the test results in the performance tuning phase. - // - // Input: - // inputs - A set of ByteBuffer holds the data in a specific stream. - // offsets - The offset position of the data of each ByteBuffer in the input stream in the stream. - // length - The total byte length of the stream. - // Decompressor - Provides a decompression function if the stream is compressed, otherwise it is 'NULL'. - // compress_buffer_size - If compression is used, give the compressed block size. - ReadOnlyFileStream(FileHandler* handler, StorageByteBuffer** shared_buffer, - Decompressor decompressor, uint32_t compress_buffer_size, - OlapReaderStatistics* stats); - - ReadOnlyFileStream(FileHandler* handler, StorageByteBuffer** shared_buffer, uint64_t offset, - uint64_t length, Decompressor decompressor, uint32_t compress_buffer_size, - OlapReaderStatistics* stats); - - ~ReadOnlyFileStream() { SAFE_DELETE(_compressed_helper); } - - Status init() { - _compressed_helper = StorageByteBuffer::create(_compress_buffer_size); - if (nullptr == _compressed_helper) { - LOG(WARNING) << "fail to create compressed buffer"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _uncompressed = nullptr; - return Status::OK(); - } - - void reset(uint64_t offset, uint64_t length) { _file_cursor.reset(offset, length); } - - // Read a byte from the data stream, move the internal pointer backward - // If the stream ends, it returns `Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_EOF)`. - Status read(char* byte); - - // Read a piece of data from a data stream. - // Input: - // buffer - Store read data. - // buf_size - The size of the buffer is given when inputting, - // and the number of bytes actually read is given when returning. - // If the stream ends, it returns `Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_EOF)`. - Status read(char* buffer, uint64_t* buf_size); - - Status read_all(char* buffer, uint64_t* buf_size); - // set read position. - Status seek(PositionProvider* position); - - // Skip streams of specified size. - Status skip(uint64_t skip_length); - - // Returns the total length of the stream. - uint64_t stream_length() { return _file_cursor.length(); } - - bool eof() { - if (_uncompressed == nullptr) { - return _file_cursor.eof(); - } else { - return _file_cursor.eof() && _uncompressed->remaining() == 0; - } - } - - // Returns the remaining readable bytes of the current block. - uint64_t available(); - - size_t get_buffer_size() { return _compress_buffer_size; } - - void get_buf(char** buf, uint32_t* remaining_bytes) { - if (UNLIKELY(_uncompressed == nullptr)) { - *buf = nullptr; - *remaining_bytes = 0; - } else { - *buf = _uncompressed->array(); - *remaining_bytes = _uncompressed->remaining(); - } - } - - void get_position(uint32_t* position) { *position = _uncompressed->position(); } - - void set_position(uint32_t pos) { _uncompressed->set_position(pos); } - - int remaining() { - if (_uncompressed == nullptr) { - return 0; - } - return _uncompressed->remaining(); - } - -private: - // Use to read a specified range in file - class FileCursor { - public: - FileCursor(FileHandler* file_handler, size_t offset, size_t length) - : _file_handler(file_handler), _offset(offset), _length(length), _used(0) {} - - ~FileCursor() {} - - void reset(size_t offset, size_t length) { - _offset = offset; - _length = length; - _used = 0; - } - - Status read(char* out_buffer, size_t length) { - if (_used + length <= _length) { - Status res = _file_handler->pread(out_buffer, length, _used + _offset); - if (!res.ok()) { - LOG(WARNING) << "fail to read from file. res = " << res; - return res; - } - - _used += length; - } else { - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_EOF); - } - - return Status::OK(); - } - - size_t position() { return _used; } - - size_t remain() { return _length - _used; } - - size_t length() { return _length; } - - bool eof() { return _used == _length; } - - Status seek(size_t offset) { - if (offset > _length) { - return Status::OLAPInternalError(OLAP_ERR_OUT_OF_BOUND); - } - - _used = offset; - return Status::OK(); - } - - const std::string& file_name() const { return _file_handler->file_name(); } - - size_t offset() const { return _offset; } - - private: - FileHandler* _file_handler; - size_t _offset; // start from where - size_t _length; // length limit - size_t _used; - }; - - Status _assure_data(); - Status _fill_compressed(size_t length); - - FileCursor _file_cursor; - StorageByteBuffer* _compressed_helper; - StorageByteBuffer* _uncompressed; - StorageByteBuffer** _shared_buffer; - - Decompressor _decompressor; - size_t _compress_buffer_size; - size_t _current_compress_position; - - OlapReaderStatistics* _stats; - - DISALLOW_COPY_AND_ASSIGN(ReadOnlyFileStream); -}; - -inline Status ReadOnlyFileStream::read(char* byte) { - Status res = _assure_data(); - - if (!res.ok()) { - return res; - } - - res = _uncompressed->get(byte); - return res; -} - -inline Status ReadOnlyFileStream::read(char* buffer, uint64_t* buf_size) { - Status res; - uint64_t read_length = *buf_size; - *buf_size = 0; - - do { - res = _assure_data(); - if (!res.ok()) { - break; - } - - uint64_t actual_length = std::min(read_length - *buf_size, _uncompressed->remaining()); - - res = _uncompressed->get(buffer, actual_length); - if (!res.ok()) { - break; - } - - *buf_size += actual_length; - buffer += actual_length; - } while (*buf_size < read_length); - - return res; -} - -inline Status ReadOnlyFileStream::read_all(char* buffer, uint64_t* buffer_size) { - Status res; - uint64_t read_length = 0; - uint64_t buffer_remain = *buffer_size; - - while (_assure_data()) { - read_length = _uncompressed->remaining(); - - if (buffer_remain < read_length) { - res = Status::OLAPInternalError(OLAP_ERR_BUFFER_OVERFLOW); - break; - } - - res = _uncompressed->get(buffer, read_length); - if (!res.ok()) { - break; - } - - buffer_remain -= read_length; - buffer += read_length; - } - - if (eof()) { - *buffer_size -= buffer_remain; - return Status::OK(); - } - - return res; -} - -} // namespace doris diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h index 2a0fcb0901..3d22ecb83f 100644 --- a/be/src/olap/in_list_predicate.h +++ b/be/src/olap/in_list_predicate.h @@ -26,7 +26,6 @@ #include "decimal12.h" #include "olap/column_predicate.h" #include "runtime/string_value.h" -#include "runtime/vectorized_row_batch.h" #include "uint24.h" #include "vec/columns/column_dictionary.h" #include "vec/core/types.h" @@ -77,8 +76,6 @@ struct equal_to { namespace doris { -class VectorizedRowBatch; - template class InListPredicateBase : public ColumnPredicate { public: @@ -88,57 +85,6 @@ public: PredicateType type() const override { return PT; } - void evaluate(VectorizedRowBatch* batch) const override { - uint16_t n = batch->size(); - if (!n) { - return; - } - - uint16_t* sel = batch->selected(); - const T* col_vector = reinterpret_cast(batch->column(_column_id)->col_data()); - uint16_t new_size = 0; - if (batch->column(_column_id)->no_nulls()) { - if (batch->selected_in_use()) { - for (uint16_t j = 0; j != n; ++j) { - uint16_t i = sel[j]; - sel[new_size] = i; - new_size += _operator(_values.find(col_vector[i]), _values.end()); - } - batch->set_size(new_size); - } else { - for (uint16_t i = 0; i != n; ++i) { - sel[new_size] = i; - new_size += _operator(_values.find(col_vector[i]), _values.end()); - } - if (new_size < n) { - batch->set_size(new_size); - batch->set_selected_in_use(true); - } - } - } else { - bool* is_null = batch->column(_column_id)->is_null(); - if (batch->selected_in_use()) { - for (uint16_t j = 0; j != n; ++j) { - uint16_t i = sel[j]; - sel[new_size] = i; - new_size += - (!is_null[i] && _operator(_values.find(col_vector[i]), _values.end())); - } - batch->set_size(new_size); - } else { - for (int i = 0; i != n; ++i) { - sel[new_size] = i; - new_size += - (!is_null[i] && _operator(_values.find(col_vector[i]), _values.end())); - } - if (new_size < n) { - batch->set_size(new_size); - batch->set_selected_in_use(true); - } - } - } - }; - void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const override { if (block->is_nullable()) { _base_evaluate(block, sel, size); diff --git a/be/src/olap/in_stream.cpp b/be/src/olap/in_stream.cpp deleted file mode 100644 index 40c45a6f22..0000000000 --- a/be/src/olap/in_stream.cpp +++ /dev/null @@ -1,305 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/in_stream.h" - -#include "olap/byte_buffer.h" -#include "olap/out_stream.h" - -namespace doris { - -InStream::InStream(std::vector* inputs, const std::vector& offsets, - uint64_t length, Decompressor decompressor, uint32_t compress_buffer_size) - : _inputs(*inputs), - _offsets(offsets), - _length(length), - _decompressor(decompressor), - _compress_buffer_size(compress_buffer_size), - _current_offset(0), - _current_range(0), - _compressed(nullptr), - _uncompressed(nullptr) {} - -InStream::~InStream() { - SAFE_DELETE(_compressed); - SAFE_DELETE(_uncompressed); -} - -Status InStream::_slice(uint64_t chunk_size, StorageByteBuffer** out_slice) { - uint64_t len = chunk_size; - uint64_t old_offset = _current_offset; - StorageByteBuffer* slice = nullptr; - - //如果buffer够读，拿出一个chunksize，并设置position - if (OLAP_LIKELY(_compressed->remaining() >= len)) { - slice = StorageByteBuffer::reference_buffer(_compressed, _compressed->position(), len); - - if (OLAP_UNLIKELY(nullptr == slice)) { - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _compressed->set_position(_compressed->position() + len); - *out_slice = slice; - // 这里之前没有设置_current_offset - _current_offset += len; - return Status::OK(); - } else if (_current_range >= _inputs.size() - 1) { - // 如果buffer用完了 - LOG(WARNING) << "EOF in InStream. [Need=" << chunk_size << "]"; - return Status::OLAPInternalError(OLAP_ERR_OUT_OF_BOUND); - } - - // 这里并不分配chuck_size, 而是分配一个最大值, 这样利于减少内存碎片 - slice = StorageByteBuffer::create(_compress_buffer_size); - - if (OLAP_UNLIKELY(nullptr == slice)) { - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - // 当前的compress里的buffer不够了 - _current_offset += _compressed->remaining(); - len -= _compressed->remaining(); - // 先拿一部分出来 - slice->put(&(_compressed->array()[_compressed->position()]), _compressed->remaining()); - - // 向后边的buffer移动 - ++_current_range; - - while (len > 0 && _current_range < _inputs.size()) { - SAFE_DELETE(_compressed); - // 再取一部分压缩过的buffer - _compressed = StorageByteBuffer::reference_buffer(_inputs[_current_range], - _inputs[_current_range]->position(), - _inputs[_current_range]->remaining()); - - if (OLAP_UNLIKELY(nullptr == _compressed)) { - SAFE_DELETE(slice); - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - // 如果剩下的大于需要取的部分，拿出来 - // 否则继续这个过程 - if (_compressed->remaining() >= len) { - slice->put(&(_compressed->array()[_compressed->position()]), len); - _compressed->set_position(_compressed->position() + len); - _current_offset += len; - slice->flip(); - *out_slice = slice; - return Status::OK(); - } else { - _current_offset += _compressed->remaining(); - len -= _compressed->remaining(); - slice->put(&(_compressed->array()[_compressed->position()]), _compressed->remaining()); - } - - ++_current_range; - } - - // 到这里就意味着上边的循环里没有取到足够的buf - // 回退到进来之前的状态 - _seek(old_offset); - LOG(WARNING) << "EOF in InStream. [Need=" << chunk_size << "]"; - return Status::OLAPInternalError(OLAP_ERR_OUT_OF_BOUND); -} - -Status InStream::_assure_data() { - Status res = Status::OK(); - - if (OLAP_LIKELY(_uncompressed != nullptr && _uncompressed->remaining() > 0)) { - return Status::OK(); - } else if (OLAP_UNLIKELY((_uncompressed == nullptr || _uncompressed->remaining() == 0) && - (_current_offset == _length))) { - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_EOF); - } - - // read head and data - SAFE_DELETE(_uncompressed); - - // 到这里说明当前uncompress没有什么可以读了，input拿数据 - // 如果没有compress。或者compress耗尽，用_seek向后一个buff移动 - if (_compressed == nullptr || _compressed->remaining() == 0) { - res = _seek(_current_offset); - if (!res.ok()) { - return res; - } - } - - // 取了compress之后，看下是不是能拿出一个head的长度，如果可以 - // 则解析head，从outstream代码看header不会分开写在两个压缩块中 - if (OLAP_LIKELY(_compressed->remaining() >= sizeof(StreamHead))) { - // 如果可以，从compress中拿出一个head，head是未压缩的。 - StreamHead head; - _compressed->get((char*)&head, sizeof(head)); - - if (head.length > _compress_buffer_size) { - LOG(WARNING) << "chunk size is larger than buffer size. [chunk=" << head.length - << " buffer_size=" << _compress_buffer_size << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_READ_STREAM); - } - - // 向后移动整体偏移 - _current_offset += sizeof(StreamHead); - StorageByteBuffer* slice = nullptr; - - // 根据head取一块buf，这里应该要调整_current_offset - res = _slice(head.length, &slice); - if (OLAP_LIKELY(!res.ok())) { - LOG(WARNING) << "fail to slice data from stream."; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_READ_STREAM); - } - - // 如果没压缩，就直接读这块 - // 否则需要解压下 - if (head.type == StreamHead::UNCOMPRESSED) { - _uncompressed = slice; - } else { - _uncompressed = StorageByteBuffer::create(_compress_buffer_size); - - if (OLAP_UNLIKELY(nullptr == _uncompressed)) { - res = Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } else { - res = _decompressor(slice, _uncompressed); - } - - // 一定要释放掉slice - SAFE_DELETE(slice); - } - } else { - LOG(WARNING) << "compressed remaining size less than stream head size. " - "[compressed_remaining_size=" - << _compressed->remaining() << " stream_head_size=" << sizeof(StreamHead) - << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_READ_STREAM); - } - - return res; -} - -uint64_t InStream::available() { - Status res; - res = _assure_data(); - - if (!res.ok()) { - return 0; - } - - return _uncompressed->remaining(); -} - -// seek的是解压前的数据。 -Status InStream::_seek(uint64_t position) { - for (uint32_t i = 0; i < _inputs.size(); i++) { - if (_offsets[i] <= position && position - _offsets[i] < _inputs[i]->remaining()) { - // don't need to malloc _compressed if current range don't be changed. - if (!(_current_range == i && nullptr != _compressed)) { - _current_range = i; - SAFE_DELETE(_compressed); - _compressed = - StorageByteBuffer::reference_buffer(_inputs[i], 0, _inputs[i]->remaining()); - } - - uint64_t pos = _inputs[i]->position() + position - _offsets[i]; - _compressed->set_position(pos); - _current_offset = position; - return Status::OK(); - } - } - - SAFE_DELETE(_compressed); - - if (!_inputs.empty() && position == _offsets.back() + _inputs.back()->remaining()) { - _current_range = _inputs.size() - 1; - _compressed = StorageByteBuffer::reference_buffer(_inputs[_current_range], 0, - _inputs[_current_range]->limit()); - _current_offset = position; - return Status::OK(); - } - - return Status::OLAPInternalError(OLAP_ERR_OUT_OF_BOUND); -} - -Status InStream::seek(PositionProvider* position) { - Status res = Status::OK(); - - // 先seek到解压前的位置，也就是writer中写入的spilled byte - res = _seek(position->get_next()); - if (!res.ok()) { - LOG(WARNING) << "fail to seek.res = " << res; - return res; - } - - // 再定位到已经解压的内存中的位置 - long uncompressed_bytes = position->get_next(); - - if (uncompressed_bytes != 0) { - SAFE_DELETE(_uncompressed); - res = _assure_data(); - - if (!res.ok()) { - LOG(WARNING) << "fail to assure data.res = " << res; - return res; - } - - res = _uncompressed->set_position(uncompressed_bytes); - - if (!res.ok()) { - LOG(WARNING) << "fail to set position. res= " << res - << ", position=" << (_uncompressed->position() + uncompressed_bytes); - return res; - } - } else if (_uncompressed != nullptr) { - // mark the uncompressed buffer as done - res = _uncompressed->set_position(_uncompressed->limit()); - - if (!res.ok()) { - LOG(WARNING) << "fail to set position.res=" << res - << ", position=" << _uncompressed->limit(); - return res; - } - } - - return Status::OK(); -} - -// skip的是解压后的数据 -Status InStream::skip(uint64_t skip_length) { - Status res = _assure_data(); - - if (!res.ok()) { - return res; - } - - uint64_t skip_byte = 0; - uint64_t byte_to_skip = skip_length; - - // 如果不够跳，则先尝试跳过整个数据块，直到当前的数据块剩下的字节数足够跳过 - // 又或者是EOF - do { - skip_byte = std::min(_uncompressed->remaining(), byte_to_skip); - _uncompressed->set_position(_uncompressed->position() + skip_byte); - - byte_to_skip -= skip_byte; - // 如果跳到当前块的尽头，那么assure可以换到下一个块。 - // 如果当前块就可以满足skip_length，那么_assure_data没任何作用。 - res = _assure_data(); - // while 放下边，通常会少一次判断 - } while (byte_to_skip != 0 && res.ok()); - - return res; -} - -} // namespace doris diff --git a/be/src/olap/in_stream.h b/be/src/olap/in_stream.h deleted file mode 100644 index 6e054c9e1a..0000000000 --- a/be/src/olap/in_stream.h +++ /dev/null @@ -1,191 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include -#include -#include -#include - -#include "olap/byte_buffer.h" -#include "olap/compress.h" -#include "olap/olap_common.h" -#include "olap/stream_index_reader.h" - -namespace doris { - -// 提供Column Reader的Seek位置, 由于ColumnReader的seek需要多个position地址 -// PositionProvider 提供了next方法, 将对position的操作封装为stack的形式 - -// 定义输入数据流接口 -class InStream { -public: - // 构造方法, 使用一组ByteBuffer创建一个InStream - // 输入的ByteBuffer在流中的位置可以不连续,例如 - // 通过Index确定某些数据不需要 - // 读取后,则不读入这部分的数据. 但InStream封装 - // 了ByteBuffer不连续这一事实, - // 从上层使用者来看,依旧是在访问一段连续的流. - // 上层使用者应该保证不读取StorageByteBuffer - // 之间没有数据的空洞位置. - // - // 当使用mmap的时候,这里会退化为只有一个ByteBuffer, 是 - // ??使用mmap取决于在性能 - // 调优阶段的测试结果 - // - // Input: - // inputs - 一组ByteBuffer保存具体的流中的数据 - // offsets - input中每个ByteBuffer的数据在流中的偏移位置 - // length - 流的总字节长度 - // Decompressor - 如果流被压缩过,则提供一个解压缩函数,否则为NULL - // compress_buffer_size - 如果使用压缩,给出压缩的块大小 - explicit InStream(std::vector* inputs, const std::vector& offsets, - uint64_t length, Decompressor decompressor, uint32_t compress_buffer_size); - - ~InStream(); - - // 从数据流中读取一个字节,内部指针后移 - // 如果数据流结束, 返回Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_EOF) - Status read(char* byte); - - // 从数据流读入一段数据 - // Input: - // buffer - 存储读入的数据 - // buf_size - 输入时给出buffer的大小,返回时给出实际读取的字节数 - // 如果数据流结束, 返回Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_EOF) - Status read(char* buffer, uint64_t* buf_size); - - // 设置读取的位置 - Status seek(PositionProvider* position); - - // 跳过指定size的流 - Status skip(uint64_t skip_length); - - // 返回流的總長度 - uint64_t stream_length() { - uint64_t length = 0; - - for (size_t buffer_index = 0; buffer_index < _inputs.size(); ++buffer_index) { - length += _inputs[buffer_index]->limit(); - } - - return length; - } - - uint64_t estimate_uncompressed_length() { return _inputs.size() * _compress_buffer_size; } - - bool eof() { return _current_offset == _length; } - - // 返回当前块剩余可读字节数 - uint64_t available(); - - // 返回当前块剩余的内存 - const char* available_buffer() { - if (_assure_data()) { - size_t offset = _uncompressed->position(); - return _uncompressed->array(offset); - } - - return nullptr; - } - -private: - Status _assure_data(); - Status _slice(uint64_t chunk_size, StorageByteBuffer** out_slice); - Status _seek(uint64_t position); - - std::vector _inputs; - std::vector _offsets; - uint64_t _length; - Decompressor _decompressor; - uint32_t _compress_buffer_size; - uint64_t _current_offset; - uint64_t _current_range; - StorageByteBuffer* _compressed; - StorageByteBuffer* _uncompressed; - - DISALLOW_COPY_AND_ASSIGN(InStream); -}; - -// byte buffer的封装， 用于流式读取（暂时用于支持pb的流式反序列化） -// 其实也可以直接和instream合在一起，先这么写着 -class InStreamBufferWrapper : public std::streambuf { -public: - InStreamBufferWrapper(InStream* input) : std::streambuf(), _stream(input), _skip_size(0) {} - virtual ~InStreamBufferWrapper() {} - virtual int_type underflow() { - if (nullptr != _stream) { - if (_stream->skip(_skip_size)) { - char* buf = const_cast(_stream->available_buffer()); - - if (nullptr != buf) { - size_t read_length = _stream->available(); - setg(buf, buf, buf + read_length); - _skip_size = read_length; - return traits_type::to_int_type(*gptr()); - } - } - } - - return traits_type::eof(); - } - -protected: - InStream* _stream; - size_t _skip_size; -}; - -inline Status InStream::read(char* byte) { - Status res; - - if (!(res = _assure_data())) { - return res; - } - - return _uncompressed->get(byte); -} - -inline Status InStream::read(char* buffer, uint64_t* buf_size) { - Status res; - - uint64_t read_length = *buf_size; - *buf_size = 0; - - do { - res = _assure_data(); - if (!res.ok()) { - break; - } - - uint64_t actual_length = std::min(read_length - *buf_size, _uncompressed->remaining()); - - res = _uncompressed->get(buffer, actual_length); - if (!res.ok()) { - break; - } - - *buf_size += actual_length; - buffer += actual_length; - } while (*buf_size < read_length); - - return res; -} - -} // namespace doris diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp index 19d8b5d4e9..146ab34c4e 100644 --- a/be/src/olap/lru_cache.cpp +++ b/be/src/olap/lru_cache.cpp @@ -14,7 +14,6 @@ #include "gutil/bits.h" #include "olap/olap_common.h" #include "olap/olap_define.h" -#include "olap/olap_index.h" #include "olap/row_block.h" #include "olap/utils.h" #include "runtime/thread_context.h" diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 1b44d0ee25..227312d421 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -19,7 +19,6 @@ #include "common/logging.h" #include "olap/row.h" -#include "olap/rowset/column_data_writer.h" #include "olap/rowset/rowset_writer.h" #include "olap/schema.h" #include "runtime/tuple.h" diff --git a/be/src/olap/null_predicate.cpp b/be/src/olap/null_predicate.cpp index bdafda6ee7..d755fd34eb 100644 --- a/be/src/olap/null_predicate.cpp +++ b/be/src/olap/null_predicate.cpp @@ -19,7 +19,6 @@ #include "olap/field.h" #include "runtime/string_value.hpp" -#include "runtime/vectorized_row_batch.h" #include "vec/columns/column_nullable.h" using namespace doris::vectorized; @@ -33,43 +32,6 @@ PredicateType NullPredicate::type() const { return _is_null ? PredicateType::IS_NULL : PredicateType::IS_NOT_NULL; } -void NullPredicate::evaluate(VectorizedRowBatch* batch) const { - uint16_t n = batch->size(); - if (n == 0) { - return; - } - uint16_t* sel = batch->selected(); - bool* null_array = batch->column(_column_id)->is_null(); - if (batch->column(_column_id)->no_nulls() && _is_null) { - batch->set_size(0); - batch->set_selected_in_use(true); - return; - } - - if (batch->column(_column_id)->no_nulls() && !_is_null) { - return; - } - - uint16_t new_size = 0; - if (batch->selected_in_use()) { - for (uint16_t j = 0; j != n; ++j) { - uint16_t i = sel[j]; - sel[new_size] = i; - new_size += (null_array[i] == _is_null); - } - batch->set_size(new_size); - } else { - for (uint16_t i = 0; i != n; ++i) { - sel[new_size] = i; - new_size += (null_array[i] == _is_null); - } - if (new_size < n) { - batch->set_size(new_size); - batch->set_selected_in_use(true); - } - } -} - void NullPredicate::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const { uint16_t new_size = 0; if (!block->is_nullable() && _is_null) { diff --git a/be/src/olap/null_predicate.h b/be/src/olap/null_predicate.h index c1eee63268..caf05af16d 100644 --- a/be/src/olap/null_predicate.h +++ b/be/src/olap/null_predicate.h @@ -25,16 +25,12 @@ namespace doris { -class VectorizedRowBatch; - class NullPredicate : public ColumnPredicate { public: NullPredicate(uint32_t column_id, bool is_null, bool opposite = false); PredicateType type() const override; - void evaluate(VectorizedRowBatch* batch) const override; - void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const override; void evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const override; diff --git a/be/src/olap/olap_index.cpp b/be/src/olap/olap_index.cpp deleted file mode 100644 index 02d8a7d4eb..0000000000 --- a/be/src/olap/olap_index.cpp +++ /dev/null @@ -1,525 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/olap_index.h" - -#include -#include -#include -#include - -#include "olap/row_block.h" -#include "olap/row_cursor.h" -#include "olap/rowset/column_data.h" -#include "olap/utils.h" -#include "olap/wrapper_field.h" - -using std::ifstream; -using std::string; -using std::vector; - -namespace doris { - -MemIndex::MemIndex() - : _key_length(0), - _num_entries(0), - _index_size(0), - _data_size(0), - _num_rows(0), - _mem_pool(new MemPool("MemIndex")) {} - -MemIndex::~MemIndex() { - _num_entries = 0; - for (vector::iterator it = _meta.begin(); it != _meta.end(); ++it) { - free(it->buffer.data); - it->buffer.data = nullptr; - it->buffer.length = 0; - } -} - -Status MemIndex::load_segment(const char* file, size_t* current_num_rows_per_row_block, - bool use_cache) { - Status res = Status::OK(); - - SegmentMetaInfo meta; - uint32_t adler_checksum = 0; - uint32_t num_entries = 0; - - if (file == nullptr) { - res = Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - LOG(WARNING) << "load index error. file=" << file << ", res=" << res; - return res; - } - - FileHandler file_handler; - if (use_cache) { - if ((res = file_handler.open_with_cache(file, O_RDONLY)) != Status::OK()) { - LOG(WARNING) << "open index error. file=" << file << ", res=" << res; - return res; - } - } else { - if ((res = file_handler.open(file, O_RDONLY)) != Status::OK()) { - LOG(WARNING) << "open index error. file=" << file << ", res=" << res; - return res; - } - } - - if ((res = meta.file_header.unserialize(&file_handler)) != Status::OK()) { - LOG(WARNING) << "load index error. file=" << file << ", res=" << res; - file_handler.close(); - return res; - } - - // 允许索引内容为空 - // 索引长度必须为索引项长度的整数倍 - size_t storage_length = meta.file_header.file_length() - meta.file_header.size(); - bool null_supported = false; - //null_supported是为了兼容之前没有NULL字节的数据。 - //目前索引里面都加入了NULL的标志位，entry length都算了NULL标志位构成的bytes - //对于没有标志位的索引，读取数据之后需要对每个字段补齐这部分。 - if (!meta.file_header.message().has_null_supported()) { - null_supported = false; - } else { - null_supported = meta.file_header.message().null_supported(); - } - size_t num_short_key_columns = short_key_num(); - bool is_align = false; - if (!null_supported) { - is_align = (0 == storage_length % (entry_length() - num_short_key_columns)); - } else { - is_align = (0 == storage_length % entry_length()); - } - if (!is_align) { - res = Status::OLAPInternalError(OLAP_ERR_INDEX_LOAD_ERROR); - LOG(WARNING) << "load index error. file=" << file << ", res=" << res; - file_handler.close(); - return res; - } - - // calculate the total size of all segments - if (!null_supported) { - _index_size += meta.file_header.file_length() + num_entries * num_short_key_columns; - num_entries = storage_length / (entry_length() - num_short_key_columns); - } else { - _index_size += meta.file_header.file_length(); - num_entries = storage_length / entry_length(); - } - _data_size += meta.file_header.extra().data_length; - _num_rows += meta.file_header.extra().num_rows; - - meta.range.first = _num_entries; - meta.range.last = meta.range.first + num_entries; - _num_entries = meta.range.last; - _meta.push_back(meta); - - (current_num_rows_per_row_block == nullptr || - (*current_num_rows_per_row_block = meta.file_header.message().num_rows_per_block())); - - if (OLAP_UNLIKELY(num_entries == 0)) { - file_handler.close(); - return Status::OK(); - } - - // convert index memory layout for string type - // previous layout is size|data, - // target type is ptr|size, ptr pointer to data - char* storage_data = reinterpret_cast(calloc(storage_length, 1)); - if (storage_data == nullptr) { - res = Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - LOG(WARNING) << "load segment for loading index error. file=" << file << " res=" << res; - file_handler.close(); - return res; - } - - // 读取索引内容 - // 为了启动加速，此处可使用mmap方式。 - if (file_handler.pread(storage_data, storage_length, meta.file_header.size()) != Status::OK()) { - res = Status::OLAPInternalError(OLAP_ERR_IO_ERROR); - LOG(WARNING) << "load segment for loading index error. file=" << file << "res=" << res; - file_handler.close(); - free(storage_data); - return res; - } - - // checksum validation - adler_checksum = olap_adler32(ADLER32_INIT, storage_data, storage_length); - if (adler_checksum != meta.file_header.checksum()) { - res = Status::OLAPInternalError(OLAP_ERR_INDEX_CHECKSUM_ERROR); - LOG(WARNING) << "checksum validation error."; - LOG(WARNING) << "load segment for loading index error. file=" << file << "res=" << res; - file_handler.close(); - free(storage_data); - return res; - } - - /* - * convert storage layout to memory layout for olap/index - * In this procedure, string type(Varchar/Char) should be - * converted with caution. Hyperloglog type will not be - * key, it can not to be handled. - */ - - size_t storage_row_bytes = entry_length(); - storage_row_bytes -= (null_supported ? 0 : num_short_key_columns); - char* storage_ptr = storage_data; - size_t storage_field_offset = 0; - - size_t mem_row_bytes = new_entry_length(); - char* mem_buf = reinterpret_cast(calloc(num_entries * mem_row_bytes, 1)); - memset(mem_buf, 0, num_entries * mem_row_bytes); - char* mem_ptr = mem_buf; - size_t mem_field_offset = 0; - - size_t null_byte = null_supported ? 1 : 0; - for (size_t i = 0; i < num_short_key_columns; ++i) { - const TabletColumn& column = (*_short_key_columns)[i]; - storage_ptr = storage_data + storage_field_offset; - storage_field_offset += column.index_length() + null_byte; - mem_ptr = mem_buf + mem_field_offset; - if (column.type() == OLAP_FIELD_TYPE_VARCHAR) { - mem_field_offset += sizeof(Slice) + 1; - for (size_t j = 0; j < num_entries; ++j) { - /* - * Varchar is null_byte|length|content in OlapIndex storage - * Varchar is in nullbyte|length|ptr in memory - * We need copy three part: nullbyte|length|content - * 1. copy null byte - * 2. copy length and content into addrs pointed by ptr - */ - - // 1. copy null_byte - memory_copy(mem_ptr, storage_ptr, null_byte); - - // 2. copy length and content - bool is_null = *reinterpret_cast(mem_ptr); - if (!is_null) { - size_t storage_field_bytes = - *reinterpret_cast(storage_ptr + null_byte); - Slice* slice = reinterpret_cast(mem_ptr + 1); - char* data = reinterpret_cast(_mem_pool->allocate(storage_field_bytes)); - memory_copy(data, storage_ptr + sizeof(VarcharLengthType) + null_byte, - storage_field_bytes); - slice->data = data; - slice->size = storage_field_bytes; - } - - mem_ptr += mem_row_bytes; - storage_ptr += storage_row_bytes; - } - } else if (column.type() == OLAP_FIELD_TYPE_STRING) { - mem_field_offset += sizeof(Slice) + 1; - for (size_t j = 0; j < num_entries; ++j) { - /* - * string is null_byte|length|content in OlapIndex storage - * string is in nullbyte|length|ptr in memory - * We need copy three part: nullbyte|length|content - * 1. copy null byte - * 2. copy length and content into addrs pointed by ptr - */ - - // 1. copy null_byte - memory_copy(mem_ptr, storage_ptr, null_byte); - - // 2. copy length and content - bool is_null = *reinterpret_cast(mem_ptr); - if (!is_null) { - size_t storage_field_bytes = - *reinterpret_cast(storage_ptr + null_byte); - Slice* slice = reinterpret_cast(mem_ptr + 1); - char* data = reinterpret_cast(_mem_pool->allocate(storage_field_bytes)); - memory_copy(data, storage_ptr + sizeof(StringLengthType) + null_byte, - storage_field_bytes); - slice->data = data; - slice->size = storage_field_bytes; - } - - mem_ptr += mem_row_bytes; - storage_ptr += storage_row_bytes; - } - } else if (column.type() == OLAP_FIELD_TYPE_CHAR) { - mem_field_offset += sizeof(Slice) + 1; - size_t storage_field_bytes = column.index_length(); - for (size_t j = 0; j < num_entries; ++j) { - /* - * Char is in nullbyte|content with fixed length in OlapIndex - * Char is in nullbyte|length|ptr in memory - * We need copy three part: nullbyte|length|content - * 1. copy null byte - * 2. copy length and content into addrs pointed by ptr - */ - - // 1. copy null_byte - memory_copy(mem_ptr, storage_ptr, null_byte); - - // 2. copy length and content - bool is_null = *reinterpret_cast(mem_ptr); - if (!is_null) { - Slice* slice = reinterpret_cast(mem_ptr + 1); - char* data = reinterpret_cast(_mem_pool->allocate(storage_field_bytes)); - memory_copy(data, storage_ptr + null_byte, storage_field_bytes); - slice->data = data; - slice->size = storage_field_bytes; - } - - mem_ptr += mem_row_bytes; - storage_ptr += storage_row_bytes; - } - } else { - size_t storage_field_bytes = column.index_length(); - mem_field_offset += storage_field_bytes + 1; - for (size_t j = 0; j < num_entries; ++j) { - // 1. copy null_byte - memory_copy(mem_ptr, storage_ptr, null_byte); - - // 2. copy content - bool is_null = *reinterpret_cast(mem_ptr); - if (!is_null) { - memory_copy(mem_ptr + 1, storage_ptr + null_byte, storage_field_bytes); - } - - mem_ptr += mem_row_bytes; - storage_ptr += storage_row_bytes; - } - } - } - - mem_ptr = mem_buf + mem_field_offset; - storage_ptr = storage_data + storage_field_offset; - size_t data_file_offset = sizeof(data_file_offset_t); - for (size_t j = 0; j < num_entries; ++j) { - memory_copy(mem_ptr, storage_ptr, data_file_offset); - mem_ptr += mem_row_bytes; - storage_ptr += storage_row_bytes; - } - - _meta.back().buffer.data = mem_buf; - _meta.back().buffer.length = num_entries * mem_row_bytes; - free(storage_data); - - file_handler.close(); - return Status::OK(); -} - -Status MemIndex::init(size_t short_key_len, size_t new_short_key_len, size_t short_key_num, - std::vector* short_key_columns) { - if (short_key_columns == nullptr) { - LOG(WARNING) << "fail to init MemIndex, nullptr short key columns."; - return Status::OLAPInternalError(OLAP_ERR_INDEX_LOAD_ERROR); - } - - _key_length = short_key_len; - _new_key_length = new_short_key_len; - _key_num = short_key_num; - _short_key_columns = short_key_columns; - - return Status::OK(); -} - -// Find and return the IndexOffset of the element prior to the first element which -// is key's lower_bound, or upper_bound if key exists, or return the last element in MemIndex -// This process is consists of two phases of binary search. -// Here try to find the first segment which range covers k, -// and find the index item inside the segment previously found. -// -// There 're a little more detail of the binary search. -// The semantics here is to guarantee there's no -// omissions for given k, consider the following case: -// [4, offset] ---------------------> [(4, xxxx), (4, xxxy), (5, xxxx), (5, xxxy)] -// [5, offset] ---------------------> [(5, yyyy), (5, yyyx), (6, ...)] -// because of our sparse index, the first item which short key equals 5(5, xxxx) is indexed -// by shortkey 4 in the first index item, if we want to find the first key not less than 6, we -// should return the first index instead the second. -const OLAPIndexOffset MemIndex::find(const RowCursor& k, RowCursor* helper_cursor, - bool find_last) const { - if (begin() == end()) { - return begin(); - } - - OLAPIndexOffset offset; - BinarySearchIterator it; - BinarySearchIterator seg_beg(0); - BinarySearchIterator seg_fin(segment_count()); - - try { - SegmentComparator seg_comparator(this, helper_cursor); - - // first step, binary search for the correct segment - if (!find_last) { - it = std::lower_bound(seg_beg, seg_fin, k, seg_comparator); - } else { - it = std::upper_bound(seg_beg, seg_fin, k, seg_comparator); - } - - iterator_offset_t off = 0; - if (*it > 0) { - off = *it - 1; - } - - // set segment id - offset.segment = off; - IndexComparator index_comparator(this, helper_cursor); - // second step, binary search index item in given segment - BinarySearchIterator index_beg(0); - BinarySearchIterator index_fin(_meta[off].count()); - - if (index_comparator.set_segment_id(off) != Status::OK()) { - throw "index of range"; - } - - if (!find_last) { - it = std::lower_bound(index_beg, index_fin, k, index_comparator); - } else { - it = std::upper_bound(index_beg, index_fin, k, index_comparator); - } - - offset.offset = *it; - VLOG_NOTICE << "show real offset iterator value. off=" << *it; - VLOG_NOTICE << "show result offset. seg_off=" << offset.segment - << ", off=" << offset.offset; - } catch (...) { - LOG(WARNING) << "fail to compare value in memindex. [cursor='" << k.to_string() - << "' find_last=" << find_last << "]"; - return end(); - } - - return offset; -} - -const OLAPIndexOffset MemIndex::next(const OLAPIndexOffset& pos) const { - OLAPIndexOffset off; - - if (pos.segment >= segment_count()) { - return end(); - } else if (pos.segment == segment_count() - 1) { - if (pos.offset >= _meta[pos.segment].count() - 1) { - return end(); - } else { - off.segment = pos.segment; - off.offset = pos.offset + 1; - return off; - } - } else { - if (pos.offset >= _meta[pos.segment].count() - 1) { - off.segment = pos.segment + 1; - off.offset = 0; - } else { - off.segment = pos.segment; - off.offset = pos.offset + 1; - } - - return off; - } -} - -const OLAPIndexOffset MemIndex::prev(const OLAPIndexOffset& pos) const { - OLAPIndexOffset off; - - if (pos.offset == 0) { - if (pos.segment == 0) { - off = pos; - } else { - off.segment = pos.segment - 1; - off.offset = _meta[off.segment].count() - 1; - } - } else { - off.segment = pos.segment; - off.offset = pos.offset - 1; - } - - return off; -} - -const OLAPIndexOffset MemIndex::get_offset(const RowBlockPosition& pos) const { - uint32_t file_header_size = _meta[pos.segment].file_header.size(); - if (pos.segment >= segment_count() || - pos.index_offset > file_header_size + _meta[pos.segment].buffer.length || - (pos.index_offset - file_header_size) % new_entry_length() != 0) { - return end(); - } - - OLAPIndexOffset off; - off.segment = pos.segment; - off.offset = (pos.index_offset - _meta[pos.segment].file_header.size()) / new_entry_length(); - - return off; -} - -Status MemIndex::get_entry(const OLAPIndexOffset& pos, EntrySlice* slice) const { - if (pos.segment >= segment_count() || pos.offset >= _meta[pos.segment].count()) { - return Status::OLAPInternalError(OLAP_ERR_INDEX_EOF); - } - - slice->length = new_entry_length(); - slice->data = _meta[pos.segment].buffer.data + pos.offset * new_entry_length(); - - return Status::OK(); -} - -Status MemIndex::get_row_block_position(const OLAPIndexOffset& pos, RowBlockPosition* rbp) const { - if (zero_num_rows()) { - return Status::OLAPInternalError(OLAP_ERR_INDEX_EOF); - } - - if (pos.segment >= segment_count() || pos.offset >= _meta[pos.segment].count()) { - LOG(WARNING) << "fail to get RowBlockPosition from OLAPIndexOffset. " - "[IndexOffset={segment=" - << pos.segment << " offset=" << pos.offset - << "} segment_count=" << segment_count() << " items_count=" - << (pos.segment < segment_count() ? _meta[pos.segment].count() : 0) << "]"; - return Status::OLAPInternalError(OLAP_ERR_INDEX_EOF); - } - - rbp->segment = pos.segment; - rbp->data_offset = - *reinterpret_cast(_meta[pos.segment].buffer.data + - pos.offset * new_entry_length() + new_short_key_length()); - rbp->index_offset = _meta[pos.segment].file_header.size() + pos.offset * new_entry_length(); - - if (pos.offset == _meta[pos.segment].count() - 1) { - rbp->block_size = _meta[pos.segment].file_header.extra().data_length - rbp->data_offset; - } else { - uint32_t next_offset = *reinterpret_cast(_meta[pos.segment].buffer.data + - (pos.offset + 1) * new_entry_length() + - new_short_key_length()); - rbp->block_size = next_offset - rbp->data_offset; - } - - return Status::OK(); -} - -const OLAPIndexOffset MemIndex::get_relative_offset(iterator_offset_t absolute_offset) const { - int begin = 0; - int end = segment_count() - 1; - OLAPIndexOffset offset(0, 0); - - while (begin <= end) { - size_t mid = (begin + end) / 2; - if (absolute_offset >= _meta[mid].range.last) { - begin = mid + 1; - } else if (absolute_offset < _meta[mid].range.first) { - end = mid - 1; - } else { - offset.segment = mid; - break; - } - } - - // 这里不考虑没有找到的情况 - offset.offset = absolute_offset - _meta[offset.segment].range.first; - return offset; -} -} // namespace doris diff --git a/be/src/olap/olap_index.h b/be/src/olap/olap_index.h deleted file mode 100644 index d7a3deaa33..0000000000 --- a/be/src/olap/olap_index.h +++ /dev/null @@ -1,380 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "gen_cpp/column_data_file.pb.h" -#include "gen_cpp/olap_file.pb.h" -#include "olap/field.h" -#include "olap/file_helper.h" -#include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/row.h" -#include "olap/row_cursor.h" -#include "olap/utils.h" - -namespace doris { -class IndexComparator; -class SegmentGroup; -class RowBlock; -class RowCursor; -class SegmentComparator; -class WrapperField; - -typedef uint32_t data_file_offset_t; - -struct OLAPIndexFixedHeader { - uint32_t data_length; - uint64_t num_rows; -}; - -struct EntrySlice { - char* data; - size_t length; - EntrySlice() : data(nullptr), length(0) {} -}; - -// Range of offset in one segment -struct IDRange { - uint32_t first; - uint32_t last; -}; - -// index offset -// 2-Dimension Offset, the first is segment id, the second is offset inside segment -struct OLAPIndexOffset { - OLAPIndexOffset() : segment(0), offset(0) {} - OLAPIndexOffset(const iterator_offset_t& seg, const iterator_offset_t& off) - : segment(seg), offset(off) {} - OLAPIndexOffset(const OLAPIndexOffset& off) : segment(off.segment), offset(off.offset) {} - - bool operator==(const OLAPIndexOffset& other) const { - return segment == other.segment && offset == other.offset; - } - - iterator_offset_t segment; - iterator_offset_t offset; -}; - -// 唯一标识一个RowBlock在Data文件和Index文件中的位置 -struct RowBlockPosition { - RowBlockPosition() : segment(0), block_size(0), data_offset(0), index_offset(0) {} - - bool operator==(const RowBlockPosition& other) const { - return (segment == other.segment && data_offset == other.data_offset && - block_size == other.block_size && index_offset == other.index_offset); - } - - bool operator!=(const RowBlockPosition& other) const { return !(*this == other); } - - bool operator>(const RowBlockPosition& other) const { - if (segment < other.segment) { - return false; - } else if (segment > other.segment) { - return true; - } else { - if (data_offset > other.data_offset) { - return true; - } else { - return false; - } - } - } - - bool operator>=(const RowBlockPosition& other) const { - if (segment < other.segment) { - return false; - } else if (segment > other.segment) { - return true; - } else { - if (data_offset >= other.data_offset) { - return true; - } else { - return false; - } - } - } - - // 供日志输出 - std::string to_string() const { - char message[1024] = {'\0'}; - snprintf(message, sizeof(message), - "{segment=%u block_size=%u data_offset=%u index_offset=%u}", segment, block_size, - data_offset, index_offset); - return std::string(message); - } - - uint32_t segment; - uint32_t block_size; - uint32_t data_offset; // offset in data file - uint32_t index_offset; // offset in index file -}; - -// In memory presentation of index meta information -struct SegmentMetaInfo { - SegmentMetaInfo() { - range.first = range.last = 0; - buffer.length = 0; - buffer.data = nullptr; - } - - const size_t count() const { return range.last - range.first; } - - IDRange range; - EntrySlice buffer; - FileHeader file_header; -}; - -// In memory index structure, all index hold here -class MemIndex { -public: - friend class SegmentGroup; - friend class IndexComparator; - friend class SegmentComparator; - - MemIndex(); - ~MemIndex(); - - // 初始化MemIndex, 传入short_key的总长度和对应的Field数组 - Status init(size_t short_key_len, size_t new_short_key_len, size_t short_key_num, - std::vector* short_key_columns); - - // 加载一个segment到内存 - Status load_segment(const char* file, size_t* current_num_rows_per_row_block, - bool use_cache = true); - - // Return the IndexOffset of the first element, physically, it's (0, 0) - const OLAPIndexOffset begin() const { - OLAPIndexOffset off; - off.segment = off.offset = 0; - return off; - } - - // Indicates a logical IndexOffset beyond MemIndex - const OLAPIndexOffset end() const { - OLAPIndexOffset off; - off.segment = segment_count(); - off.offset = _num_entries; - return off; - } - - // Return the IndexOffset of position prior to the first element which is either - // not less than(find_last is false) or greater than (find_last is true) 'key'; - // or, return the IndexOffset of last element inside MemIndex - // in case of nothing matched with 'key' - const OLAPIndexOffset find(const RowCursor& key, RowCursor* helper_cursor, - bool find_last) const; - - // Same with begin() - const OLAPIndexOffset find_first() const { return begin(); } - - // Return IndexOffset of last element if exists, or return (0, 0) if MemIndex is empty() - const OLAPIndexOffset find_last() const { - if (_num_entries == 0 || segment_count() == 0) { - return end(); - } else { - OLAPIndexOffset off; - off.segment = segment_count() - 1; - off.offset = _meta[segment_count() - 1].count() - 1; - return off; - } - } - - // Return IndexOffset of next element - const OLAPIndexOffset next(const OLAPIndexOffset& pos) const; - // Return IndexOffset of prev element - const OLAPIndexOffset prev(const OLAPIndexOffset& pos) const; - // Calculate IndexOffset from RowBlockPosition - const OLAPIndexOffset get_offset(const RowBlockPosition& pos) const; - - // Return the 1-dimension, plain offset from IndexOffset - // For example, there're 1000 index items, divided into 10 segments, 100 items in each segment, - // the 2-dimension offset of the first element of first segment is (0, 0), - // it's plain offset is 0, - // the 2-dimension offset of the first element of second segment is (1, 0), - // it's plain offset is 100 - const iterator_offset_t get_absolute_offset(const OLAPIndexOffset& offset) const { - if (offset.segment >= segment_count() || offset.offset >= _meta[offset.segment].count()) { - return _num_entries; - } else { - return _meta[offset.segment].range.first + offset.offset; - } - } - - // Return the 2-dimension, logical Offset from plain offset - const OLAPIndexOffset get_relative_offset(iterator_offset_t absolute_offset) const; - - // Return content of index item, which IndexOffset is pos - Status get_entry(const OLAPIndexOffset& pos, EntrySlice* slice) const; - - // Return RowBlockPosition from IndexOffset - Status get_row_block_position(const OLAPIndexOffset& pos, RowBlockPosition* rbp) const; - - const size_t short_key_num() const { return _key_num; } - - // Return length of short keys in bytes, for example, there're two short key columns: - // uint32_t/uint64_t the length is sizeof(uint32_t) + sizeof(uint64_t) - const size_t short_key_length() const { return _key_length; } - - const size_t new_short_key_length() const { return _new_key_length; } - - // Return length of full index item, - // which actually equals to short_key_length() plus sizeof(data_file_offset_t) - const size_t entry_length() const { return short_key_length() + sizeof(data_file_offset_t); } - - const size_t new_entry_length() const { return _new_key_length + sizeof(data_file_offset_t); } - - // Return short key FieldInfo array - const std::vector& short_key_columns() const { return *_short_key_columns; } - - // Return the number of indices in MemIndex - size_t count() const { return _num_entries; } - - // Return the number of segments in MemIndex - size_t segment_count() const { return _meta.size(); } - - bool zero_num_rows() const { return _num_entries == 0; } - - const size_t index_size() const { return _index_size; }; - - const size_t data_size() const { return _data_size; }; - - const size_t num_rows() const { return _num_rows; } - - bool delete_flag() const { - if (_meta[0].file_header.message().has_delete_flag()) { - return _meta[0].file_header.message().delete_flag(); - } else { - return false; - } - } - - bool get_null_supported(uint32_t seg_id) { - if (false == _meta[seg_id].file_header.message().has_null_supported()) { - return false; - } else { - return _meta[seg_id].file_header.message().null_supported(); - } - } - -private: - std::vector _meta; - size_t _key_length; - size_t _new_key_length; - size_t _key_num; - size_t _num_entries; - size_t _index_size; - size_t _data_size; - size_t _num_rows; - std::vector* _short_key_columns; - - std::unique_ptr _mem_pool; - DISALLOW_COPY_AND_ASSIGN(MemIndex); -}; - -// 在同一个Segment内进行二分查找的比较类 -class IndexComparator { -public: - IndexComparator(const MemIndex* index, RowCursor* cursor) - : _index(index), _cur_seg(0), _helper_cursor(cursor) {} - - // Destructor do nothing - ~IndexComparator() {} - - bool operator()(const iterator_offset_t& index, const RowCursor& key) { - return _compare(index, key, COMPARATOR_LESS); - } - - bool operator()(const RowCursor& key, const iterator_offset_t& index) { - return _compare(index, key, COMPARATOR_LARGER); - } - - Status set_segment_id(const iterator_offset_t& segment_id) { - if (segment_id >= _index->segment_count()) { - return Status::OLAPInternalError(OLAP_ERR_INDEX_EOF); - } - - _cur_seg = segment_id; - return Status::OK(); - } - -private: - bool _compare(const iterator_offset_t& index, const RowCursor& key, ComparatorEnum comparator) { - EntrySlice slice; - OLAPIndexOffset offset(_cur_seg, index); - _index->get_entry(offset, &slice); - - _helper_cursor->attach(slice.data); - - if (comparator == COMPARATOR_LESS) { - return index_compare_row(*_helper_cursor, key) < 0; - } else { - return index_compare_row(*_helper_cursor, key) > 0; - } - } - - const MemIndex* _index; - iterator_offset_t _cur_seg; - RowCursor* _helper_cursor; -}; - -// 用于寻找索引所在的Segment的比较类 -class SegmentComparator { -public: - SegmentComparator(const MemIndex* index, RowCursor* cursor) - : _index(index), _helper_cursor(cursor) {} - - // Destructor do nothing - ~SegmentComparator() {} - - bool operator()(const iterator_offset_t index, const RowCursor& key) { - return _compare(index, key, COMPARATOR_LESS); - } - - bool operator()(const RowCursor& key, iterator_offset_t index) { - return _compare(index, key, COMPARATOR_LARGER); - } - -private: - bool _compare(const iterator_offset_t& index, const RowCursor& key, ComparatorEnum comparator) { - EntrySlice slice; - slice.data = _index->_meta[index].buffer.data; - //slice.length = _index->short_key_length(); - slice.length = _index->new_short_key_length(); - - _helper_cursor->attach(slice.data); - - if (comparator == COMPARATOR_LESS) { - return index_compare_row(*_helper_cursor, key) < 0; - } else { - return index_compare_row(*_helper_cursor, key) > 0; - } - } - - const MemIndex* _index; - RowCursor* _helper_cursor; -}; - -} // namespace doris diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index e08ef5ca5b..2b893cac75 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -28,7 +28,6 @@ #include "agent/cgroups_mgr.h" #include "common/status.h" #include "gutil/strings/substitute.h" -#include "olap/convert_rowset.h" #include "olap/cumulative_compaction.h" #include "olap/olap_common.h" #include "olap/olap_define.h" @@ -78,21 +77,6 @@ Status StorageEngine::start_bg_threads() { .set_max_threads(max_thread_num) .build(&_cumu_compaction_thread_pool); - int32_t convert_rowset_thread_num = config::convert_rowset_thread_num; - if (convert_rowset_thread_num > 0) { - ThreadPoolBuilder("ConvertRowsetTaskThreadPool") - .set_min_threads(convert_rowset_thread_num) - .set_max_threads(convert_rowset_thread_num) - .build(&_convert_rowset_thread_pool); - - // alpha rowset scan thread - RETURN_IF_ERROR(Thread::create( - "StorageEngine", "alpha_rowset_scan_thread", - [this]() { this->_alpha_rowset_scan_thread_callback(); }, - &_alpha_rowset_scan_thread)); - LOG(INFO) << "alpha rowset scan thread started"; - } - ThreadPoolBuilder("BaseCompactionTaskThreadPool") .set_min_threads(config::max_base_compaction_threads) .set_max_threads(config::max_base_compaction_threads) @@ -338,40 +322,6 @@ void StorageEngine::_tablet_checkpoint_callback(const std::vector& dat } while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval))); } -void StorageEngine::_alpha_rowset_scan_thread_callback() { - LOG(INFO) << "try to start alpha rowset scan thread!"; - - auto scan_interval_sec = config::scan_alpha_rowset_min_interval_sec; - auto max_convert_task = config::convert_rowset_thread_num * 2; - do { - std::vector tablet_have_alpha_rowset; - _tablet_manager->find_tablet_have_alpha_rowset(tablet_have_alpha_rowset); - - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(tablet_have_alpha_rowset.begin(), tablet_have_alpha_rowset.end(), g); - - for (int i = 0; i < max_convert_task && i < tablet_have_alpha_rowset.size(); ++i) { - auto tablet = tablet_have_alpha_rowset[i]; - auto st = _convert_rowset_thread_pool->submit_func([=]() { - CgroupsMgr::apply_system_cgroup(); - auto convert_rowset = std::make_shared(tablet); - convert_rowset->do_convert(); - }); - if (!st.ok()) { - LOG(WARNING) << "submit convert tablet tasks failed."; - } - } - - if (tablet_have_alpha_rowset.size() == 0) { - scan_interval_sec = std::min(3600, scan_interval_sec * 2); - } else { - _convert_rowset_thread_pool->wait(); - scan_interval_sec = config::scan_alpha_rowset_min_interval_sec; - } - } while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(scan_interval_sec))); -} - void StorageEngine::_adjust_compaction_thread_num() { if (_base_compaction_thread_pool->max_threads() != config::max_base_compaction_threads) { int old_max_threads = _base_compaction_thread_pool->max_threads(); diff --git a/be/src/olap/out_stream.cpp b/be/src/olap/out_stream.cpp deleted file mode 100644 index 3cca7c3803..0000000000 --- a/be/src/olap/out_stream.cpp +++ /dev/null @@ -1,401 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/out_stream.h" - -#include "olap/byte_buffer.h" -#include "olap/file_helper.h" -#include "olap/utils.h" -#include "util/mem_util.hpp" - -namespace doris { - -OutStreamFactory::OutStreamFactory(CompressKind compress_kind, uint32_t stream_buffer_size) - : _stream_buffer_size(stream_buffer_size) { - switch (compress_kind) { - case COMPRESS_NONE: - _compressor = nullptr; - break; - -#ifdef DORIS_WITH_LZO - case COMPRESS_LZO: - _compressor = lzo_compress; - break; -#endif - - case COMPRESS_LZ4: - _compressor = lz4_compress; - break; - - default: - LOG(FATAL) << "unknown compress kind. kind=" << compress_kind; - } -} - -OutStreamFactory::~OutStreamFactory() { - for (std::map::iterator it = _streams.begin(); it != _streams.end(); - ++it) { - SAFE_DELETE(it->second); - } -} - -OutStream* OutStreamFactory::create_stream(uint32_t column_unique_id, - StreamInfoMessage::Kind kind) { - OutStream* stream = nullptr; - - if (StreamInfoMessage::ROW_INDEX == kind || StreamInfoMessage::BLOOM_FILTER == kind) { - stream = new (std::nothrow) OutStream(_stream_buffer_size, nullptr); - } else { - stream = new (std::nothrow) OutStream(_stream_buffer_size, _compressor); - } - - if (nullptr == stream) { - LOG(WARNING) << "fail to allocate OutStream."; - return nullptr; - } - - StreamName stream_name(column_unique_id, kind); - _streams[stream_name] = stream; - return stream; -} - -OutStream::OutStream(uint32_t buffer_size, Compressor compressor) - : _buffer_size(buffer_size), - _compressor(compressor), - _is_suppressed(false), - _current(nullptr), - _compressed(nullptr), - _overflow(nullptr), - _spilled_bytes(0) {} - -OutStream::~OutStream() { - SAFE_DELETE(_current); - SAFE_DELETE(_compressed); - SAFE_DELETE(_overflow); - - for (std::vector::iterator it = _output_buffers.begin(); - it != _output_buffers.end(); ++it) { - SAFE_DELETE(*it); - } -} - -Status OutStream::_create_new_input_buffer() { - SAFE_DELETE(_current); - _current = StorageByteBuffer::create(_buffer_size + sizeof(StreamHead)); - - if (nullptr != _current) { - _current->set_position(sizeof(StreamHead)); - return Status::OK(); - } else { - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } -} - -Status OutStream::_write_head(StorageByteBuffer* buf, uint64_t position, - StreamHead::StreamType type, uint32_t length) { - if (buf->limit() < sizeof(StreamHead) + length) { - return Status::OLAPInternalError(OLAP_ERR_BUFFER_OVERFLOW); - } - - StreamHead* head = reinterpret_cast(&(buf->array()[position])); - head->type = type; - head->length = length; - head->checksum = 0; - return Status::OK(); -} - -Status OutStream::_compress(StorageByteBuffer* input, StorageByteBuffer* output, - StorageByteBuffer* overflow, bool* smaller) { - Status res = Status::OK(); - - res = _compressor(input, overflow, smaller); - - if (res.ok() && *smaller) { - if (output->remaining() >= overflow->position()) { - memory_copy(&(output->array()[output->position()]), overflow->array(), - overflow->position()); - output->set_position(output->position() + overflow->position()); - overflow->set_position(0); - } else if (0 != output->remaining()) { - uint64_t to_copy = output->remaining(); - memory_copy(&(output->array()[output->position()]), overflow->array(), to_copy); - output->set_position(output->limit()); - - memmove(overflow->array(), &(overflow->array()[to_copy]), - overflow->position() - to_copy); - overflow->set_position(overflow->position() - to_copy); - } - } - - return Status::OK(); -} - -void OutStream::_output_uncompress() { - _spilled_bytes += _current->limit(); - _write_head(_current, 0, StreamHead::UNCOMPRESSED, _current->limit() - sizeof(StreamHead)); - _output_buffers.push_back(_current); - _current = nullptr; -} - -void OutStream::_output_compressed() { - _compressed->flip(); - _output_buffers.push_back(_compressed); - _compressed = _overflow; - _overflow = nullptr; -} - -Status OutStream::_make_sure_output_buffer() { - if (nullptr == _compressed) { - _compressed = StorageByteBuffer::create(_buffer_size + sizeof(StreamHead)); - - if (nullptr == _compressed) { - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - } - - if (nullptr == _overflow) { - _overflow = StorageByteBuffer::create(_buffer_size + sizeof(StreamHead)); - - if (nullptr == _overflow) { - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - } - - return Status::OK(); -} - -Status OutStream::_spill() { - Status res = Status::OK(); - - if (_current == nullptr || _current->position() == sizeof(StreamHead)) { - return Status::OK(); - } - - // If it is not compressed, read current directly. Note that current will be cleared and set to NULL after output - if (_compressor == nullptr) { - _current->flip(); - _output_uncompress(); - } else { - //If compression is required, - // The current moves to the position behind the head, leaving space for the head - _current->set_limit(_current->position()); - _current->set_position(sizeof(StreamHead)); - - //Allocate compress and overflow, the two buffer sizes are actually the same - if (!(res = _make_sure_output_buffer())) { - return res; - } - - // Decompress current to compress and overflow - uint64_t head_pos = _compressed->position(); - _compressed->set_position(head_pos + sizeof(StreamHead)); - bool smaller = false; - res = _compress(_current, _compressed, _overflow, &smaller); - - if (!res.ok()) { - LOG(WARNING) << "fail to compress data."; - return Status::OLAPInternalError(OLAP_ERR_COMPRESS_ERROR); - } - - if (smaller) { - // Data are compressed into _output and _overflow, reset _current - // Note that in this case, current is not released, because the compress actually output - _current->set_position(sizeof(StreamHead)); - _current->set_limit(_current->capacity()); - - uint32_t output_bytes = _compressed->position() - head_pos - sizeof(StreamHead); - output_bytes += _overflow->position(); - _write_head(_compressed, head_pos, StreamHead::COMPRESSED, output_bytes); - - if (_compressed->remaining() < sizeof(StreamHead)) { - _output_compressed(); - } - - _spilled_bytes += sizeof(StreamHead) + output_bytes; - } else { - // directly output _current - // If there is _compress before, output m_compress first - // Note that there must be no _overflow at this time - _compressed->set_position(head_pos); - - if (head_pos != 0) { - // There was data in _compressed before, in this case, output compressed first, - // At this time _overflow must be empty - _output_compressed(); - } - - _output_uncompress(); - } - } - - return Status::OK(); -} - -Status OutStream::write(const char* buffer, uint64_t length) { - Status res = Status::OK(); - uint64_t offset = 0; - uint64_t remain = length; - - while (remain > 0) { - // The reason why it was thrown in is because in the case of compression, _current will only be created once - // It has been multiplexing since then, and the output is compress - // In the case of uncompressed, current will be put into the list and cannot be reused. The reason is - // If it is reused, the previous content will be modified, so it needs to be redistributed. - // Only allocate once and the second block will hang up - if (nullptr == _current) { - res = _create_new_input_buffer(); - if (!res.ok()) { - return res; - } - } - - uint64_t to_put = std::min(_current->remaining(), remain); - - if (OLAP_LIKELY(0 != to_put)) { - res = _current->put(&buffer[offset], to_put); - if (!res.ok()) { - LOG(WARNING) << "fail to put buffer."; - return res; - } - - offset += to_put; - remain -= to_put; - } - - if (_current->remaining() == 0) { - res = _spill(); - if (!res.ok()) { - LOG(WARNING) << "fail to spill current buffer."; - return res; - } - } - } - - return Status::OK(); -} - -void OutStream::get_position(PositionEntryWriter* index_entry) const { - index_entry->add_position(_spilled_bytes); - - if (nullptr != _current) { - index_entry->add_position(_current->position() - sizeof(StreamHead)); - } else { - index_entry->add_position(0); - } -} - -uint64_t OutStream::get_stream_length() const { - uint64_t result = 0; - - for (std::vector::const_iterator it = _output_buffers.begin(); - it != _output_buffers.end(); ++it) { - result += (*it)->limit(); - } - - return result; -} - -uint64_t OutStream::get_total_buffer_size() const { - uint64_t result = 0; - - for (std::vector::const_iterator it = _output_buffers.begin(); - it != _output_buffers.end(); ++it) { - result += (*it)->capacity(); - } - - if (_current) { - result += _current->capacity(); - } - - if (_compressed) { - result += _compressed->capacity(); - } - - if (_overflow) { - result += _overflow->capacity(); - } - - return result; -} - -Status OutStream::write_to_file(FileHandler* file_handle, uint32_t write_mbytes_per_sec) const { - Status res = Status::OK(); - - uint64_t total_stream_len = 0; - OlapStopWatch speed_limit_watch; - - speed_limit_watch.reset(); - - for (std::vector::const_iterator it = _output_buffers.begin(); - it != _output_buffers.end(); ++it) { - VLOG_TRACE << "write stream begin:" << file_handle->tell(); - - res = file_handle->write((*it)->array(), (*it)->limit()); - if (!res.ok()) { - LOG(WARNING) << "fail to write stream to fail."; - return res; - } - - VLOG_TRACE << "write stream end:" << file_handle->tell(); - - total_stream_len += (*it)->limit(); - if (write_mbytes_per_sec > 0) { - uint64_t delta_time_us = speed_limit_watch.get_elapse_time_us(); - int64_t sleep_time = total_stream_len / write_mbytes_per_sec - delta_time_us; - if (sleep_time > 0) { - VLOG_TRACE << "sleep to limit merge speed. time=" << sleep_time - << ", bytes=" << total_stream_len; - std::this_thread::sleep_for(std::chrono::microseconds(sleep_time)); - } - } - } - - return res; -} - -Status OutStream::flush() { - Status res = Status::OK(); - - res = _spill(); - if (!res.ok()) { - LOG(WARNING) << "fail to spill stream."; - return res; - } - - if (nullptr != _compressed && 0 != _compressed->position()) { - _output_compressed(); - SAFE_DELETE(_compressed); - } - - SAFE_DELETE(_current); - SAFE_DELETE(_overflow); - - return res; -} - -uint32_t OutStream::crc32(uint32_t checksum) const { - uint32_t result = CRC32_INIT; - - for (std::vector::const_iterator it = _output_buffers.begin(); - it != _output_buffers.end(); ++it) { - result = olap_crc32(result, (*it)->array(), (*it)->limit()); - } - - return result; -} - -} // namespace doris diff --git a/be/src/olap/out_stream.h b/be/src/olap/out_stream.h deleted file mode 100644 index 50f1914649..0000000000 --- a/be/src/olap/out_stream.h +++ /dev/null @@ -1,169 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "olap/byte_buffer.h" -#include "olap/compress.h" -#include "olap/olap_define.h" -#include "olap/stream_index_writer.h" -#include "olap/stream_name.h" - -namespace doris { -class FileHandler; - -// Unlike OrcFile, we cannot guarantee the reliability of stored data without HDFS at the bottom, so we must write -// Check value, check this check value when reading data -// Adopt TLV type header, which has sufficient scalability -struct StreamHead { - enum StreamType { UNCOMPRESSED = 0, COMPRESSED = 1 }; - uint8_t type; // 256 types, should be enough for future expansion - uint32_t length : 24; // 24-bit length - uint32_t checksum; // 32-bit check value - StreamHead() : type(COMPRESSED), length(0), checksum(0) {} -} __attribute__((packed)); - -// Output stream, use a set of ByteBuffer to buffer all data -class OutStream { -public: - // The output stream supports two modes: compressed or uncompressed. If compression is enabled, the compression function is given - explicit OutStream(uint32_t buffer_size, Compressor compressor); - - ~OutStream(); - - // Output a byte to the stream - Status write(char byte) { - Status res = Status::OK(); - if (_current == nullptr) { - res = _create_new_input_buffer(); - if (!res.ok()) { - return res; - } - } - if (_current->remaining() < 1) { - res = _spill(); - if (!res.ok()) { - LOG(WARNING) << "fail to spill current buffer."; - return res; - } - if (_current == nullptr) { - res = _create_new_input_buffer(); - if (!res.ok()) { - return res; - } - } - } - return _current->put(byte); - } - - // Output a piece of data to the stream - Status write(const char* buffer, uint64_t length); - - // Record the current position of the stream in the index entry - void get_position(PositionEntryWriter* index_entry) const; - - // Returns the size of all data in the stream - uint64_t get_stream_length() const; - - // Returns the size of the buffer that has been allocated - uint64_t get_total_buffer_size() const; - - // Output the cached data stream to a file - Status write_to_file(FileHandler* file_handle, uint32_t write_mbytes_per_sec) const; - - bool is_suppressed() const { return _is_suppressed; } - void suppress() { _is_suppressed = true; } - // Output data to output_buffers - Status flush(); - // Calculate the crc32 value of the output data - uint32_t crc32(uint32_t checksum) const; - const std::vector& output_buffers() { return _output_buffers; } - - void print_position_debug_info() { - VLOG_TRACE << "compress: " << _spilled_bytes; - - if (_current != nullptr) { - VLOG_TRACE << "uncompress=" << (_current->position() - sizeof(StreamHead)); - } else { - VLOG_TRACE << "uncompress 0"; - } - } - -private: - Status _create_new_input_buffer(); - Status _write_head(StorageByteBuffer* buf, uint64_t position, StreamHead::StreamType type, - uint32_t length); - Status _spill(); - Status _compress(StorageByteBuffer* input, StorageByteBuffer* output, - StorageByteBuffer* overflow, bool* smaller); - void _output_uncompress(); - void _output_compressed(); - Status _make_sure_output_buffer(); - - uint32_t _buffer_size; // Compressed block size - Compressor _compressor; // Compression function, if NULL means no compression - std::vector _output_buffers; // Buffer all output - bool _is_suppressed; // Whether the stream is terminated - StorageByteBuffer* _current; // Cache uncompressed data - StorageByteBuffer* _compressed; // Bytes to be output to output_buffers - StorageByteBuffer* _overflow; // Bytes that can't fit in _output - uint64_t _spilled_bytes; // The number of bytes that have been output to output - - DISALLOW_COPY_AND_ASSIGN(OutStream); -}; - -// Define the factory method of the output stream -// Host all output streams, and encapsulate information such as compression algorithm, whether to enable Index, block size, etc. -class OutStreamFactory { -public: - explicit OutStreamFactory(CompressKind compress_kind, uint32_t stream_buffer_size); - - ~OutStreamFactory(); - - //The lifetime of the stream after creation is still managed by OutStreamFactory - OutStream* create_stream(uint32_t column_unique_id, StreamInfoMessage::Kind kind); - - const std::map& streams() const { return _streams; } - -private: - std::map _streams; // All created streams - Compressor _compressor; - uint32_t _stream_buffer_size; - - DISALLOW_COPY_AND_ASSIGN(OutStreamFactory); -}; - -/* -class OutStreamBufferWrapper : public std::streambuf { -public: - OutStreamBufferWrapper(OutStream* output) - : std::streambuf(), - _stream(output), - _skip_size(0) { - - } - virtual ~OutStreamBufferWrapper() {} - virtual int_type overflow(typename traits::int_type c = traits::eof()) { - return c; - } -protected: - OutStream* _stream; - size_t _skip_size; -}; -*/ - -} // namespace doris diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index 4ab7ecfddd..5b3d1b2cbc 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -162,7 +162,6 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR request.partition_id, tablet_var.tablet, request.transaction_id); // has to check rollback status to ensure not delete a committed rowset if (rollback_status.ok()) { - // actually, olap_index may has been deleted in delete_transaction() StorageEngine::instance()->add_unused_rowset(tablet_var.rowset_to_add); } } diff --git a/be/src/olap/reader.h b/be/src/olap/reader.h index 2593a51bae..c0fb42ebf8 100644 --- a/be/src/olap/reader.h +++ b/be/src/olap/reader.h @@ -77,7 +77,6 @@ public: std::vector conditions; std::vector>> bloom_filters; - // The ColumnData will be set when using Merger, eg Cumulative, BE. std::vector rs_readers; std::vector return_columns; RuntimeProfile* profile = nullptr; diff --git a/be/src/olap/row_block.h b/be/src/olap/row_block.h index f40f2fd93b..ac157991eb 100644 --- a/be/src/olap/row_block.h +++ b/be/src/olap/row_block.h @@ -26,7 +26,6 @@ #include "olap/olap_define.h" #include "olap/row_cursor.h" #include "olap/utils.h" -#include "runtime/vectorized_row_batch.h" namespace doris { @@ -53,7 +52,6 @@ class RowBlock { // Please keep these classes as 'friend'. They have to use lots of private fields for // faster operation. friend class RowBlockChanger; - friend class VectorizedRowBatch; public: RowBlock(const TabletSchema* schema); @@ -128,7 +126,6 @@ private: // Field offset of memory row format, used to get field ptr in memory row std::vector _field_offset_in_memory; - // only used for SegmentReader to covert VectorizedRowBatch to RowBlock // Be careful to use this size_t _pos = 0; size_t _limit = 0; diff --git a/be/src/olap/row_block2.h b/be/src/olap/row_block2.h index fda9a49d6f..22a5745ddf 100644 --- a/be/src/olap/row_block2.h +++ b/be/src/olap/row_block2.h @@ -38,7 +38,6 @@ class RowBlockRow; class RowCursor; // This struct contains a block of rows, in which each column's data is stored -// in a vector. We don't use VectorizedRowBatch because it doesn't own the data // in block, however it is used by old code, which we don't want to change. class RowBlockV2 { public: diff --git a/be/src/olap/rowset/CMakeLists.txt b/be/src/olap/rowset/CMakeLists.txt index 2f4720e5ac..131978a9c4 100644 --- a/be/src/olap/rowset/CMakeLists.txt +++ b/be/src/olap/rowset/CMakeLists.txt @@ -22,26 +22,10 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/olap/rowset") set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/olap/rowset") add_library(Rowset STATIC - bit_field_reader.cpp - bit_field_writer.cpp - column_data.cpp - column_reader.cpp - column_writer.cpp - column_data_writer.cpp - segment_group.cpp - run_length_byte_reader.cpp - run_length_byte_writer.cpp - run_length_integer_reader.cpp - run_length_integer_writer.cpp unique_rowset_id_generator.cpp - segment_reader.cpp - segment_writer.cpp rowset.cpp rowset_factory.cpp rowset_meta_manager.cpp - alpha_rowset.cpp - alpha_rowset_reader.cpp - alpha_rowset_writer.cpp alpha_rowset_meta.cpp beta_rowset.cpp beta_rowset_reader.cpp diff --git a/be/src/olap/rowset/alpha_rowset.cpp b/be/src/olap/rowset/alpha_rowset.cpp deleted file mode 100644 index 7f2e74c744..0000000000 --- a/be/src/olap/rowset/alpha_rowset.cpp +++ /dev/null @@ -1,407 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/alpha_rowset.h" - -#include - -#include "olap/row.h" -#include "olap/rowset/alpha_rowset_meta.h" -#include "olap/rowset/alpha_rowset_reader.h" -#include "olap/rowset/rowset_meta_manager.h" -#include "util/hash_util.hpp" - -namespace doris { - -AlphaRowset::AlphaRowset(const TabletSchema* schema, const FilePathDesc& rowset_path_desc, - RowsetMetaSharedPtr rowset_meta) - : Rowset(schema, rowset_path_desc, std::move(rowset_meta)) {} - -Status AlphaRowset::do_load(bool use_cache) { - for (auto& segment_group : _segment_groups) { - // validate segment group - if (segment_group->validate() != Status::OK()) { - LOG(WARNING) << "fail to validate segment_group. [version=" << start_version() << "-" - << end_version(); - // if load segment group failed, rowset init failed - return Status::OLAPInternalError(OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR); - } - Status res = segment_group->load(use_cache); - if (!res.ok()) { - LOG(WARNING) << "fail to load segment_group. res=" << res << ", " - << "version=" << start_version() << "-" << end_version(); - return res; - } - } - return Status::OK(); -} - -Status AlphaRowset::create_reader(std::shared_ptr* result) { - result->reset(new AlphaRowsetReader(_schema->num_rows_per_row_block(), - std::static_pointer_cast(shared_from_this()))); - return Status::OK(); -} - -Status AlphaRowset::remove() { - VLOG_NOTICE << "begin to remove files in rowset " << unique_id() - << ", version:" << start_version() << "-" << end_version() - << ", tabletid:" << _rowset_meta->tablet_id(); - for (auto segment_group : _segment_groups) { - bool ret = segment_group->delete_all_files(); - if (!ret) { - LOG(WARNING) << "delete segment group files failed." - << " tablet id:" << segment_group->get_tablet_id() - << ", rowset path:" << segment_group->rowset_path_prefix(); - return Status::OLAPInternalError(OLAP_ERR_ROWSET_DELETE_FILE_FAILED); - } - } - return Status::OK(); -} - -void AlphaRowset::make_visible_extra(Version version) { - AlphaRowsetMetaSharedPtr alpha_rowset_meta = - std::dynamic_pointer_cast(_rowset_meta); - std::vector published_segment_groups; - alpha_rowset_meta->get_segment_groups(&published_segment_groups); - int32_t segment_group_idx = 0; - for (auto& segment_group : _segment_groups) { - segment_group->set_version(version); - segment_group->set_pending_finished(); - published_segment_groups.at(segment_group_idx).clear_load_id(); - ++segment_group_idx; - } - alpha_rowset_meta->clear_segment_group(); - for (auto& segment_group_meta : published_segment_groups) { - alpha_rowset_meta->add_segment_group(segment_group_meta); - } -} - -Status AlphaRowset::link_files_to(const FilePathDesc& dir_desc, RowsetId new_rowset_id) { - for (auto& segment_group : _segment_groups) { - auto status = segment_group->link_segments_to_path(dir_desc.filepath, new_rowset_id); - if (!status.ok()) { - LOG(WARNING) << "create hard links failed for segment group:" - << segment_group->segment_group_id(); - return status; - } - } - return Status::OK(); -} - -Status AlphaRowset::copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) { - for (auto& segment_group : _segment_groups) { - Status status = segment_group->copy_files_to(dir); - if (!status.ok()) { - LOG(WARNING) << "copy files failed for segment group." - << " segment_group_id:" << segment_group->segment_group_id() - << ", dest_path:" << dir; - return status; - } - } - return Status::OK(); -} - -Status AlphaRowset::convert_from_old_files(const std::string& snapshot_path, - std::vector* success_files) { - for (auto& segment_group : _segment_groups) { - Status status = segment_group->convert_from_old_files(snapshot_path, success_files); - if (!status.ok()) { - LOG(WARNING) << "create hard links failed for segment group:" - << segment_group->segment_group_id(); - return status; - } - } - return Status::OK(); -} - -Status AlphaRowset::convert_to_old_files(const std::string& snapshot_path, - std::vector* success_files) { - for (auto& segment_group : _segment_groups) { - Status status = segment_group->convert_to_old_files(snapshot_path, success_files); - if (!status.ok()) { - LOG(WARNING) << "create hard links failed for segment group:" - << segment_group->segment_group_id(); - return status; - } - } - return Status::OK(); -} - -Status AlphaRowset::remove_old_files(std::vector* files_to_remove) { - for (auto& segment_group : _segment_groups) { - Status status = segment_group->remove_old_files(files_to_remove); - if (!status.ok()) { - LOG(WARNING) << "remove old files failed for segment group:" - << segment_group->segment_group_id(); - return status; - } - } - return Status::OK(); -} - -Status AlphaRowset::split_range(const RowCursor& start_key, const RowCursor& end_key, - uint64_t request_block_row_count, size_t key_num, - std::vector* ranges) { - if (key_num > _schema->num_short_key_columns()) { - // should not happen - // But since aloha rowset is deprecated in future and it will not fail the query, - // just use VLOG to avoid too many warning logs. - VLOG_NOTICE << "key num " << key_num - << " should less than or equal to short key column number: " - << _schema->num_short_key_columns(); - return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA); - } - EntrySlice entry; - RowBlockPosition start_pos; - RowBlockPosition end_pos; - RowBlockPosition step_pos; - - std::shared_ptr largest_segment_group = _segment_group_with_largest_size(); - if (largest_segment_group == nullptr || - largest_segment_group->current_num_rows_per_row_block() == 0) { - VLOG_NOTICE << "failed to get largest_segment_group. is null: " - << (largest_segment_group == nullptr) << ". version: " << start_version() << "-" - << end_version() << ". tablet: " << rowset_meta()->tablet_id(); - ranges->emplace_back(start_key.to_tuple()); - ranges->emplace_back(end_key.to_tuple()); - return Status::OK(); - } - uint64_t expected_rows = - request_block_row_count / largest_segment_group->current_num_rows_per_row_block(); - if (expected_rows == 0) { - LOG(WARNING) << "expected_rows less than 1. [request_block_row_count = " - << request_block_row_count << "]"; - return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA); - } - - // find the start position of start key - RowCursor helper_cursor; - if (helper_cursor.init(*_schema, key_num) != Status::OK()) { - LOG(WARNING) << "fail to parse strings to key with RowCursor type."; - return Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA); - } - if (!largest_segment_group->find_short_key(start_key, &helper_cursor, false, &start_pos)) { - if (!largest_segment_group->find_first_row_block(&start_pos)) { - LOG(WARNING) << "fail to get first block pos"; - return Status::OLAPInternalError(OLAP_ERR_TABLE_INDEX_FIND_ERROR); - } - } - - step_pos = start_pos; - VLOG_NOTICE << "start_pos=" << start_pos.segment << ", " << start_pos.index_offset; - - //find last row_block is end_key is given, or using last_row_block - if (!largest_segment_group->find_short_key(end_key, &helper_cursor, false, &end_pos)) { - if (!largest_segment_group->find_last_row_block(&end_pos)) { - LOG(WARNING) << "fail find last row block."; - return Status::OLAPInternalError(OLAP_ERR_TABLE_INDEX_FIND_ERROR); - } - } - - VLOG_NOTICE << "end_pos=" << end_pos.segment << ", " << end_pos.index_offset; - - //get rows between first and last - Status res = Status::OK(); - RowCursor cur_start_key; - RowCursor last_start_key; - - if (cur_start_key.init(*_schema, key_num) != Status::OK() || - last_start_key.init(*_schema, key_num) != Status::OK()) { - LOG(WARNING) << "fail to init cursor"; - return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); - } - - std::vector cids; - for (uint32_t cid = 0; cid < key_num; ++cid) { - cids.push_back(cid); - } - - if (largest_segment_group->get_row_block_entry(start_pos, &entry) != Status::OK()) { - LOG(WARNING) << "get block entry failed."; - return Status::OLAPInternalError(OLAP_ERR_ROWBLOCK_FIND_ROW_EXCEPTION); - } - - cur_start_key.attach(entry.data); - last_start_key.allocate_memory_for_string_type(*_schema); - direct_copy_row(&last_start_key, cur_start_key); - // start_key是last start_key, 但返回的实际上是查询层给出的key - ranges->emplace_back(start_key.to_tuple()); - - while (end_pos > step_pos) { - res = largest_segment_group->advance_row_block(expected_rows, &step_pos); - if (res == Status::OLAPInternalError(OLAP_ERR_INDEX_EOF) || !(end_pos > step_pos)) { - break; - } else if (!res.ok()) { - LOG(WARNING) << "advance_row_block failed."; - return Status::OLAPInternalError(OLAP_ERR_ROWBLOCK_FIND_ROW_EXCEPTION); - } - - if (largest_segment_group->get_row_block_entry(step_pos, &entry) != Status::OK()) { - LOG(WARNING) << "get block entry failed."; - return Status::OLAPInternalError(OLAP_ERR_ROWBLOCK_FIND_ROW_EXCEPTION); - } - cur_start_key.attach(entry.data); - - if (!equal_row(cids, cur_start_key, last_start_key)) { - ranges->emplace_back(cur_start_key.to_tuple()); // end of last section - ranges->emplace_back(cur_start_key.to_tuple()); // start a new section - direct_copy_row(&last_start_key, cur_start_key); - } - } - - ranges->emplace_back(end_key.to_tuple()); - return Status::OK(); -} - -bool AlphaRowset::check_path(const std::string& path) { - std::set valid_paths; - for (auto& segment_group : _segment_groups) { - for (int i = 0; i < segment_group->num_segments(); ++i) { - std::string data_path = segment_group->construct_data_file_path(i); - std::string index_path = segment_group->construct_index_file_path(i); - valid_paths.insert(data_path); - valid_paths.insert(index_path); - } - } - return valid_paths.find(path) != valid_paths.end(); -} - -bool AlphaRowset::check_file_exist() { - for (auto& segment_group : _segment_groups) { - for (int i = 0; i < segment_group->num_segments(); ++i) { - std::string data_path = segment_group->construct_data_file_path(i); - if (!FileUtils::check_exist(data_path)) { - LOG(WARNING) << "data file not existed: " << data_path - << " for rowset_id: " << rowset_id(); - return false; - } - std::string index_path = segment_group->construct_index_file_path(i); - if (!FileUtils::check_exist(index_path)) { - LOG(WARNING) << "index file not existed: " << index_path - << " for rowset_id: " << rowset_id(); - return false; - } - } - } - return true; -} - -Status AlphaRowset::init() { - std::vector segment_group_metas; - AlphaRowsetMetaSharedPtr _alpha_rowset_meta = - std::dynamic_pointer_cast(_rowset_meta); - _alpha_rowset_meta->get_segment_groups(&segment_group_metas); - for (auto& segment_group_meta : segment_group_metas) { - std::shared_ptr segment_group; - if (_is_pending) { - segment_group.reset(new SegmentGroup( - _rowset_meta->tablet_id(), _rowset_meta->rowset_id(), _schema, - _rowset_path_desc.filepath, false, segment_group_meta.segment_group_id(), - segment_group_meta.num_segments(), true, _rowset_meta->partition_id(), - _rowset_meta->txn_id())); - } else { - segment_group.reset(new SegmentGroup( - _rowset_meta->tablet_id(), _rowset_meta->rowset_id(), _schema, - _rowset_path_desc.filepath, _rowset_meta->version(), false, - segment_group_meta.segment_group_id(), segment_group_meta.num_segments())); - } - if (segment_group == nullptr) { - LOG(WARNING) << "fail to create olap segment_group. rowset_id='" - << _rowset_meta->rowset_id(); - return Status::OLAPInternalError(OLAP_ERR_CREATE_FILE_ERROR); - } - if (segment_group_meta.has_empty()) { - segment_group->set_empty(segment_group_meta.empty()); - } - - if (segment_group_meta.zone_maps_size() != 0) { - size_t zone_maps_size = segment_group_meta.zone_maps_size(); - // after 0.12.10 the value column in duplicate table also has zone map. - // after 0.14 the value column in duplicate table also has zone map. - size_t expect_zone_maps_num = _schema->keys_type() != KeysType::AGG_KEYS - ? _schema->num_columns() - : _schema->num_key_columns(); - if ((_schema->keys_type() == KeysType::AGG_KEYS && - expect_zone_maps_num != zone_maps_size) || - (_schema->keys_type() != KeysType::AGG_KEYS && - expect_zone_maps_num < zone_maps_size)) { - LOG(ERROR) << "column pruning size is error. " - << "KeysType=" << KeysType_Name(_schema->keys_type()) << ", " - << "zone_maps_size=" << zone_maps_size << ", " - << "num_key_columns=" << _schema->num_key_columns() << ", " - << "num_columns=" << _schema->num_columns(); - return Status::OLAPInternalError(OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR); - } - // Before 0.12.10, the zone map columns number in duplicate/unique table is the same with the key column numbers, - // but after 0.12.10 we build zone map for duplicate table value column, after 0.14 we build zone map for unique - // table value column, so when first start the two number is not the same, - // it causes start failed. When `expect_zone_maps_num > zone_maps_size` it may be the first start after upgrade - if (expect_zone_maps_num > zone_maps_size) { - VLOG_CRITICAL - << "tablet: " << _rowset_meta->tablet_id() << " expect zone map size is " - << expect_zone_maps_num << ", actual num is " << zone_maps_size - << ". If this is not the first start after upgrade, please pay attention!"; - } - zone_maps_size = std::min(zone_maps_size, expect_zone_maps_num); - std::vector> zone_map_strings(zone_maps_size); - std::vector null_vec(zone_maps_size); - for (size_t j = 0; j < zone_maps_size; ++j) { - const ZoneMap& zone_map = segment_group_meta.zone_maps(j); - zone_map_strings[j].first = zone_map.min(); - zone_map_strings[j].second = zone_map.max(); - if (zone_map.has_null_flag()) { - null_vec[j] = zone_map.null_flag(); - } else { - null_vec[j] = false; - } - } - Status status = segment_group->add_zone_maps(zone_map_strings, null_vec); - if (!status.ok()) { - LOG(WARNING) << "segment group add column statistics failed, status:" << status; - return status; - } - } - _segment_groups.push_back(segment_group); - } - if (_is_cumulative && _segment_groups.size() > 1) { - LOG(WARNING) << "invalid segment group meta for cumulative rowset. segment group size:" - << _segment_groups.size(); - return Status::OLAPInternalError(OLAP_ERR_ENGINE_LOAD_INDEX_TABLE_ERROR); - } - return Status::OK(); -} - -std::shared_ptr AlphaRowset::_segment_group_with_largest_size() { - std::shared_ptr largest_segment_group = nullptr; - size_t largest_segment_group_sizes = 0; - - for (auto segment_group : _segment_groups) { - if (!segment_group->index_loaded()) { - continue; - } - if (segment_group->empty() || segment_group->zero_num_rows()) { - continue; - } - if (segment_group->index_size() > largest_segment_group_sizes) { - largest_segment_group = segment_group; - largest_segment_group_sizes = segment_group->index_size(); - } - } - return largest_segment_group; -} - -} // namespace doris diff --git a/be/src/olap/rowset/alpha_rowset.h b/be/src/olap/rowset/alpha_rowset.h deleted file mode 100644 index 9f4c838723..0000000000 --- a/be/src/olap/rowset/alpha_rowset.h +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_H -#define DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_H - -#include -#include - -#include "olap/data_dir.h" -#include "olap/rowset/rowset.h" -#include "olap/rowset/rowset_meta.h" -#include "olap/rowset/segment_group.h" -#include "olap/tuple.h" - -namespace doris { - -class AlphaRowset; -using AlphaRowsetSharedPtr = std::shared_ptr; -class AlphaRowsetWriter; -class AlphaRowsetReader; -class RowsetFactory; - -class AlphaRowset : public Rowset { -public: - virtual ~AlphaRowset() {} - - Status create_reader(std::shared_ptr* result) override; - - Status split_range(const RowCursor& start_key, const RowCursor& end_key, - uint64_t request_block_row_count, size_t key_num, - std::vector* ranges) override; - - Status remove() override; - - Status link_files_to(const FilePathDesc& dir_desc, RowsetId new_rowset_id) override; - - Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) override; - - Status convert_from_old_files(const std::string& snapshot_path, - std::vector* success_files); - - Status convert_to_old_files(const std::string& snapshot_path, - std::vector* success_files); - - Status remove_old_files(std::vector* files_to_remove) override; - - bool check_path(const std::string& path) override; - - bool check_file_exist() override; - -protected: - friend class RowsetFactory; - - AlphaRowset(const TabletSchema* schema, const FilePathDesc& rowset_path_desc, - RowsetMetaSharedPtr rowset_meta); - - // init segment groups - Status init() override; - - Status do_load(bool use_cache) override; - - void do_close() override {} - - // add custom logic when rowset is published - void make_visible_extra(Version version) override; - -private: - std::shared_ptr _segment_group_with_largest_size(); - -private: - friend class AlphaRowsetWriter; - friend class AlphaRowsetReader; - - std::vector> _segment_groups; -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_H diff --git a/be/src/olap/rowset/alpha_rowset_reader.cpp b/be/src/olap/rowset/alpha_rowset_reader.cpp deleted file mode 100644 index 3b0d7472ee..0000000000 --- a/be/src/olap/rowset/alpha_rowset_reader.cpp +++ /dev/null @@ -1,400 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/alpha_rowset_reader.h" - -#include "olap/row.h" -#include "olap/rowset/alpha_rowset.h" - -namespace doris { - -AlphaRowsetReader::AlphaRowsetReader(int num_rows_per_row_block, AlphaRowsetSharedPtr rowset) - : _num_rows_per_row_block(num_rows_per_row_block), - _rowset(std::move(rowset)), - _alpha_rowset_meta( - std::static_pointer_cast(_rowset->rowset_meta()).get()), - _segment_groups(_rowset->_segment_groups), - _key_range_size(0) { - _rowset->acquire(); -} - -AlphaRowsetReader::~AlphaRowsetReader() { - delete _dst_cursor; - _rowset->release(); - while (!_merge_heap.empty()) { - auto ctx = _merge_heap.top(); - _merge_heap.pop(); - delete ctx; - } - for (auto ctx : _sequential_ctxs) { - delete ctx; - } - _sequential_ctxs.clear(); -} - -Status AlphaRowsetReader::init(RowsetReaderContext* read_context) { - RETURN_NOT_OK(_rowset->load()); - if (read_context == nullptr) { - return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); - } - _current_read_context = read_context; - if (_current_read_context->stats != nullptr) { - _stats = _current_read_context->stats; - } - - _is_segments_overlapping = _alpha_rowset_meta->is_segments_overlapping(); - - RETURN_NOT_OK(_init_merge_ctxs(read_context)); - - // needs to sort merge only when - // 1) we are told to return sorted result (need_ordered_result) - // 2) we have several segment groups (_is_segments_overlapping && _sequential_ctxs.size() > 1) - if (_current_read_context->need_ordered_result && _is_segments_overlapping && - _sequential_ctxs.size() > 1) { - _next_block = &AlphaRowsetReader::_merge_block; - _read_block.reset(new (std::nothrow) RowBlock(_current_read_context->tablet_schema)); - if (_read_block == nullptr) { - LOG(WARNING) << "new row block failed in reader"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - RowBlockInfo block_info; - block_info.row_num = _current_read_context->tablet_schema->num_rows_per_row_block(); - block_info.null_supported = true; - _read_block->init(block_info); - _dst_cursor = new (std::nothrow) RowCursor(); - if (_dst_cursor == nullptr) { - LOG(WARNING) << "allocate memory for row cursor failed"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - if (_current_read_context->reader_type == READER_ALTER_TABLE) { - // Upon rollup/alter table, seek_columns is nullptr. - // Under this circumstance, init RowCursor with all columns. - _dst_cursor->init(*(_current_read_context->tablet_schema)); - for (auto ctx : _sequential_ctxs) { - ctx->row_cursor.reset(new (std::nothrow) RowCursor()); - ctx->row_cursor->init(*(_current_read_context->tablet_schema)); - } - } else { - _dst_cursor->init(*(_current_read_context->tablet_schema), - *(_current_read_context->seek_columns)); - for (auto ctx : _sequential_ctxs) { - ctx->row_cursor.reset(new (std::nothrow) RowCursor()); - ctx->row_cursor->init(*(_current_read_context->tablet_schema), - *(_current_read_context->seek_columns)); - } - } - RETURN_NOT_OK(_init_merge_heap()); - } else { - _next_block = &AlphaRowsetReader::_union_block; - _cur_ctx = *(_sequential_ctxs.begin()); - } - return Status::OK(); -} - -Status AlphaRowsetReader::next_block(RowBlock** block) { - return (this->*_next_block)(block); -} - -bool AlphaRowsetReader::delete_flag() { - return _alpha_rowset_meta->delete_flag(); -} - -Version AlphaRowsetReader::version() { - return _alpha_rowset_meta->version(); -} - -int64_t AlphaRowsetReader::filtered_rows() { - return _stats->rows_del_filtered; -} - -Status AlphaRowsetReader::_union_block(RowBlock** block) { - while (_cur_ctx != nullptr) { - // union block only use one block to store - Status status = _pull_next_block(_cur_ctx); - if (status == Status::OLAPInternalError(OLAP_ERR_DATA_EOF)) { - delete _cur_ctx; - _cur_ctx = nullptr; - _sequential_ctxs.pop_front(); - if (!_sequential_ctxs.empty()) { - _cur_ctx = *(_sequential_ctxs.begin()); - } - } else if (!status.ok()) { - return status; - } else { - (*block) = _cur_ctx->row_block; - return Status::OK(); - } - } - if (_sequential_ctxs.empty()) { - *block = nullptr; - return Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } - - return Status::OK(); -} - -Status AlphaRowsetReader::_merge_block(RowBlock** block) { - // Row among different segment groups may overlap with each other. - // Iterate all row_blocks to fetch min row each round. - Status status = Status::OK(); - _read_block->clear(); - size_t num_rows_in_block = 0; - while (_read_block->pos() < _num_rows_per_row_block) { - // 1. Read one row from heap - RowCursor* row_cursor = nullptr; - status = _pull_next_row_for_merge_rowset_v2(&row_cursor); - if (status == Status::OLAPInternalError(OLAP_ERR_DATA_EOF) && _read_block->pos() > 0) { - status = Status::OK(); - break; - } else if (!status.ok()) { - return status; - } - - VLOG_TRACE << "get merged row: " << row_cursor->to_string(); - - // 2. Copy the row to buffer block - _read_block->get_row(_read_block->pos(), _dst_cursor); - copy_row(_dst_cursor, *row_cursor, _read_block->mem_pool()); - _read_block->pos_inc(); - num_rows_in_block++; - - // 3. Adjust heap - // MergeHeap should advance one step after row been read. - // This function must be called after copy_row - // Otherwise, the row has read will be modified instantly before handled. - // For example: - // If I have (1, 1), (2, 2), (3, 3) three records. - // Now I have read (1, 1). - // Before copy_row, I rebuild the heap - // The returned row will be (2, 2) instead of (1, 1) - AlphaMergeContext* merge_ctx = _merge_heap.top(); - _merge_heap.pop(); - // merge_ctx will not be pushed back into heap if it is EOF - RETURN_NOT_OK(_update_merge_ctx_and_build_merge_heap(merge_ctx)); - } - _read_block->set_pos(0); - _read_block->set_limit(num_rows_in_block); - _read_block->finalize(num_rows_in_block); - *block = _read_block.get(); - return status; -} - -Status AlphaRowsetReader::_init_merge_heap() { - DCHECK(_merge_heap.empty()); - DCHECK(!_sequential_ctxs.empty()); - for (auto merge_ctx : _sequential_ctxs) { - RETURN_NOT_OK(_update_merge_ctx_and_build_merge_heap(merge_ctx)); - } - _sequential_ctxs.clear(); - return Status::OK(); -} - -Status AlphaRowsetReader::_update_merge_ctx_and_build_merge_heap(AlphaMergeContext* merge_ctx) { - if (OLAP_UNLIKELY(merge_ctx->is_eof)) { - // nothing in this merge ctx, release and return - delete merge_ctx; - return Status::OK(); - } - - // get next row block of this merge ctx - if (merge_ctx->row_block == nullptr || !merge_ctx->row_block->has_remaining()) { - Status status = _pull_next_block(merge_ctx); - if (status == Status::OLAPInternalError(OLAP_ERR_DATA_EOF)) { - // nothing in this merge ctx, release and return - delete merge_ctx; - return Status::OK(); - } else if (!status.ok()) { - delete merge_ctx; - LOG(WARNING) << "read next row of singleton rowset failed:" << status; - return status; - } - } - - // read the first row, push it into merge heap, and step forward - RowCursor* current_row = merge_ctx->row_cursor.get(); - merge_ctx->row_block->get_row(merge_ctx->row_block->pos(), current_row); - _merge_heap.push(merge_ctx); - merge_ctx->row_block->pos_inc(); - return Status::OK(); -} - -Status AlphaRowsetReader::_pull_next_row_for_merge_rowset_v2(RowCursor** row) { - // if _merge_heap is not empty, return the row at top, and insert a new row - // from corresponding merge_ctx - if (OLAP_LIKELY(!_merge_heap.empty())) { - AlphaMergeContext* merge_ctx = _merge_heap.top(); - *row = merge_ctx->row_cursor.get(); - // Must not rebuild merge_heap in this place. - // Because row have not been copied and is a pointer. - // If rebuild merge_heap, content in row will be modified. - return Status::OK(); - } else { - // all rows are read - return Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } -} - -Status AlphaRowsetReader::_pull_next_block(AlphaMergeContext* merge_ctx) { - Status status = Status::OK(); - if (OLAP_UNLIKELY(merge_ctx->first_read_symbol)) { - if (_key_range_size > 0) { - status = _pull_first_block(merge_ctx); - } else { - status = merge_ctx->column_data->get_first_row_block(&(merge_ctx->row_block)); - if (!status.ok() && status != Status::OLAPInternalError(OLAP_ERR_DATA_EOF)) { - LOG(WARNING) << "get first row block failed, status:" << status; - } - } - merge_ctx->first_read_symbol = false; - return status; - } else { - // get next block - status = merge_ctx->column_data->get_next_block(&(merge_ctx->row_block)); - if (status == Status::OLAPInternalError(OLAP_ERR_DATA_EOF) && _key_range_size > 0) { - // reach the end of one predicate - // currently, SegmentReader can only support filter one key range a time - // refresh the predicate and continue read - return _pull_first_block(merge_ctx); - } - } - return status; -} - -Status AlphaRowsetReader::_pull_first_block(AlphaMergeContext* merge_ctx) { - Status status = Status::OK(); - merge_ctx->key_range_index++; - while (merge_ctx->key_range_index < _key_range_size) { - status = merge_ctx->column_data->prepare_block_read( - &_current_read_context->lower_bound_keys->at(merge_ctx->key_range_index), - _current_read_context->is_lower_keys_included->at(merge_ctx->key_range_index), - &_current_read_context->upper_bound_keys->at(merge_ctx->key_range_index), - _current_read_context->is_upper_keys_included->at(merge_ctx->key_range_index), - &(merge_ctx->row_block)); - if (status == Status::OLAPInternalError(OLAP_ERR_DATA_EOF)) { - merge_ctx->key_range_index++; - continue; - } else if (!status.ok()) { - LOG(WARNING) << "prepare block read failed. status=" << status; - return status; - } else { - break; - } - } - if (merge_ctx->key_range_index >= _key_range_size) { - merge_ctx->row_block = nullptr; - return Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } - return status; -} - -Status AlphaRowsetReader::_init_merge_ctxs(RowsetReaderContext* read_context) { - if (read_context->reader_type == READER_QUERY) { - if (read_context->lower_bound_keys->size() != - read_context->is_lower_keys_included->size() || - read_context->lower_bound_keys->size() != read_context->upper_bound_keys->size() || - read_context->upper_bound_keys->size() != - read_context->is_upper_keys_included->size()) { - std::string error_msg = "invalid key range arguments"; - LOG(WARNING) << error_msg; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - _key_range_size = read_context->lower_bound_keys->size(); - } - - // avoid polluting index stream cache by non-query workload (compaction/alter/checksum) - const bool use_index_stream_cache = read_context->reader_type == READER_QUERY; - - for (auto& segment_group : _segment_groups) { - std::unique_ptr new_column_data(ColumnData::create(segment_group.get())); - Status status = new_column_data->init(); - if (!status.ok()) { - LOG(WARNING) << "init column data failed"; - return Status::OLAPInternalError(OLAP_ERR_READER_READING_ERROR); - } - new_column_data->set_delete_handler(read_context->delete_handler); - new_column_data->set_stats(_stats); - if (read_context->reader_type == READER_ALTER_TABLE) { - new_column_data->schema_change_init(); - new_column_data->set_using_cache(use_index_stream_cache); - if (new_column_data->empty() && new_column_data->zero_num_rows()) { - continue; - } - } else { - std::shared_ptr> predicates = - std::make_shared>(); - if (read_context->predicates != nullptr) { - predicates->insert(predicates->end(), read_context->predicates->begin(), - read_context->predicates->end()); - } - // if unique table with rowset [0-x] or [0-1] [2-y] [...], - // value column predicates can be pushdown on rowset [0-x] or [2-y] - if (read_context->value_predicates != nullptr && _rowset->keys_type() == UNIQUE_KEYS && - (_rowset->start_version() == 0 || _rowset->start_version() == 2)) { - predicates->insert(predicates->end(), read_context->value_predicates->begin(), - read_context->value_predicates->end()); - } - new_column_data->set_read_params( - *read_context->return_columns, *read_context->seek_columns, - *read_context->load_bf_columns, *read_context->conditions, predicates, - use_index_stream_cache, read_context->runtime_state); - // filter - if (new_column_data->rowset_pruning_filter()) { - _stats->rows_stats_filtered += new_column_data->num_rows(); - VLOG_NOTICE << "filter segment group in query in condition. version=" - << new_column_data->version(); - continue; - } - } - - int ret = new_column_data->delete_pruning_filter(); - if (ret == DEL_SATISFIED) { - _stats->rows_del_filtered += new_column_data->num_rows(); - VLOG_NOTICE << "filter segment group in delete predicate:" - << new_column_data->version(); - continue; - } else if (ret == DEL_PARTIAL_SATISFIED) { - VLOG_NOTICE << "filter segment group partially in delete predicate:" - << new_column_data->version(); - new_column_data->set_delete_status(DEL_PARTIAL_SATISFIED); - } else { - VLOG_NOTICE << "not filter segment group in delete predicate:" - << new_column_data->version(); - new_column_data->set_delete_status(DEL_NOT_SATISFIED); - } - auto merge_ctx = new AlphaMergeContext(); - merge_ctx->column_data = std::move(new_column_data); - _sequential_ctxs.emplace_back(merge_ctx); - } - - if (!_is_segments_overlapping && _sequential_ctxs.size() > 1) { - LOG(WARNING) << "invalid column_data for cumulative rowset. column_data size:" - << _sequential_ctxs.size(); - return Status::OLAPInternalError(OLAP_ERR_READER_READING_ERROR); - } - return Status::OK(); -} - -RowsetSharedPtr AlphaRowsetReader::rowset() { - return std::static_pointer_cast(_rowset); -} - -bool AlphaMergeContextComparator::operator()(const AlphaMergeContext* x, - const AlphaMergeContext* y) const { - return compare_row(*(x->row_cursor.get()), *(y->row_cursor.get())) > 0; -} - -} // namespace doris diff --git a/be/src/olap/rowset/alpha_rowset_reader.h b/be/src/olap/rowset/alpha_rowset_reader.h deleted file mode 100644 index 3dffec2457..0000000000 --- a/be/src/olap/rowset/alpha_rowset_reader.h +++ /dev/null @@ -1,136 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_READER_H -#define DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_READER_H - -#include -#include - -#include "olap/rowset/alpha_rowset.h" -#include "olap/rowset/alpha_rowset_meta.h" -#include "olap/rowset/column_data.h" -#include "olap/rowset/rowset_reader.h" -#include "olap/rowset/segment_group.h" - -namespace doris { - -// Each segment group corresponds to a MergeContext, which is able to produce ordered rows. -struct AlphaMergeContext { - std::unique_ptr column_data = nullptr; - - int key_range_index = -1; - - // Read data from ColumnData for the first time. - // ScanKey should be sought in this case. - bool first_read_symbol = true; - - RowBlock* row_block = nullptr; - - std::unique_ptr row_cursor = nullptr; - - bool is_eof = false; -}; - -struct AlphaMergeContextComparator { - bool operator()(const AlphaMergeContext* x, const AlphaMergeContext* y) const; -}; - -class AlphaRowsetReader : public RowsetReader { -public: - AlphaRowsetReader(int num_rows_per_row_block, AlphaRowsetSharedPtr rowset); - - ~AlphaRowsetReader() override; - - // reader init - Status init(RowsetReaderContext* read_context) override; - - // read next block data - // It's ok, because we only get ref here, the block's owner is this reader. - Status next_block(RowBlock** block) override; - - Status next_block(vectorized::Block* block) override { - return Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } - - bool delete_flag() override; - - Version version() override; - - RowsetSharedPtr rowset() override; - - int64_t filtered_rows() override; - - RowsetTypePB type() const override { return RowsetTypePB::ALPHA_ROWSET; } - -private: - Status _init_merge_ctxs(RowsetReaderContext* read_context); - - Status _union_block(RowBlock** block); - Status _merge_block(RowBlock** block); - Status _pull_next_block(AlphaMergeContext* merge_ctx); - - // Doris will split query predicates to several scan keys - // This function is used to fetch block when advancing - // current scan key to next scan key. - Status _pull_first_block(AlphaMergeContext* merge_ctx); - - // merge by priority queue(_merge_heap) - Status _pull_next_row_for_merge_rowset_v2(RowCursor** row); - // init the merge heap, this should be call before calling _pull_next_row_for_merge_rowset_v2(); - Status _init_merge_heap(); - // update the merge ctx. - // 1. get next row block of this ctx, if current row block is empty. - // 2. read the current row of the row block and push it to merge heap. - // 3. point to the next row of the row block - Status _update_merge_ctx_and_build_merge_heap(AlphaMergeContext* merge_ctx); - -private: - int _num_rows_per_row_block; - AlphaRowsetSharedPtr _rowset; - std::string _rowset_path; - AlphaRowsetMeta* _alpha_rowset_meta; - const std::vector>& _segment_groups; - - // In '_union_block' mode, it has items to traverse. - // In '_merge_block' mode, it will be cleared after '_merge_heap' has been built. - std::list _sequential_ctxs; - - std::unique_ptr _read_block; - Status (AlphaRowsetReader::*_next_block)(RowBlock** block) = nullptr; - RowCursor* _dst_cursor = nullptr; - int _key_range_size; - - // In streaming ingestion, row among different segment - // groups may overlap, and is necessary to be taken - // into consideration deliberately. - bool _is_segments_overlapping; - - // Current AlphaMergeContext to read data, just valid in '_union_block' mode. - AlphaMergeContext* _cur_ctx = nullptr; - // A priority queue for merging rowsets, just valid in '_merge_block' mode. - std::priority_queue, AlphaMergeContextComparator> - _merge_heap; - - RowsetReaderContext* _current_read_context; - OlapReaderStatistics _owned_stats; - OlapReaderStatistics* _stats = &_owned_stats; -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_ALPHA_ROWSET_READER_H diff --git a/be/src/olap/rowset/alpha_rowset_writer.cpp b/be/src/olap/rowset/alpha_rowset_writer.cpp deleted file mode 100644 index 138b9a73cf..0000000000 --- a/be/src/olap/rowset/alpha_rowset_writer.cpp +++ /dev/null @@ -1,313 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/alpha_rowset_writer.h" - -#include "olap/row.h" -#include "olap/rowset/alpha_rowset.h" -#include "olap/rowset/alpha_rowset_meta.h" -#include "olap/rowset/rowset_factory.h" -#include "olap/rowset/rowset_meta_manager.h" - -namespace doris { - -AlphaRowsetWriter::AlphaRowsetWriter() - : _segment_group_id(0), - _cur_segment_group(nullptr), - _column_data_writer(nullptr), - _current_rowset_meta(nullptr), - _is_pending_rowset(false), - _num_rows_written(0), - _rowset_build(false), - _writer_state(WRITER_CREATED), - _need_column_data_writer(true) {} - -AlphaRowsetWriter::~AlphaRowsetWriter() { - SAFE_DELETE(_column_data_writer); - if (!_rowset_build) { - _garbage_collection(); - } - for (auto& segment_group : _segment_groups) { - segment_group->release(); - delete segment_group; - } - _segment_groups.clear(); -} - -Status AlphaRowsetWriter::init(const RowsetWriterContext& rowset_writer_context) { - _rowset_writer_context = rowset_writer_context; - _current_rowset_meta.reset(new (std::nothrow) AlphaRowsetMeta()); - _current_rowset_meta->set_rowset_id(_rowset_writer_context.rowset_id); - _current_rowset_meta->set_partition_id(_rowset_writer_context.partition_id); - _current_rowset_meta->set_tablet_uid(_rowset_writer_context.tablet_uid); - _current_rowset_meta->set_tablet_id(_rowset_writer_context.tablet_id); - _current_rowset_meta->set_tablet_schema_hash(_rowset_writer_context.tablet_schema_hash); - _current_rowset_meta->set_rowset_type(_rowset_writer_context.rowset_type); - _current_rowset_meta->set_rowset_state(rowset_writer_context.rowset_state); - _current_rowset_meta->set_segments_overlap(rowset_writer_context.segments_overlap); - RowsetStatePB rowset_state = _rowset_writer_context.rowset_state; - if (rowset_state == PREPARED || rowset_state == COMMITTED) { - _is_pending_rowset = true; - } - if (_is_pending_rowset) { - _current_rowset_meta->set_txn_id(_rowset_writer_context.txn_id); - _current_rowset_meta->set_load_id(_rowset_writer_context.load_id); - } else { - _current_rowset_meta->set_version(_rowset_writer_context.version); - } - RETURN_NOT_OK(_init()); - return Status::OK(); -} - -template -Status AlphaRowsetWriter::_add_row(const RowType& row) { - if (_writer_state != WRITER_INITED) { - RETURN_NOT_OK(_init()); - } - Status status = _column_data_writer->write(row); - if (!status.ok()) { - std::string error_msg = "add row failed"; - LOG(WARNING) << error_msg; - return status; - } - ++_num_rows_written; - return Status::OK(); -} - -template Status AlphaRowsetWriter::_add_row(const RowCursor& row); -template Status AlphaRowsetWriter::_add_row(const ContiguousRow& row); - -Status AlphaRowsetWriter::add_rowset(RowsetSharedPtr rowset) { - _need_column_data_writer = false; - // this api is for clone - AlphaRowsetSharedPtr alpha_rowset = std::dynamic_pointer_cast(rowset); - for (auto& segment_group : alpha_rowset->_segment_groups) { - RETURN_NOT_OK(_init()); - RETURN_NOT_OK(segment_group->link_segments_to_path( - _rowset_writer_context.path_desc.filepath, _rowset_writer_context.rowset_id)); - _cur_segment_group->set_empty(segment_group->empty()); - _cur_segment_group->set_num_segments(segment_group->num_segments()); - _cur_segment_group->add_zone_maps(segment_group->get_zone_maps()); - RETURN_NOT_OK(flush()); - _num_rows_written += segment_group->num_rows(); - } - // process delete predicate - if (rowset->rowset_meta()->has_delete_predicate()) { - _current_rowset_meta->set_delete_predicate(rowset->rowset_meta()->delete_predicate()); - } - return Status::OK(); -} - -Status AlphaRowsetWriter::add_rowset_for_linked_schema_change(RowsetSharedPtr rowset, - const SchemaMapping& schema_mapping) { - _need_column_data_writer = false; - // this api is for LinkedSchemaChange - // use create hard link to copy rowset for performance - // this is feasible because LinkedSchemaChange is done on the same disk - AlphaRowsetSharedPtr alpha_rowset = std::dynamic_pointer_cast(rowset); - for (auto& segment_group : alpha_rowset->_segment_groups) { - RETURN_NOT_OK(_init()); - RETURN_NOT_OK(segment_group->link_segments_to_path( - _rowset_writer_context.path_desc.filepath, _rowset_writer_context.rowset_id)); - _cur_segment_group->set_empty(segment_group->empty()); - _cur_segment_group->set_num_segments(segment_group->num_segments()); - _cur_segment_group->add_zone_maps_for_linked_schema_change(segment_group->get_zone_maps(), - schema_mapping); - RETURN_NOT_OK(flush()); - _num_rows_written += segment_group->num_rows(); - } - return Status::OK(); -} - -Status AlphaRowsetWriter::add_rowset_for_migration(RowsetSharedPtr rowset) { - LOG(WARNING) << "alpha_rowset_writer doesn't support add_rowset_for_migration"; - return Status::NotSupported("alpha_rowset_writer doesn't support add_rowset_for_migration"); -} - -Status AlphaRowsetWriter::flush() { - if (_writer_state == WRITER_FLUSHED) { - return Status::OK(); - } - DCHECK(_writer_state == WRITER_INITED); - if (_need_column_data_writer) { - // column_data_writer finalize will call segment_group->set_empty() - RETURN_NOT_OK(_column_data_writer->finalize()); - } - SAFE_DELETE(_column_data_writer); - _writer_state = WRITER_FLUSHED; - return Status::OK(); -} - -RowsetSharedPtr AlphaRowsetWriter::build() { - if (_current_rowset_meta->rowset_id().version == 0) { - LOG(WARNING) << "invalid rowset id, version == 0, rowset id=" - << _current_rowset_meta->rowset_id().to_string(); - return nullptr; - } - if (_writer_state != WRITER_FLUSHED) { - LOG(WARNING) << "invalid writer state before build, state:" << _writer_state; - return nullptr; - } - int total_num_segments = 0; - for (auto& segment_group : _segment_groups) { - if (segment_group->load() != Status::OK()) { - return nullptr; - } - if (!segment_group->check()) { - return nullptr; - } - _current_rowset_meta->set_data_disk_size(_current_rowset_meta->data_disk_size() + - segment_group->data_size()); - _current_rowset_meta->set_index_disk_size(_current_rowset_meta->index_disk_size() + - segment_group->index_size()); - _current_rowset_meta->set_total_disk_size(_current_rowset_meta->total_disk_size() + - segment_group->index_size() + - segment_group->data_size()); - SegmentGroupPB segment_group_pb; - segment_group_pb.set_segment_group_id(segment_group->segment_group_id()); - segment_group_pb.set_num_segments(segment_group->num_segments()); - total_num_segments += segment_group->num_segments(); - segment_group_pb.set_index_size(segment_group->index_size()); - segment_group_pb.set_data_size(segment_group->data_size()); - segment_group_pb.set_num_rows(segment_group->num_rows()); - const std::vector& zone_maps = segment_group->get_zone_maps(); - if (!zone_maps.empty()) { - for (size_t i = 0; i < zone_maps.size(); ++i) { - ZoneMap* new_zone_map = segment_group_pb.add_zone_maps(); - new_zone_map->set_min(zone_maps.at(i).first->to_string()); - new_zone_map->set_max(zone_maps.at(i).second->to_string()); - new_zone_map->set_null_flag(zone_maps.at(i).first->is_null()); - } - } - if (_is_pending_rowset) { - PUniqueId* unique_id = segment_group_pb.mutable_load_id(); - unique_id->set_hi(_rowset_writer_context.load_id.hi()); - unique_id->set_lo(_rowset_writer_context.load_id.lo()); - } - segment_group_pb.set_empty(segment_group->empty()); - AlphaRowsetMetaSharedPtr alpha_rowset_meta = - std::dynamic_pointer_cast(_current_rowset_meta); - alpha_rowset_meta->add_segment_group(segment_group_pb); - } - _current_rowset_meta->set_num_segments(total_num_segments); - if (total_num_segments <= 1) { - _current_rowset_meta->set_segments_overlap(NONOVERLAPPING); - } - if (_is_pending_rowset) { - _current_rowset_meta->set_rowset_state(COMMITTED); - } else { - _current_rowset_meta->set_rowset_state(VISIBLE); - } - - _current_rowset_meta->set_empty(_num_rows_written == 0); - _current_rowset_meta->set_num_rows(_num_rows_written); - _current_rowset_meta->set_creation_time(time(nullptr)); - - // validate rowset arguments before create rowset - bool ret = _validate_rowset(); - if (!ret) { - LOG(FATAL) << "validate rowset arguments failed"; - return nullptr; - } - - RowsetSharedPtr rowset; - auto status = RowsetFactory::create_rowset(_rowset_writer_context.tablet_schema, - _rowset_writer_context.path_desc, - _current_rowset_meta, &rowset); - if (!status.ok()) { - LOG(WARNING) << "rowset init failed when build new rowset, res=" << status; - return nullptr; - } - _rowset_build = true; - return rowset; -} - -Status AlphaRowsetWriter::_garbage_collection() { - for (auto& segment_group : _segment_groups) { - bool ret = segment_group->delete_all_files(); - if (!ret) { - LOG(WARNING) << "delete segment group files failed." - << " tablet id:" << segment_group->get_tablet_id() - << ", rowset path:" << segment_group->rowset_path_prefix(); - return Status::OLAPInternalError(OLAP_ERR_ROWSET_DELETE_FILE_FAILED); - } - } - return Status::OK(); -} - -Status AlphaRowsetWriter::_init() { - if (_writer_state == WRITER_INITED) { - return Status::OK(); - } - if (_is_pending_rowset) { - _cur_segment_group = new (std::nothrow) SegmentGroup( - _rowset_writer_context.tablet_id, _rowset_writer_context.rowset_id, - _rowset_writer_context.tablet_schema, _rowset_writer_context.path_desc.filepath, - false, _segment_group_id, 0, true, _rowset_writer_context.partition_id, - _rowset_writer_context.txn_id); - } else { - _cur_segment_group = new (std::nothrow) SegmentGroup( - _rowset_writer_context.tablet_id, _rowset_writer_context.rowset_id, - _rowset_writer_context.tablet_schema, _rowset_writer_context.path_desc.filepath, - _rowset_writer_context.version, false, _segment_group_id, 0); - } - DCHECK(_cur_segment_group != nullptr) << "failed to malloc SegmentGroup"; - _cur_segment_group->acquire(); - //_cur_segment_group->set_load_id(_rowset_writer_context.load_id); - _segment_groups.push_back(_cur_segment_group); - - _column_data_writer = ColumnDataWriter::create( - _cur_segment_group, true, _rowset_writer_context.tablet_schema->compress_kind(), - _rowset_writer_context.tablet_schema->bloom_filter_fpp()); - DCHECK(_column_data_writer != nullptr) << "memory error occurs when creating writer"; - Status res = _column_data_writer->init(); - if (!res.ok()) { - LOG(WARNING) << "column data writer init failed"; - return res; - } - - _segment_group_id++; - _writer_state = WRITER_INITED; - return Status::OK(); -} - -bool AlphaRowsetWriter::_validate_rowset() { - if (_is_pending_rowset) { - int64_t partition_id = _current_rowset_meta->partition_id(); - if (partition_id <= 0) { - LOG(WARNING) << "invalid partition id:" << partition_id << " for pending rowset." - << ", rowset_id:" << _current_rowset_meta->rowset_id() - << ", tablet_id:" << _current_rowset_meta->tablet_id() - << ", schema_hash:" << _current_rowset_meta->tablet_schema_hash(); - return false; - } - } - int64_t num_rows = 0; - for (auto& segment_group : _segment_groups) { - num_rows += segment_group->num_rows(); - } - if (num_rows != _current_rowset_meta->num_rows()) { - LOG(WARNING) << "num_rows between rowset and segment_groups do not match. " - << "num_rows of segment_groups:" << num_rows - << ", num_rows of rowset:" << _current_rowset_meta->num_rows(); - - return false; - } - return true; -} - -} // namespace doris diff --git a/be/src/olap/rowset/alpha_rowset_writer.h b/be/src/olap/rowset/alpha_rowset_writer.h deleted file mode 100644 index 411beabf62..0000000000 --- a/be/src/olap/rowset/alpha_rowset_writer.h +++ /dev/null @@ -1,85 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "olap/rowset/column_data_writer.h" -#include "olap/rowset/rowset_writer.h" -#include "olap/rowset/segment_group.h" - -namespace doris { - -enum WriterState { WRITER_CREATED, WRITER_INITED, WRITER_FLUSHED }; - -class AlphaRowsetWriter : public RowsetWriter { -public: - AlphaRowsetWriter(); - ~AlphaRowsetWriter() override; - - Status init(const RowsetWriterContext& rowset_writer_context) override; - - Status add_row(const RowCursor& row) override { return _add_row(row); } - Status add_row(const ContiguousRow& row) override { return _add_row(row); } - - // add rowset by create hard link - Status add_rowset(RowsetSharedPtr rowset) override; - Status add_rowset_for_linked_schema_change(RowsetSharedPtr rowset, - const SchemaMapping& schema_mapping) override; - - Status add_rowset_for_migration(RowsetSharedPtr rowset) override; - Status flush() override; - - // get a rowset - RowsetSharedPtr build() override; - - Version version() override { return _rowset_writer_context.version; } - - int64_t num_rows() const override { return _num_rows_written; } - - RowsetId rowset_id() override { return _rowset_writer_context.rowset_id; } - - RowsetTypePB type() const override { return RowsetTypePB::ALPHA_ROWSET; } - -private: - Status _init(); - - template - Status _add_row(const RowType& row); - - // validate rowset build arguments before create rowset to make sure correctness - bool _validate_rowset(); - - Status _garbage_collection(); - -private: - int32_t _segment_group_id; - SegmentGroup* _cur_segment_group; - ColumnDataWriter* _column_data_writer; - std::shared_ptr _current_rowset_meta; - bool _is_pending_rowset; - int64_t _num_rows_written; - RowsetWriterContext _rowset_writer_context; - std::vector _segment_groups; - bool _rowset_build; - WriterState _writer_state; - // add_rowset does not need to call column_data_writer.finalize() - bool _need_column_data_writer; -}; - -} // namespace doris diff --git a/be/src/olap/rowset/bit_field_reader.cpp b/be/src/olap/rowset/bit_field_reader.cpp deleted file mode 100644 index 5fbcad3015..0000000000 --- a/be/src/olap/rowset/bit_field_reader.cpp +++ /dev/null @@ -1,127 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/bit_field_reader.h" - -#include "olap/in_stream.h" -#include "olap/rowset/column_reader.h" -#include "olap/rowset/run_length_byte_reader.h" - -namespace doris { - -BitFieldReader::BitFieldReader(ReadOnlyFileStream* input) - : _input(input), _byte_reader(nullptr), _current('\0'), _bits_left(0) {} - -BitFieldReader::~BitFieldReader() { - SAFE_DELETE(_byte_reader); -} - -Status BitFieldReader::init() { - if (nullptr == _byte_reader) { - _byte_reader = new (std::nothrow) RunLengthByteReader(_input); - - if (nullptr == _byte_reader) { - LOG(WARNING) << "fail to create RunLengthByteReader"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - } - - return Status::OK(); -} - -Status BitFieldReader::_read_byte() { - Status res = Status::OK(); - - if (_byte_reader->has_next()) { - if (!(res = _byte_reader->next(&_current))) { - return res; - } - - _bits_left = 8; - } else { - return Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } - - return Status::OK(); -} - -Status BitFieldReader::next(char* value) { - Status res = Status::OK(); - - if (0 == _bits_left) { - if (!(res = _read_byte())) { - return res; - } - } - - --_bits_left; - - *value = (_current >> _bits_left) & 0x01; - - return Status::OK(); -} - -Status BitFieldReader::seek(PositionProvider* position) { - Status res = Status::OK(); - - if (!(res = _byte_reader->seek(position))) { - return res; - } - - int64_t consumed = position->get_next(); - - if (consumed > 8) { - LOG(WARNING) << "read past end of bit field"; - return Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } else if (consumed != 0) { - if (!(res = _read_byte())) { - return res; - } - - _bits_left = 8 - consumed; - } else { - _bits_left = 0; - } - - return Status::OK(); -} - -Status BitFieldReader::skip(uint64_t num_values) { - Status res = Status::OK(); - - uint64_t total_bits = num_values; - - if (_bits_left >= total_bits) { - _bits_left -= total_bits; - } else { - total_bits -= _bits_left; - - if (!(res = _byte_reader->skip(total_bits / 8))) { - return res; - } - - if (!(res = _byte_reader->next(&_current))) { - return res; - } - - _bits_left = 8 - (total_bits % 8); - } - - return Status::OK(); -} - -} // namespace doris \ No newline at end of file diff --git a/be/src/olap/rowset/bit_field_reader.h b/be/src/olap/rowset/bit_field_reader.h deleted file mode 100644 index 6498ae3826..0000000000 --- a/be/src/olap/rowset/bit_field_reader.h +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_BIT_FIELD_READER_H -#define DORIS_BE_SRC_OLAP_ROWSET_BIT_FIELD_READER_H - -#include "olap/olap_define.h" -#include "olap/stream_index_reader.h" - -namespace doris { - -class ReadOnlyFileStream; -class RunLengthByteReader; - -class BitFieldReader { -public: - BitFieldReader(ReadOnlyFileStream* input); - ~BitFieldReader(); - Status init(); - // 获取下一条数据, 如果没有更多的数据了, 返回Status::OLAPInternalError(OLAP_ERR_DATA_EOF) - // 返回的value只可能是0或1 - Status next(char* value); - Status seek(PositionProvider* position); - Status skip(uint64_t num_values); - -private: - Status _read_byte(); - - ReadOnlyFileStream* _input; - RunLengthByteReader* _byte_reader; - char _current; - uint32_t _bits_left; - - DISALLOW_COPY_AND_ASSIGN(BitFieldReader); -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_BIT_FIELD_READER_H \ No newline at end of file diff --git a/be/src/olap/rowset/bit_field_writer.cpp b/be/src/olap/rowset/bit_field_writer.cpp deleted file mode 100644 index 5dc4553c3a..0000000000 --- a/be/src/olap/rowset/bit_field_writer.cpp +++ /dev/null @@ -1,99 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/bit_field_writer.h" - -#include - -#include "olap/rowset/run_length_byte_writer.h" - -namespace doris { - -BitFieldWriter::BitFieldWriter(OutStream* output) - : _output(output), _byte_writer(nullptr), _current(0), _bits_left(8) {} - -BitFieldWriter::~BitFieldWriter() { - SAFE_DELETE(_byte_writer); -} - -Status BitFieldWriter::init() { - _byte_writer = new (std::nothrow) RunLengthByteWriter(_output); - - if (nullptr == _byte_writer) { - LOG(WARNING) << "fail to create RunLengthByteWriter"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - return Status::OK(); -} - -Status BitFieldWriter::_write_byte() { - Status res = Status::OK(); - - if (!(res = _byte_writer->write(_current))) { - LOG(WARNING) << "fail to write byte to byte writer"; - return res; - } - - _current = 0; - _bits_left = 8; - return res; -} - -Status BitFieldWriter::write(bool bit_value) { - Status res = Status::OK(); - - _bits_left--; - - if (bit_value) { - _current |= 1 << _bits_left; - } - - if (_bits_left == 0) { - res = _write_byte(); - } - - return res; -} - -Status BitFieldWriter::flush() { - Status res = Status::OK(); - - if (_bits_left != 8) { - if (!(res = _write_byte())) { - return res; - } - } - - return _byte_writer->flush(); -} - -void BitFieldWriter::get_position(PositionEntryWriter* index_entry) const { - if (nullptr != _byte_writer) { - _byte_writer->get_position(index_entry); - } else { - // for stream - index_entry->add_position(0); - index_entry->add_position(0); - // for rle byte writer - index_entry->add_position(0); - } - - index_entry->add_position(8 - _bits_left); -} - -} // namespace doris \ No newline at end of file diff --git a/be/src/olap/rowset/bit_field_writer.h b/be/src/olap/rowset/bit_field_writer.h deleted file mode 100644 index 87480c51ab..0000000000 --- a/be/src/olap/rowset/bit_field_writer.h +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_BIT_FIELD_WRITER_H -#define DORIS_BE_SRC_OLAP_ROWSET_BIT_FIELD_WRITER_H - -#include "olap/olap_define.h" -#include "olap/stream_index_writer.h" - -namespace doris { - -class OutStream; -class RunLengthByteWriter; - -class BitFieldWriter { -public: - explicit BitFieldWriter(OutStream* output); - ~BitFieldWriter(); - Status init(); - // Write a bit, bit_value is true means writing 1, false means writing 0 - Status write(bool bit_value); - Status flush(); - void get_position(PositionEntryWriter* index_entry) const; - -private: - Status _write_byte(); - - OutStream* _output; - RunLengthByteWriter* _byte_writer; - char _current; - uint8_t _bits_left; - - DISALLOW_COPY_AND_ASSIGN(BitFieldWriter); -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_BIT_FIELD_WRITER_H \ No newline at end of file diff --git a/be/src/olap/rowset/column_data.cpp b/be/src/olap/rowset/column_data.cpp deleted file mode 100644 index 889581d8a9..0000000000 --- a/be/src/olap/rowset/column_data.cpp +++ /dev/null @@ -1,661 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/column_data.h" - -#include "olap/olap_cond.h" -#include "olap/row_block.h" -#include "olap/rowset/segment_reader.h" -#include "olap/storage_engine.h" - -namespace doris { - -ColumnData* ColumnData::create(SegmentGroup* segment_group) { - ColumnData* data = new (std::nothrow) ColumnData(segment_group); - return data; -} - -ColumnData::ColumnData(SegmentGroup* segment_group) - : _segment_group(segment_group), - _eof(false), - _conditions(nullptr), - _col_predicates(nullptr), - _delete_status(DEL_NOT_SATISFIED), - _runtime_state(nullptr), - _schema(segment_group->get_tablet_schema()), - _is_using_cache(false), - _segment_reader(nullptr), - _lru_cache(nullptr) { - if (StorageEngine::instance() != nullptr) { - _lru_cache = StorageEngine::instance()->index_stream_lru_cache(); - } else { - // for independent usage, eg: unit test/segment tool - _lru_cache = FileHandler::get_fd_cache(); - } - _num_rows_per_block = _segment_group->get_num_rows_per_row_block(); -} - -ColumnData::~ColumnData() { - _segment_group->release(); - SAFE_DELETE(_segment_reader); -} - -Status ColumnData::init() { - _segment_group->acquire(); - - auto res = _short_key_cursor.init(_segment_group->short_key_columns()); - if (!res.ok()) { - LOG(WARNING) << "key cursor init failed, res:" << res; - return res; - } - return res; -} - -Status ColumnData::get_next_block(RowBlock** row_block) { - SCOPED_RAW_TIMER(&_stats->block_fetch_ns); - _is_normal_read = true; - auto res = _get_block(false); - if (!res.ok()) { - if (res.precise_code() != OLAP_ERR_DATA_EOF) { - LOG(WARNING) << "Get next block failed."; - } - *row_block = nullptr; - return res; - } - *row_block = _read_block.get(); - return Status::OK(); -} - -Status ColumnData::_next_row(const RowCursor** row, bool without_filter) { - _read_block->pos_inc(); - do { - if (_read_block->has_remaining()) { - // 1. get one row for vectorized_row_batch - size_t pos = _read_block->pos(); - _read_block->get_row(pos, &_cursor); - if (without_filter) { - *row = &_cursor; - return Status::OK(); - } - - // when without_filter is true, _include_blocks is nullptr - if (_read_block->block_status() == DEL_NOT_SATISFIED) { - *row = &_cursor; - return Status::OK(); - } else { - DCHECK(_read_block->block_status() == DEL_PARTIAL_SATISFIED); - bool row_del_filter = - _delete_handler->is_filter_data(_segment_group->version().second, _cursor); - if (!row_del_filter) { - *row = &_cursor; - return Status::OK(); - } - // This row is filtered, continue to process next row - _stats->rows_del_filtered++; - _read_block->pos_inc(); - } - } else { - // get_next_block - auto res = _get_block(without_filter); - if (!res.ok()) { - return res; - } - } - } while (true); - - return Status::OK(); -} - -Status ColumnData::_seek_to_block(const RowBlockPosition& block_pos, bool without_filter) { - // TODO(zc): _segment_readers??? - // open segment reader if needed - if (_segment_reader == nullptr || block_pos.segment != _current_segment) { - if (block_pos.segment >= _segment_group->num_segments() || - (_end_key_is_set && block_pos.segment > _end_segment)) { - _eof = true; - return Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } - SAFE_DELETE(_segment_reader); - std::string file_name; - file_name = segment_group()->construct_data_file_path(block_pos.segment); - _segment_reader = new (std::nothrow) SegmentReader( - file_name, segment_group(), block_pos.segment, _seek_columns, _load_bf_columns, - _conditions, _delete_handler, _delete_status, _lru_cache, _runtime_state, _stats); - if (_segment_reader == nullptr) { - LOG(WARNING) << "fail to malloc segment reader."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _current_segment = block_pos.segment; - auto res = _segment_reader->init(_is_using_cache); - if (!res.ok()) { - LOG(WARNING) << "fail to init segment reader. res = " << res; - return res; - } - } - - uint32_t end_block; - if (_end_key_is_set && block_pos.segment == _end_segment) { - end_block = _end_block; - } else { - end_block = _segment_reader->block_count() - 1; - } - - VLOG_NOTICE << "seek from " << block_pos.data_offset << " to " << end_block; - return _segment_reader->seek_to_block(block_pos.data_offset, end_block, without_filter, - &_next_block, &_segment_eof); -} - -Status ColumnData::_find_position_by_short_key(const RowCursor& key, bool find_last_key, - RowBlockPosition* position) { - RowBlockPosition tmp_pos; - auto res = _segment_group->find_short_key(key, &_short_key_cursor, find_last_key, &tmp_pos); - if (!res.ok()) { - if (res.precise_code() == OLAP_ERR_INDEX_EOF) { - res = Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } else { - LOG(WARNING) << "find row block failed. res = " << res; - } - return res; - } - res = segment_group()->find_prev_point(tmp_pos, position); - if (!res.ok()) { - LOG(WARNING) << "find prev row block failed. res = " << res; - return res; - } - return Status::OK(); -} - -Status ColumnData::_find_position_by_full_key(const RowCursor& key, bool find_last_key, - RowBlockPosition* position) { - RowBlockPosition tmp_pos; - auto res = _segment_group->find_short_key(key, &_short_key_cursor, false, &tmp_pos); - if (!res.ok()) { - if (res.precise_code() == OLAP_ERR_INDEX_EOF) { - res = Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } else { - LOG(WARNING) << "find row block failed. res = " << res; - } - return res; - } - RowBlockPosition start_position; - res = segment_group()->find_prev_point(tmp_pos, &start_position); - if (!res.ok()) { - LOG(WARNING) << "find prev row block failed. res = " << res; - return res; - } - - RowBlockPosition end_position; - res = _segment_group->find_short_key(key, &_short_key_cursor, true, &end_position); - if (!res.ok()) { - if (res.precise_code() == OLAP_ERR_INDEX_EOF) { - res = Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } else { - LOG(WARNING) << "find row block failed. res = " << res; - } - return res; - } - - // choose min value of end_position and m_end_key_block_position as real end_position - if (_end_key_is_set) { - RowBlockPosition end_key_position; - end_key_position.segment = _end_segment; - end_key_position.data_offset = _end_block; - if (end_position > end_key_position) { - OLAPIndexOffset index_offset; - index_offset.segment = _end_segment; - index_offset.offset = _end_block; - res = segment_group()->get_row_block_position(index_offset, &end_position); - if (!res.ok()) { - LOG(WARNING) << "fail to get row block position. res = " << res; - return res; - } - } - } - - // ????end_position - uint32_t distance = segment_group()->compute_distance(start_position, end_position); - - BinarySearchIterator it_start(0u); - BinarySearchIterator it_end(distance + 1); - BinarySearchIterator it_result(0u); - ColumnDataComparator comparator(start_position, this, segment_group()); - try { - if (!find_last_key) { - it_result = std::lower_bound(it_start, it_end, key, comparator); - } else { - it_result = std::upper_bound(it_start, it_end, key, comparator); - } - VLOG_NOTICE << "get result iterator. offset=" << *it_result - << ", start_pos=" << start_position.to_string(); - } catch (std::exception& e) { - LOG(WARNING) << "exception happens when doing seek. exception=" << e.what(); - return Status::OLAPInternalError(OLAP_ERR_STL_ERROR); - } - - if (*it_result != *it_start) { - it_result -= 1; - } - - if (!(res = segment_group()->advance_row_block(*it_result, &start_position))) { - LOG(WARNING) << "fail to advance row_block. res=" << res << " it_offset=" << *it_result - << " start_pos=" << start_position.to_string(); - return res; - } - - if (_end_key_is_set) { - RowBlockPosition end_key_position; - end_key_position.segment = _end_segment; - end_key_position.data_offset = _end_block; - if (end_position > end_key_position) { - return Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } - } - - *position = start_position; - return Status::OK(); -} - -Status ColumnData::_seek_to_row(const RowCursor& key, bool find_last_key, bool is_end_key) { - RowBlockPosition position; - Status res = Status::OK(); - const TabletSchema& tablet_schema = _segment_group->get_tablet_schema(); - FieldType type = tablet_schema.column(key.field_count() - 1).type(); - if (key.field_count() > _segment_group->get_num_short_key_columns() || - OLAP_FIELD_TYPE_VARCHAR == type || OLAP_FIELD_TYPE_STRING == type) { - res = _find_position_by_full_key(key, find_last_key, &position); - } else { - res = _find_position_by_short_key(key, find_last_key, &position); - } - if (!res.ok()) { - if (res.precise_code() != OLAP_ERR_DATA_EOF) { - LOG(WARNING) << "Fail to find the key.[res=" << res << " key=" << key.to_string() - << " find_last_key=" << find_last_key << "]"; - } - return res; - } - bool without_filter = is_end_key; - res = _seek_to_block(position, without_filter); - if (!res.ok()) { - LOG(WARNING) << "fail to get row block. res=" << res << " segment=" << position.segment - << " block_size=" << position.block_size - << " data_offset=" << position.data_offset - << " index_offset=" << position.index_offset; - return res; - } - res = _get_block(without_filter); - if (!res.ok()) { - if (res.precise_code() != OLAP_ERR_DATA_EOF) { - LOG(WARNING) << "Fail to find the key.[res=" << res << " key=" << key.to_string() - << " find_last_key=" << find_last_key << "]"; - } - return res; - } - - const RowCursor* row_cursor = _current_row(); - if (!find_last_key) { - // 不找last key。 那么应该返回大于等于这个key的第一个，也就是 - // row_cursor >= key - // 此处比较2个block的行数，是存在一种极限情况：若未找到满足的block， - // Index模块会返回倒数第二个block，此时key可能是最后一个block的最后一行 - while (res.ok() && compare_row_key(*row_cursor, key) < 0) { - res = _next_row(&row_cursor, without_filter); - } - } else { - // 找last key。返回大于这个key的第一个。也就是 - // row_cursor > key - while (res.ok() && compare_row_key(*row_cursor, key) <= 0) { - res = _next_row(&row_cursor, without_filter); - } - } - - return res; -} - -const RowCursor* ColumnData::seek_and_get_current_row(const RowBlockPosition& position) { - auto res = _seek_to_block(position, true); - if (!res.ok()) { - LOG(WARNING) << "Fail to seek to block in seek_and_get_current_row, res=" << res - << ", segment:" << position.segment << ", block:" << position.data_offset; - return nullptr; - } - res = _get_block(true, 1); - if (!res.ok()) { - LOG(WARNING) << "Fail to get block in seek_and_get_current_row, res=" << res - << ", segment:" << position.segment << ", block:" << position.data_offset - << ", tablet: " << _segment_group->get_tablet_id(); - return nullptr; - } - return _current_row(); -} - -Status ColumnData::prepare_block_read(const RowCursor* start_key, bool find_start_key, - const RowCursor* end_key, bool find_end_key, - RowBlock** first_block) { - SCOPED_RAW_TIMER(&_stats->block_fetch_ns); - set_eof(false); - _end_key_is_set = false; - _is_normal_read = false; - // set end position - if (end_key != nullptr) { - auto res = _seek_to_row(*end_key, find_end_key, true); - if (res.ok()) { - // we find a - _end_segment = _current_segment; - _end_block = _current_block; - _end_row_index = _read_block->pos(); - _end_key_is_set = true; - } else if (res.precise_code() != OLAP_ERR_DATA_EOF) { - LOG(WARNING) << "Find end key failed.key=" << end_key->to_string(); - return res; - } - // res.precise_code() == OLAP_ERR_DATA_EOF means there is no end key, then we read to - // the end of this ColumnData - } - set_eof(false); - if (start_key != nullptr) { - auto res = _seek_to_row(*start_key, !find_start_key, false); - if (res.ok()) { - *first_block = _read_block.get(); - } else if (res.precise_code() == OLAP_ERR_DATA_EOF) { - _eof = true; - *first_block = nullptr; - return res; - } else { - LOG(WARNING) << "start_key can't be found.key=" << start_key->to_string(); - return res; - } - } else { - // This is used to - _is_normal_read = true; - - RowBlockPosition pos; - pos.segment = 0u; - pos.data_offset = 0u; - auto res = _seek_to_block(pos, false); - if (!res.ok()) { - LOG(WARNING) << "failed to seek to block in, res=" << res << ", segment:" << pos.segment - << ", block:" << pos.data_offset; - return res; - } - res = _get_block(false); - if (!res.ok()) { - LOG(WARNING) << "failed to get block in , res=" << res << ", segment:" << pos.segment - << ", block:" << pos.data_offset; - return res; - } - *first_block = _read_block.get(); - } - return Status::OK(); -} - -// ColumnData向上返回的列至少由几部分组成: -// 1. return_columns中要求返回的列,即Fetch命令中指定要查询的列. -// 2. condition中涉及的列, 绝大多数情况下这些列都已经在return_columns中. -// 在这个函数里,合并上述几种情况 -void ColumnData::set_read_params(const std::vector& return_columns, - const std::vector& seek_columns, - const std::set& load_bf_columns, - const Conditions& conditions, - std::shared_ptr> col_predicates, - bool is_using_cache, RuntimeState* runtime_state) { - _conditions = &conditions; - _col_predicates = col_predicates; - _need_eval_predicates = !col_predicates->empty(); - _is_using_cache = is_using_cache; - _runtime_state = runtime_state; - _return_columns = return_columns; - _seek_columns = seek_columns; - _load_bf_columns = load_bf_columns; - - auto res = _cursor.init(_segment_group->get_tablet_schema(), _seek_columns); - if (!res.ok()) { - LOG(WARNING) << "fail to init row_cursor"; - } - - _read_vector_batch.reset(new VectorizedRowBatch(&(_segment_group->get_tablet_schema()), - _return_columns, _num_rows_per_block)); - - _seek_vector_batch.reset(new VectorizedRowBatch(&(_segment_group->get_tablet_schema()), - _seek_columns, _num_rows_per_block)); - - _read_block.reset(new RowBlock(&(_segment_group->get_tablet_schema()))); - RowBlockInfo block_info; - block_info.row_num = _num_rows_per_block; - block_info.null_supported = true; - block_info.column_ids = _seek_columns; - _read_block->init(block_info); -} - -Status ColumnData::get_first_row_block(RowBlock** row_block) { - DCHECK(!_end_key_is_set) << "end key is set while use block interface."; - _is_normal_read = true; - _eof = false; - - // to be same with OLAPData, we use segment_group. - RowBlockPosition block_pos; - Status res = segment_group()->find_first_row_block(&block_pos); - if (!res.ok()) { - if (res.precise_code() == OLAP_ERR_INDEX_EOF) { - *row_block = nullptr; - _eof = true; - return res; - } - LOG(WARNING) << "fail to find first row block with SegmentGroup."; - return res; - } - - res = _seek_to_block(block_pos, false); - if (!res.ok()) { - if (res.precise_code() != OLAP_ERR_DATA_EOF) { - LOG(WARNING) << "seek to block fail. res = " << res; - } - *row_block = nullptr; - return res; - } - - res = _get_block(false); - if (!res.ok()) { - if (res.precise_code() != OLAP_ERR_DATA_EOF) { - LOG(WARNING) << "fail to load data to row block. res=" << res - << ", version=" << version().first << "-" << version().second; - } - *row_block = nullptr; - return res; - } - - *row_block = _read_block.get(); - return Status::OK(); -} - -bool ColumnData::rowset_pruning_filter() { - if (empty() || zero_num_rows()) { - return true; - } - - if (!_segment_group->has_zone_maps()) { - return false; - } - - return _conditions->rowset_pruning_filter(_segment_group->get_zone_maps()); -} - -int ColumnData::delete_pruning_filter() { - if (empty() || zero_num_rows()) { - // should return DEL_NOT_SATISFIED, because that when creating rollup tablet, - // the delete version file should preserved for filter data. - return DEL_NOT_SATISFIED; - } - - int num_zone_maps = _schema.keys_type() == KeysType::DUP_KEYS ? _schema.num_columns() - : _schema.num_key_columns(); - // _segment_group->get_zone_maps().size() < num_zone_maps for a table is schema changed from older version that not support - // generate zone map for duplicated mode value column, using DEL_PARTIAL_SATISFIED - if (!_segment_group->has_zone_maps() || - _segment_group->get_zone_maps().size() < num_zone_maps) { - /* - * if segment_group has no column statistics, we cannot judge whether the data can be filtered or not - */ - return DEL_PARTIAL_SATISFIED; - } - - /* - * the relationship between delete condition A and B is A || B. - * if any delete condition is satisfied, the data can be filtered. - * elseif all delete condition is not satisfied, the data can't be filtered. - * else is the partial satisfied. - */ - int ret = DEL_PARTIAL_SATISFIED; - bool del_partial_satisfied = false; - bool del_satisfied = false; - for (auto& delete_condition : _delete_handler->get_delete_conditions()) { - if (delete_condition.filter_version <= _segment_group->version().first) { - continue; - } - - Conditions* del_cond = delete_condition.del_cond; - int del_ret = del_cond->delete_pruning_filter(_segment_group->get_zone_maps()); - if (DEL_SATISFIED == del_ret) { - del_satisfied = true; - break; - } else if (DEL_PARTIAL_SATISFIED == del_ret) { - del_partial_satisfied = true; - } else { - continue; - } - } - - if (del_satisfied) { - ret = DEL_SATISFIED; - } else if (del_partial_satisfied) { - ret = DEL_PARTIAL_SATISFIED; - } else { - ret = DEL_NOT_SATISFIED; - } - - return ret; -} - -uint64_t ColumnData::get_filtered_rows() { - return _stats->rows_del_filtered; -} - -Status ColumnData::schema_change_init() { - _is_using_cache = false; - - for (int i = 0; i < _segment_group->get_tablet_schema().num_columns(); ++i) { - _return_columns.push_back(i); - _seek_columns.push_back(i); - } - - auto res = _cursor.init(_segment_group->get_tablet_schema()); - if (!res.ok()) { - LOG(WARNING) << "fail to init row_cursor"; - return res; - } - - _read_vector_batch.reset(new VectorizedRowBatch(&(_segment_group->get_tablet_schema()), - _return_columns, _num_rows_per_block)); - - _read_block.reset(new RowBlock(&(_segment_group->get_tablet_schema()))); - - RowBlockInfo block_info; - block_info.row_num = _num_rows_per_block; - block_info.null_supported = true; - _read_block->init(block_info); - return Status::OK(); -} - -Status ColumnData::_get_block_from_reader(VectorizedRowBatch** got_batch, bool without_filter, - int rows_read) { - VectorizedRowBatch* vec_batch = nullptr; - if (_is_normal_read) { - vec_batch = _read_vector_batch.get(); - } else { - vec_batch = _seek_vector_batch.get(); - } - // If this is normal read - do { - vec_batch->clear(); - if (rows_read > 0) { - vec_batch->set_limit(rows_read); - } - // If we are going to read last block, we need to set batch limit to the end of key - // if without_filter is true and _end_key_is_set is true, this must seek to start row's - // block, we must load the entire block. - if (OLAP_UNLIKELY(!without_filter && _end_key_is_set && _next_block == _end_block && - _current_segment == _end_segment)) { - vec_batch->set_limit(_end_row_index); - if (_end_row_index == 0) { - _segment_eof = true; - } - } - - if (!_segment_eof) { - _current_block = _next_block; - auto res = _segment_reader->get_block(vec_batch, &_next_block, &_segment_eof); - if (!res.ok()) { - return res; - } - // Normal case - *got_batch = vec_batch; - return Status::OK(); - } - // When this segment is read over, we reach here. - // Seek to next segment - RowBlockPosition block_pos; - block_pos.segment = _current_segment + 1; - block_pos.data_offset = 0; - auto res = _seek_to_block(block_pos, without_filter); - if (!res.ok()) { - return res; - } - } while (true); - - return Status::OK(); -} - -Status ColumnData::_get_block(bool without_filter, int rows_read) { - do { - VectorizedRowBatch* vec_batch = nullptr; - auto res = _get_block_from_reader(&vec_batch, without_filter, rows_read); - if (!res.ok()) { - return res; - } - // evaluate predicates - if (!without_filter && _need_eval_predicates) { - SCOPED_RAW_TIMER(&_stats->vec_cond_ns); - size_t old_size = vec_batch->size(); - for (auto pred : *_col_predicates) { - pred->evaluate(vec_batch); - } - _stats->rows_vec_cond_filtered += old_size - vec_batch->size(); - } - // if vector is empty after predicate evaluate, get next block - if (vec_batch->size() == 0) { - continue; - } - SCOPED_RAW_TIMER(&_stats->block_convert_ns); - // when reach here, we have already read a block successfully - _read_block->clear(); - vec_batch->dump_to_row_block(_read_block.get()); - return Status::OK(); - } while (true); - return Status::OK(); -} - -} // namespace doris diff --git a/be/src/olap/rowset/column_data.h b/be/src/olap/rowset/column_data.h deleted file mode 100644 index c2c91fab79..0000000000 --- a/be/src/olap/rowset/column_data.h +++ /dev/null @@ -1,230 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_COLUMN_DATA_H -#define DORIS_BE_SRC_OLAP_ROWSET_COLUMN_DATA_H - -#include - -#include "gen_cpp/olap_file.pb.h" -#include "olap/column_predicate.h" -#include "olap/delete_handler.h" -#include "olap/olap_common.h" -#include "olap/olap_cond.h" -#include "olap/row.h" -#include "olap/row_block.h" -#include "olap/row_cursor.h" -#include "olap/rowset/segment_group.h" -#include "util/runtime_profile.h" - -namespace doris { - -class Tablet; -class SegmentReader; - -// This class is column data reader. this class will be used in two case. -class ColumnData { -public: - static ColumnData* create(SegmentGroup* segment_group); - ColumnData(SegmentGroup* segment_group); - ~ColumnData(); - - // 为了与之前兼容, 暴露部分index的接口 - Version version() const { return _segment_group->version(); } - bool delete_flag() const { return _segment_group->delete_flag(); } - uint32_t num_segments() const { return _segment_group->num_segments(); } - - Status init(); - - Status prepare_block_read(const RowCursor* start_key, bool find_start_key, - const RowCursor* end_key, bool find_end_key, RowBlock** first_block); - - Status get_next_block(RowBlock** row_block); - - void set_read_params(const std::vector& return_columns, - const std::vector& seek_columns, - const std::set& load_bf_columns, const Conditions& conditions, - std::shared_ptr> col_predicates, - bool is_using_cache, RuntimeState* runtime_state); - - Status get_first_row_block(RowBlock** row_block); - - // Only used to binary search in full-key find row - const RowCursor* seek_and_get_current_row(const RowBlockPosition& position); - - void set_using_cache(bool is_using_cache) { _is_using_cache = is_using_cache; } - - void set_stats(OlapReaderStatistics* stats) { _stats = stats; } - - void set_delete_handler(const DeleteHandler* delete_handler) { - _delete_handler = delete_handler; - } - - void set_delete_status(const DelCondSatisfied delete_status) { _delete_status = delete_status; } - - // 开放接口查询_eof，让外界知道数据读取是否正常终止 - // 因为这个函数被频繁访问, 从性能考虑, 放在基类而不是虚函数 - bool eof() { return _eof; } - void set_eof(bool eof) { _eof = eof; } - bool* eof_ptr() { return &_eof; } - - bool empty() const { return _segment_group->empty(); } - bool zero_num_rows() const { return _segment_group->zero_num_rows(); } - - // Return true if should be filtered out - bool rowset_pruning_filter(); - int delete_pruning_filter(); - uint64_t get_filtered_rows(); - - SegmentGroup* segment_group() const { return _segment_group; } - void set_segment_group(SegmentGroup* segment_group) { _segment_group = segment_group; } - int64_t num_rows() const { return _segment_group->num_rows(); } - - // To compatible with schema change read, use this function to init column data - // for schema change read. Only called in get_first_row_block - Status schema_change_init(); - -private: - DISALLOW_COPY_AND_ASSIGN(ColumnData); - - // Try to seek to 'key'. If this function returned with OLAP_SUCCESS, current_row() - // point to the first row meet the requirement. - // If there is no such row, Status::OLAPInternalError(OLAP_ERR_DATA_EOF) will return. - // If error happened, other code will return - Status _seek_to_row(const RowCursor& key, bool find_key, bool is_end_key); - - // seek to block_pos without load that block, caller must call _get_block() - // to load _read_block with data. If without_filter is false, this will seek to - // other block. Because the seeked block may be filtered by condition or delete. - Status _seek_to_block(const RowBlockPosition& block_pos, bool without_filter); - - Status _find_position_by_short_key(const RowCursor& key, bool find_last_key, - RowBlockPosition* position); - Status _find_position_by_full_key(const RowCursor& key, bool find_last_key, - RowBlockPosition* position); - - // Used in _seek_to_row, this function will goto next row that valid for this - // ColumnData - Status _next_row(const RowCursor** row, bool without_filter); - - // get block from reader, just read vector batch from _current_segment. - // The read batch return by got_batch. - Status _get_block_from_reader(VectorizedRowBatch** got_batch, bool without_filter, - int rows_read); - - // get block from segment reader. If this function returns OLAP_SUCCESS - Status _get_block(bool without_filter, int rows_read = 0); - - const RowCursor* _current_row() { - _read_block->get_row(_read_block->pos(), &_cursor); - return &_cursor; - } - -private: - SegmentGroup* _segment_group; - // 当到达文件末尾或者到达end key时设置此标志 - bool _eof; - const Conditions* _conditions; - std::shared_ptr> _col_predicates; - const DeleteHandler* _delete_handler = nullptr; - DelCondSatisfied _delete_status; - RuntimeState* _runtime_state; - OlapReaderStatistics* _stats; - - const TabletSchema& _schema; - // whether in normal read, use return columns to load block - bool _is_normal_read = false; - bool _end_key_is_set = false; - bool _is_using_cache; - bool _segment_eof = false; - bool _need_eval_predicates = false; - - std::vector _return_columns; - std::vector _seek_columns; - std::set _load_bf_columns; - - SegmentReader* _segment_reader; - - std::unique_ptr _seek_vector_batch; - std::unique_ptr _read_vector_batch; - - std::unique_ptr _read_block = nullptr; - RowCursor _cursor; - RowCursor _short_key_cursor; - - // Record when last key is found - uint32_t _current_block = 0; - uint32_t _current_segment; - uint32_t _next_block; - - uint32_t _end_segment; - uint32_t _end_block; - int64_t _end_row_index = 0; - - size_t _num_rows_per_block; - Cache* _lru_cache; -}; - -class ColumnDataComparator { -public: - ColumnDataComparator(RowBlockPosition position, ColumnData* olap_data, - const SegmentGroup* segment_group) - : _start_block_position(position), - _olap_data(olap_data), - _segment_group(segment_group) {} - - ~ColumnDataComparator() {} - - // less comparator function - bool operator()(const iterator_offset_t& index, const RowCursor& key) const { - return _compare(index, key, COMPARATOR_LESS); - } - // larger comparator function - bool operator()(const RowCursor& key, const iterator_offset_t& index) const { - return _compare(index, key, COMPARATOR_LARGER); - } - -private: - bool _compare(const iterator_offset_t& index, const RowCursor& key, - ComparatorEnum comparator_enum) const { - Status res = Status::OK(); - RowBlockPosition position = _start_block_position; - if (!(res = _segment_group->advance_row_block(index, &position))) { - LOG(WARNING) << "fail to advance row block. res=" << res; - throw ComparatorException(); - } - const RowCursor* helper_cursor = _olap_data->seek_and_get_current_row(position); - if (helper_cursor == nullptr) { - LOG(WARNING) << "fail to seek and get current row."; - throw ComparatorException(); - } - - if (COMPARATOR_LESS == comparator_enum) { - return compare_row_key(*helper_cursor, key) < 0; - } else { - return compare_row_key(*helper_cursor, key) > 0; - } - } - - const RowBlockPosition _start_block_position; - ColumnData* _olap_data; - const SegmentGroup* _segment_group; -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_COLUMN_DATA_H diff --git a/be/src/olap/rowset/column_data_writer.cpp b/be/src/olap/rowset/column_data_writer.cpp deleted file mode 100644 index f1eb60b36d..0000000000 --- a/be/src/olap/rowset/column_data_writer.cpp +++ /dev/null @@ -1,320 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/column_data_writer.h" - -#include - -#include "olap/row.h" -#include "olap/row_block.h" -#include "olap/rowset/segment_group.h" -#include "olap/rowset/segment_writer.h" - -namespace doris { - -ColumnDataWriter* ColumnDataWriter::create(SegmentGroup* segment_group, bool is_push_write, - CompressKind compress_kind, double bloom_filter_fpp) { - ColumnDataWriter* writer = new (std::nothrow) - ColumnDataWriter(segment_group, is_push_write, compress_kind, bloom_filter_fpp); - return writer; -} - -ColumnDataWriter::ColumnDataWriter(SegmentGroup* segment_group, bool is_push_write, - CompressKind compress_kind, double bloom_filter_fpp) - : _segment_group(segment_group), - _is_push_write(is_push_write), - _compress_kind(compress_kind), - _bloom_filter_fpp(bloom_filter_fpp), - _zone_maps(segment_group->get_num_zone_map_columns(), KeyRange(nullptr, nullptr)), - _row_index(0), - _row_block(nullptr), - _segment_writer(nullptr), - _num_rows(0), - _block_id(0), - _max_segment_size(OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE), - _segment(0), - _all_num_rows(0), - _new_segment_created(false) {} - -ColumnDataWriter::~ColumnDataWriter() { - for (size_t i = 0; i < _zone_maps.size(); ++i) { - SAFE_DELETE(_zone_maps[i].first); - SAFE_DELETE(_zone_maps[i].second); - } - SAFE_DELETE(_row_block); - SAFE_DELETE(_segment_writer); -} - -Status ColumnDataWriter::init() { - Status res = Status::OK(); - - for (size_t i = 0; i < _zone_maps.size(); ++i) { - _zone_maps[i].first = WrapperField::create(_segment_group->get_tablet_schema().column(i)); - DCHECK(_zone_maps[i].first != nullptr) << "fail to create column statistics field."; - _zone_maps[i].first->set_to_max(); - - _zone_maps[i].second = WrapperField::create(_segment_group->get_tablet_schema().column(i)); - DCHECK(_zone_maps[i].second != nullptr) << "fail to create column statistics field."; - _zone_maps[i].second->set_null(); - _zone_maps[i].second->set_to_min(); - } - - double size = static_cast(OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE); - size *= OLAP_COLUMN_FILE_SEGMENT_SIZE_SCALE; - _max_segment_size = static_cast(lround(size)); - - _row_block = new (std::nothrow) RowBlock(&(_segment_group->get_tablet_schema())); - - if (nullptr == _row_block) { - LOG(WARNING) << "fail to new RowBlock."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - res = _cursor.init(_segment_group->get_tablet_schema()); - if (!res.ok()) { - LOG(WARNING) << "fail to initiate row cursor. [res=" << res << "]"; - return res; - } - - VLOG_NOTICE << "init ColumnData writer. segment_group_id=" << _segment_group->segment_group_id() - << ", block_row_number=" << _segment_group->get_num_rows_per_row_block(); - RowBlockInfo block_info(0U, _segment_group->get_num_rows_per_row_block()); - block_info.null_supported = true; - - _row_block->init(block_info); - return Status::OK(); -} - -Status ColumnDataWriter::_init_segment() { - Status res = _add_segment(); - if (!res.ok()) { - LOG(WARNING) << "fail to add segment. [res=" << res << "]"; - return res; - } - - res = _segment_group->add_segment(); - if (!res.ok()) { - LOG(WARNING) << "fail to add index segment. [res=" << res << "]"; - return res; - } - - _new_segment_created = true; - return res; -} - -template -Status ColumnDataWriter::write(const RowType& row) { - // copy input row to row block - _row_block->get_row(_row_index, &_cursor); - copy_row(&_cursor, row, _row_block->mem_pool()); - next(row); - if (_row_index >= _segment_group->get_num_rows_per_row_block()) { - if (!_flush_row_block(false)) { - LOG(WARNING) << "failed to flush data while attaching row cursor."; - return Status::OLAPInternalError(OLAP_ERR_OTHER_ERROR); - } - RETURN_NOT_OK(_flush_segment_with_verification()); - } - return Status::OK(); -} - -template -void ColumnDataWriter::next(const RowType& row) { - for (size_t cid = 0; cid < _segment_group->get_num_zone_map_columns(); ++cid) { - auto field = row.schema()->column(cid); - auto cell = row.cell(cid); - - if (field->compare_cell(*_zone_maps[cid].first, cell) > 0) { - field->direct_copy(_zone_maps[cid].first, cell); - } - - if (field->compare_cell(*_zone_maps[cid].second, cell) < 0) { - field->direct_copy(_zone_maps[cid].second, cell); - } - } - - ++_row_index; -} - -Status ColumnDataWriter::finalize() { - if (_all_num_rows == 0 && _row_index == 0) { - _segment_group->set_empty(true); - return Status::OK(); - } - - // Segment which size reaches OLAP_MAX_COLUMN_SEGMENT_FILE_SIZE - // will be flushed into disk. If the previous segment reach - // the threshold just right, and been flushed into disk. - // The following finalize() when closing ColumnDataWriter - // will generate a non-sense segment. - // In this scenario, undefined behavior will happens. - if (_num_rows == 0 && _row_index == 0) { - // If the two conditions are all satisfied, - // it dedicates that there is no necessity - // to generate segment object and file. - // Return OLAP_SUCCESS is OK. - return Status::OK(); - } - - Status res = _flush_row_block(true); - if (!res.ok()) { - LOG(WARNING) << "failed to flush data while attaching row cursor.res = " << res; - return res; - } - - res = _finalize_segment(); - if (!res.ok()) { - LOG(WARNING) << "fail to finalize segment. res=" << res << ", _row_index=" << _row_index - << ", _all_num_rows=" << _all_num_rows; - return res; - } - - res = _segment_group->add_zone_maps(_zone_maps); - if (!res.ok()) { - LOG(WARNING) << "Fail to set zone_map! res=" << res; - return res; - } - - return Status::OK(); -} - -Status ColumnDataWriter::_flush_row_block(bool finalize) { - if (!_new_segment_created) { - RETURN_NOT_OK(_init_segment()); - } - - if (_row_index < 1) { - return Status::OK(); - } - // 与OLAPDataWriter不同,这里不是真的写RowBlock,所以并不需要finalize RowBlock - // 但考虑到兼容Row Block的使用方式,还是调用了finalize - Status res = _row_block->finalize(_row_index); - if (!res.ok()) { - LOG(WARNING) << "fail to finalize row block. num_rows=" << _row_index << "res=" << res; - return Status::OLAPInternalError(OLAP_ERR_WRITER_ROW_BLOCK_ERROR); - } - - // 目标是将自己的block按条写入目标block中。 - res = _segment_writer->write_batch(_row_block, &_cursor, finalize); - if (!res.ok()) { - LOG(WARNING) << "fail to write row to segment. res = " << res; - return Status::OLAPInternalError(OLAP_ERR_WRITER_DATA_WRITE_ERROR); - } - - // 在SegmentGroup中记录的不是数据文件的偏移,而是block的编号 - if (!_segment_group->add_row_block(*_row_block, _block_id++)) { - LOG(WARNING) << "fail to update index."; - return Status::OLAPInternalError(OLAP_ERR_WRITER_INDEX_WRITE_ERROR); - } - - // In order to reuse row_block, clear the row_block after finalize - _row_block->clear(); - _num_rows += _row_index; - _all_num_rows += _row_index; - _row_index = 0; - return Status::OK(); -} - -Status ColumnDataWriter::_add_segment() { - std::string file_name; - - if (nullptr != _segment_writer) { - LOG(WARNING) << "previous segment is not finalized before add new segment."; - return Status::OLAPInternalError(OLAP_ERR_WRITER_SEGMENT_NOT_FINALIZED); - } - - file_name = _segment_group->construct_data_file_path(_segment); - _segment_writer = new (std::nothrow) - SegmentWriter(file_name, _segment_group, OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, - _compress_kind, _bloom_filter_fpp); - - if (nullptr == _segment_writer) { - LOG(WARNING) << "fail to allocate SegmentWriter"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - Status res = Status::OK(); - if (_is_push_write) { - res = _segment_writer->init(config::push_write_mbytes_per_sec); - } else { - res = _segment_writer->init(config::base_compaction_write_mbytes_per_sec); - } - - if (!res.ok()) { - LOG(WARNING) << "fail to init segment writer"; - return res; - } - - ++_segment; - _block_id = 0; - return Status::OK(); -} - -Status ColumnDataWriter::_flush_segment_with_verification() { - uint64_t segment_size = _segment_writer->estimate_segment_size(); - if (UNLIKELY(segment_size < _max_segment_size)) { - return Status::OK(); - } - - Status res = _finalize_segment(); - if (!res.ok()) { - LOG(WARNING) << "fail to finalize segment. res = " << res; - return res; - } - - _new_segment_created = false; - _num_rows = 0; - return Status::OK(); -} - -Status ColumnDataWriter::_finalize_segment() { - uint32_t data_segment_size; - Status res = _segment_writer->finalize(&data_segment_size); - if (res != Status::OK()) { - LOG(WARNING) << "fail to finish segment from olap_data."; - return res; - } - res = _segment_group->finalize_segment(data_segment_size, _num_rows); - if (res != Status::OK()) { - LOG(WARNING) << "fail to finish segment from olap_index."; - return res; - } - - SAFE_DELETE(_segment_writer); - return res; -} - -uint64_t ColumnDataWriter::written_bytes() { - uint64_t size = _segment * _max_segment_size + _segment_writer->estimate_segment_size(); - return size; -} - -MemPool* ColumnDataWriter::mem_pool() { - return _row_block->mem_pool(); -} - -CompressKind ColumnDataWriter::compress_kind() { - return _compress_kind; -} - -template Status ColumnDataWriter::write(const RowCursor& row); -template Status ColumnDataWriter::write(const ContiguousRow& row); - -template void ColumnDataWriter::next(const RowCursor& row); -template void ColumnDataWriter::next(const ContiguousRow& row); - -} // namespace doris diff --git a/be/src/olap/rowset/column_data_writer.h b/be/src/olap/rowset/column_data_writer.h deleted file mode 100644 index 66b72c69ce..0000000000 --- a/be/src/olap/rowset/column_data_writer.h +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_COLUMN_DATA_WRITER_H -#define DORIS_BE_SRC_OLAP_ROWSET_COLUMN_DATA_WRITER_H - -#include "gen_cpp/olap_common.pb.h" -#include "olap/row_block.h" -#include "olap/rowset/segment_group.h" -#include "olap/schema.h" -#include "olap/wrapper_field.h" - -namespace doris { -class RowBlock; -class SegmentWriter; - -class ColumnDataWriter { -public: - // Factory function - // 调用者获得新建的对象, 并负责delete释放 - static ColumnDataWriter* create(SegmentGroup* segment_group, bool is_push_write, - CompressKind compress_kind, double bloom_filter_fpp); - ColumnDataWriter(SegmentGroup* segment_group, bool is_push_write, CompressKind compress_kind, - double bloom_filter_fpp); - ~ColumnDataWriter(); - Status init(); - - template - Status write(const RowType& row); - - template - void next(const RowType& row); - - Status finalize(); - uint64_t written_bytes(); - MemPool* mem_pool(); - CompressKind compress_kind(); - -private: - Status _add_segment(); - Status _flush_segment_with_verification(); - Status _finalize_segment(); - Status _flush_row_block(bool finalize); - Status _init_segment(); - -private: - SegmentGroup* _segment_group; - bool _is_push_write; - CompressKind _compress_kind; - double _bloom_filter_fpp; - // first is min, second is max - std::vector> _zone_maps; - uint32_t _row_index; - - RowBlock* _row_block; // 使用RowBlock缓存要写入的数据 - RowCursor _cursor; - SegmentWriter* _segment_writer; - int64_t _num_rows; - uint32_t _block_id; // 当前Segment内的block编号 - uint32_t _max_segment_size; - uint32_t _segment; - int64_t _all_num_rows; - bool _new_segment_created; -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_COLUMN_DATA_WRITER_H diff --git a/be/src/olap/rowset/column_reader.cpp b/be/src/olap/rowset/column_reader.cpp deleted file mode 100644 index 8c6cc1676c..0000000000 --- a/be/src/olap/rowset/column_reader.cpp +++ /dev/null @@ -1,1186 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/column_reader.h" - -#include - -#include "olap/file_stream.h" -#include "olap/olap_define.h" -#include "olap/rowset/bit_field_reader.h" - -namespace doris { -IntegerColumnReader::IntegerColumnReader(uint32_t column_unique_id) - : _eof(false), _column_unique_id(column_unique_id), _data_reader(nullptr) {} - -IntegerColumnReader::~IntegerColumnReader() { - SAFE_DELETE(_data_reader); -} - -Status IntegerColumnReader::init(std::map* streams, bool is_sign) { - if (nullptr == streams) { - LOG(WARNING) << "input streams is nullptr"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - // Get data stream according to column id and type - ReadOnlyFileStream* data_stream = - extract_stream(_column_unique_id, StreamInfoMessage::DATA, streams); - - if (data_stream == nullptr) { - LOG(WARNING) << "specified stream is nullptr"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - _data_reader = new (std::nothrow) RunLengthIntegerReader(data_stream, is_sign); - - if (nullptr == _data_reader) { - LOG(WARNING) << "fail to malloc RunLengthIntegerReader"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - // reset eof flag when init, to support reinit - return Status::OK(); -} - -Status IntegerColumnReader::seek(PositionProvider* position) { - return _data_reader->seek(position); -} - -Status IntegerColumnReader::skip(uint64_t row_count) { - return _data_reader->skip(row_count); -} - -Status IntegerColumnReader::next(int64_t* value) { - return _data_reader->next(value); -} - -StringColumnDirectReader::StringColumnDirectReader(uint32_t column_unique_id, - uint32_t dictionary_size) - : _column_unique_id(column_unique_id), - _values(nullptr), - _data_stream(nullptr), - _length_reader(nullptr) {} - -StringColumnDirectReader::~StringColumnDirectReader() { - SAFE_DELETE(_length_reader); -} - -Status StringColumnDirectReader::init(std::map* streams, int size, - MemPool* mem_pool) { - if (nullptr == streams) { - LOG(WARNING) << "input streams is nullptr"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - // Get data stream according to column id and type - _data_stream = extract_stream(_column_unique_id, StreamInfoMessage::DATA, streams); - - if (nullptr == _data_stream) { - LOG(WARNING) << "specified stream not found. [unique_id = " << _column_unique_id << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(Slice))); - - ReadOnlyFileStream* length_stream = - extract_stream(_column_unique_id, StreamInfoMessage::LENGTH, streams); - - if (nullptr == length_stream) { - LOG(WARNING) << "specified stream not found. [unique_id = " << _column_unique_id << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - _length_reader = new (std::nothrow) RunLengthIntegerReader(length_stream, false); - - if (nullptr == _length_reader) { - LOG(WARNING) << "fail to malloc RunLengthIntegerReader"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - return Status::OK(); -} - -Status StringColumnDirectReader::seek(PositionProvider* position) { - Status res = _data_stream->seek(position); - - // All strings in segment may be empty, so the data stream is EOF and - // and length stream is not EOF. - if (res.ok() || OLAP_ERR_COLUMN_STREAM_EOF == res.precise_code()) { - res = _length_reader->seek(position); - } - - return res; -} - -Status StringColumnDirectReader::skip(uint64_t row_count) { - Status res = Status::OK(); - int64_t skip_length = 0; - int64_t tmp_length = 0; - - for (size_t i = 0; i < row_count; ++i) { - res = _length_reader->next(&tmp_length); - - if (!res.ok()) { - return res; - } - - skip_length += tmp_length; - } - - if (res.ok()) { - // TODO: skip function of instream is implemented, but not tested - return _data_stream->skip(skip_length); - } - - return res; -} - -// Return string field of current row_count -Status StringColumnDirectReader::next(char* buffer, uint32_t* length) { - int64_t read_length = 0; - Status res = _length_reader->next(&read_length); - *length = read_length; - while (res.ok() && read_length > 0) { - uint64_t buf_size = read_length; - res = _data_stream->read(buffer, &buf_size); - read_length -= buf_size; - buffer += buf_size; - } - *length -= read_length; - return res; -} - -Status StringColumnDirectReader::next_vector(ColumnVector* column_vector, uint32_t size, - MemPool* mem_pool, int64_t* read_bytes) { - /* - * MemPool here is not the same as MemPool in init function - * 1. MemPool is created by VectorizedRowBatch, - * and reset when load row batch - * 2. MemPool in init function is created by SegmentReader, - * and free by SegmentReader deconstruction. - */ - Status res = Status::OK(); - int64_t length = 0; - int64_t string_buffer_size = 0; - - column_vector->set_col_data(_values); - if (column_vector->no_nulls()) { - for (int i = 0; i < size; ++i) { - res = _length_reader->next(&length); - if (!res.ok()) { - return res; - } - _values[i].size = length; - string_buffer_size += length; - } - - uint8_t* allocated_mem = nullptr; - res = mem_pool->allocate_safely(string_buffer_size, allocated_mem); - if (!res.ok()) { - return res; - } - char* string_buffer = reinterpret_cast(allocated_mem); - - for (int i = 0; i < size; ++i) { - length = _values[i].size; - if (UNLIKELY(length == 0)) { - _values[i].data = nullptr; - } else { - _values[i].data = string_buffer; - } - while (length > 0) { - uint64_t buf_size = length; - res = _data_stream->read(string_buffer, &buf_size); - if (!res.ok()) { - return res; - } - length -= buf_size; - string_buffer += buf_size; - } - } - } else { - bool* is_null = column_vector->is_null(); - for (int i = 0; i < size; ++i) { - if (!is_null[i]) { - res = _length_reader->next(&length); - if (!res.ok()) { - return res; - } - _values[i].size = length; - string_buffer_size += length; - } else { - _values[i].size = 0; - } - } - - uint8_t* allocated_mem = nullptr; - res = mem_pool->allocate_safely(string_buffer_size, allocated_mem); - if (!res.ok()) { - return res; - } - char* string_buffer = reinterpret_cast(allocated_mem); - - for (int i = 0; i < size; ++i) { - if (!is_null[i]) { - length = _values[i].size; - if (UNLIKELY(length == 0)) { - _values[i].data = nullptr; - } else { - _values[i].data = string_buffer; - } - while (length > 0) { - uint64_t buf_size = length; - res = _data_stream->read(string_buffer, &buf_size); - if (!res.ok()) { - return res; - } - length -= buf_size; - string_buffer += buf_size; - } - } else { - _values[i].data = nullptr; - _values[i].size = 0; - } - } - } - *read_bytes += string_buffer_size; - - return res; -} - -StringColumnDictionaryReader::StringColumnDictionaryReader(uint32_t column_unique_id, - uint32_t dictionary_size) - : _eof(false), - _dictionary_size(dictionary_size), - _column_unique_id(column_unique_id), - _values(nullptr), - //_dictionary_size(0), - //_offset_dictionary(nullptr), - //_dictionary_data_buffer(nullptr), - _read_buffer(nullptr), - _data_reader(nullptr) {} - -StringColumnDictionaryReader::~StringColumnDictionaryReader() { - //SAFE_DELETE_ARRAY(_offset_dictionary); - //SAFE_DELETE(_dictionary_data_buffer); - SAFE_DELETE(_data_reader); - SAFE_DELETE_ARRAY(_read_buffer); -} - -Status StringColumnDictionaryReader::init(std::map* streams, - int size, MemPool* mem_pool) { - ReadOnlyFileStream* dictionary_data_stream = - extract_stream(_column_unique_id, StreamInfoMessage::DICTIONARY_DATA, streams); - - if (nullptr == dictionary_data_stream) { - LOG(WARNING) << "dictionary data stream not found. [unique id = " << _column_unique_id - << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - ReadOnlyFileStream* dictionary_length_stream = - extract_stream(_column_unique_id, StreamInfoMessage::LENGTH, streams); - - if (nullptr == dictionary_length_stream) { - LOG(WARNING) << "dictionary length stream not found. [unique id = " << _column_unique_id - << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - RunLengthIntegerReader* dictionary_length_reader = - new (std::nothrow) RunLengthIntegerReader(dictionary_length_stream, false); - Status res = Status::OK(); - - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(Slice))); - int64_t read_buffer_size = 1024; - char* _read_buffer = new (std::nothrow) char[read_buffer_size]; - - if (nullptr == _read_buffer) { - LOG(WARNING) << "fail to malloc read buffer. [size = " << read_buffer_size << "]"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - int64_t length = 0; - uint64_t read_length = 0; - std::string dictionary_item; - - for (size_t dictionary_entry = 0; dictionary_entry < _dictionary_size; ++dictionary_entry) { - res = dictionary_length_reader->next(&length); - // 理论上应该足够读，读出eof也是不对的。 - if (!res.ok() || length < 0) { - LOG(WARNING) << "build offset dictionary failed. res = " << res; - return res; - } - - if (length > read_buffer_size) { - SAFE_DELETE_ARRAY(_read_buffer); - read_buffer_size = length; - - if (nullptr == (_read_buffer = new (std::nothrow) char[read_buffer_size])) { - LOG(WARNING) << "fail to malloc read buffer. [size = " << read_buffer_size << "]"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - } - - read_length = length; - dictionary_data_stream->read(_read_buffer, &read_length); - - if (static_cast(read_length) != length) { - LOG(WARNING) << "read stream fail."; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_READ_STREAM); - } - - dictionary_item.assign(_read_buffer, length); - _dictionary.push_back(dictionary_item); - } - - // 建立数据流读取器 - ReadOnlyFileStream* data_stream = - extract_stream(_column_unique_id, StreamInfoMessage::DATA, streams); - - if (nullptr == data_stream) { - LOG(WARNING) << "data stream not found. [unique id = " << _column_unique_id << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - _data_reader = new (std::nothrow) RunLengthIntegerReader(data_stream, false); - - if (nullptr == _data_reader) { - LOG(WARNING) << "fail to malloc data reader"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - SAFE_DELETE_ARRAY(_read_buffer); - SAFE_DELETE(dictionary_length_reader); - return Status::OK(); -} - -Status StringColumnDictionaryReader::seek(PositionProvider* position) { - return _data_reader->seek(position); -} - -Status StringColumnDictionaryReader::skip(uint64_t row_count) { - return _data_reader->skip(row_count); -} - -Status StringColumnDictionaryReader::next(char* buffer, uint32_t* length) { - int64_t value; - Status res = _data_reader->next(&value); - // 错误或是EOF - if (!res.ok()) { - if (OLAP_ERR_DATA_EOF == res.precise_code()) { - _eof = true; - } - - return res; - } - - if (value >= static_cast(_dictionary.size())) { - LOG(WARNING) << "value may indicated an invalid dictionary entry. " - "[value = " - << value << ", dictionary_size = " << _dictionary.size() << "]"; - return Status::OLAPInternalError(OLAP_ERR_BUFFER_OVERFLOW); - } - - memcpy(buffer, _dictionary[value].c_str(), _dictionary[value].size()); - *length = _dictionary[value].size(); - - return Status::OK(); -} - -Status StringColumnDictionaryReader::next_vector(ColumnVector* column_vector, uint32_t size, - MemPool* mem_pool, int64_t* read_bytes) { - int64_t index[size]; - int64_t buffer_size = 0; - Status res = Status::OK(); - - column_vector->set_col_data(_values); - if (column_vector->no_nulls()) { - for (int i = 0; i < size; ++i) { - res = _data_reader->next(&index[i]); - if (!res.ok()) { - return res; - } - if (index[i] >= static_cast(_dictionary.size())) { - LOG(WARNING) << "value may indicated an invalid dictionary entry. " - "[index = " - << index[i] << ", dictionary_size = " << _dictionary.size() << "]"; - return Status::OLAPInternalError(OLAP_ERR_BUFFER_OVERFLOW); - } - _values[i].size = _dictionary[index[i]].size(); - buffer_size += _values[i].size; - } - - char* string_buffer = reinterpret_cast(mem_pool->allocate(buffer_size)); - for (int i = 0; i < size; ++i) { - memory_copy(string_buffer, _dictionary[index[i]].c_str(), _values[i].size); - _values[i].data = string_buffer; - string_buffer += _values[i].size; - } - } else { - bool* is_null = column_vector->is_null(); - for (int i = 0; i < size; ++i) { - if (!is_null[i]) { - res = _data_reader->next(&index[i]); - if (!res.ok()) { - return res; - } - if (index[i] >= static_cast(_dictionary.size())) { - LOG(WARNING) << "value may indicated an invalid dictionary entry. " - "[index = " - << index[i] << ", dictionary_size = " << _dictionary.size() << "]"; - return Status::OLAPInternalError(OLAP_ERR_BUFFER_OVERFLOW); - } - _values[i].size = _dictionary[index[i]].size(); - buffer_size += _values[i].size; - } - } - - char* string_buffer = reinterpret_cast(mem_pool->allocate(buffer_size)); - for (int i = 0; i < size; ++i) { - if (!is_null[i]) { - memory_copy(string_buffer, _dictionary[index[i]].c_str(), _values[i].size); - _values[i].data = string_buffer; - string_buffer += _values[i].size; - } - } - } - *read_bytes += buffer_size; - - return res; -} - -ColumnReader::ColumnReader(uint32_t column_id, uint32_t column_unique_id) - : _value_present(false), - _is_null(nullptr), - _column_id(column_id), - _column_unique_id(column_unique_id), - _present_reader(nullptr) {} - -ColumnReader* ColumnReader::create(uint32_t column_id, const TabletSchema& schema, - const UniqueIdToColumnIdMap& included, - UniqueIdToColumnIdMap& segment_included, - const UniqueIdEncodingMap& encodings) { - return create(column_id, schema.columns(), included, segment_included, encodings); -} - -ColumnReader* ColumnReader::create(uint32_t column_id, const std::vector& schema, - const UniqueIdToColumnIdMap& included, - UniqueIdToColumnIdMap& segment_included, - const UniqueIdEncodingMap& encodings) { - if (column_id >= schema.size()) { - LOG(WARNING) << "invalid column_id, column_id=" << column_id - << ", columns_size=" << schema.size(); - return nullptr; - } - - const TabletColumn& column = schema[column_id]; - ColumnReader* reader = nullptr; - int32_t column_unique_id = column.unique_id(); - - if (0 == included.count(column_unique_id)) { - return nullptr; - } - - if (0 == segment_included.count(column_unique_id)) { - if (column.has_default_value()) { - if (0 == strcasecmp("NULL", column.default_value().c_str()) && column.is_nullable()) { - return new (std::nothrow) NullValueReader(column_id, column_unique_id); - } else { - return new (std::nothrow) - DefaultValueReader(column_id, column_unique_id, column.default_value(), - column.type(), column.length()); - } - } else if (column.is_nullable()) { - return new (std::nothrow) NullValueReader(column_id, column_unique_id); - } else { - LOG(WARNING) << "not null field has no default value"; - return nullptr; - } - } - - uint32_t dictionary_size = 0; - ColumnEncodingMessage::Kind encode_kind = ColumnEncodingMessage::DIRECT; - UniqueIdEncodingMap::const_iterator it = encodings.find(column_unique_id); - - if (it != encodings.end()) { - encode_kind = (*it).second.kind(); - dictionary_size = (*it).second.dictionary_size(); - } - - switch (column.type()) { - case OLAP_FIELD_TYPE_BOOL: - case OLAP_FIELD_TYPE_TINYINT: - case OLAP_FIELD_TYPE_UNSIGNED_TINYINT: { - reader = new (std::nothrow) TinyColumnReader(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_SMALLINT: { - reader = new (std::nothrow) - IntegerColumnReaderWrapper(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_UNSIGNED_SMALLINT: { - reader = new (std::nothrow) - IntegerColumnReaderWrapper(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_INT: { - reader = new (std::nothrow) - IntegerColumnReaderWrapper(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_UNSIGNED_INT: { - reader = new (std::nothrow) - IntegerColumnReaderWrapper(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_BIGINT: { - reader = new (std::nothrow) - IntegerColumnReaderWrapper(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_UNSIGNED_BIGINT: { - reader = new (std::nothrow) - IntegerColumnReaderWrapper(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_FLOAT: { - reader = new (std::nothrow) FloatColumnReader(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_DOUBLE: { - reader = new (std::nothrow) DoubleColumnReader(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_DISCRETE_DOUBLE: { - reader = new (std::nothrow) DiscreteDoubleColumnReader(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_CHAR: { - if (ColumnEncodingMessage::DIRECT == encode_kind) { - reader = new (std::nothrow) FixLengthStringColumnReader( - column_id, column_unique_id, column.length(), dictionary_size); - } else if (ColumnEncodingMessage::DICTIONARY == encode_kind) { - reader = new (std::nothrow) FixLengthStringColumnReader( - column_id, column_unique_id, column.length(), dictionary_size); - } else { - LOG(WARNING) << "known encoding format. data may be generated by higher version,try " - "updating olap/ngine binary to solve this problem"; - // TODO. define a new return code - return nullptr; - } - - break; - } - - case OLAP_FIELD_TYPE_DATETIME: { - reader = new (std::nothrow) DateTimeColumnReader(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_DATE: { - reader = new (std::nothrow) DateColumnReader(column_id, column_unique_id); - - break; - } - - case OLAP_FIELD_TYPE_DATEV2: { - reader = new (std::nothrow) DateV2ColumnReader(column_id, column_unique_id); - - break; - } - - case OLAP_FIELD_TYPE_DECIMAL: { - reader = new (std::nothrow) DecimalColumnReader(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_LARGEINT: { - reader = new (std::nothrow) LargeIntColumnReader(column_id, column_unique_id); - break; - } - - case OLAP_FIELD_TYPE_VARCHAR: - case OLAP_FIELD_TYPE_OBJECT: - case OLAP_FIELD_TYPE_HLL: - case OLAP_FIELD_TYPE_STRING: { - if (ColumnEncodingMessage::DIRECT == encode_kind) { - reader = new (std::nothrow) VarStringColumnReader( - column_id, column_unique_id, column.length(), dictionary_size); - } else if (ColumnEncodingMessage::DICTIONARY == encode_kind) { - reader = new (std::nothrow) VarStringColumnReader( - column_id, column_unique_id, column.length(), dictionary_size); - } else { - LOG(WARNING) << "known encoding format. data may be generated by higher version, " - << "try updating olap/ngine binary to solve this problem"; - // TODO. define a new return code - return nullptr; - } - - break; - } - - case OLAP_FIELD_TYPE_STRUCT: - case OLAP_FIELD_TYPE_ARRAY: - case OLAP_FIELD_TYPE_MAP: - default: { - LOG(WARNING) << "unsupported field type. field=" << column.name() - << ", type=" << column.type(); - break; - } - } - - return reader; -} - -ColumnReader::~ColumnReader() { - SAFE_DELETE(_present_reader); -} - -Status ColumnReader::init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats) { - if (nullptr == streams) { - LOG(WARNING) << "null parameters given."; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - _stats = stats; - - // 从map中找到需要的流，ColumnReader的数据应该由一条PRESENT流和一条ROW_INDEX流组成 - ReadOnlyFileStream* present_stream = - extract_stream(_column_unique_id, StreamInfoMessage::PRESENT, streams); - - _is_null = reinterpret_cast(mem_pool->allocate(size)); - memset(_is_null, 0, size); - - if (nullptr == present_stream) { - _present_reader = nullptr; - _value_present = false; - } else { - VLOG_TRACE << "create null present_stream for column_id:" << _column_unique_id; - _present_reader = new (std::nothrow) BitFieldReader(present_stream); - - if (nullptr == _present_reader) { - LOG(WARNING) << "malloc present reader failed."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - if (!_present_reader->init()) { - LOG(WARNING) << "fail to init present reader."; - return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); - } - - _value_present = true; - } - - return Status::OK(); -} - -Status ColumnReader::seek(PositionProvider* position) { - if (nullptr != _present_reader) { - return _present_reader->seek(position); - } - - return Status::OK(); -} - -Status ColumnReader::skip(uint64_t row_count) { - return Status::OK(); -} - -Status ColumnReader::next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool) { - Status res = Status::OK(); - column_vector->set_is_null(_is_null); - if (nullptr != _present_reader) { - column_vector->set_no_nulls(false); - for (uint32_t i = 0; i < size; ++i) { - bool value = false; - res = _present_reader->next((char*)&value); - if (!res.ok()) { - break; - } - _is_null[i] = value; - } - _stats->bytes_read += size; - } else { - column_vector->set_no_nulls(true); - } - - return res; -} - -uint64_t ColumnReader::_count_none_nulls(uint64_t rows) { - if (_present_reader != nullptr) { - Status res = Status::OK(); - uint64_t result = 0; - - for (uint64_t counter = 0; counter < rows; ++counter) { - res = _present_reader->next(reinterpret_cast(&_value_present)); - - if (res.ok() && (false == _value_present)) { - result += 1; - } else { - break; - } - } - - return result; - } else { - return rows; - } -} - -TinyColumnReader::TinyColumnReader(uint32_t column_id, uint32_t column_unique_id) - : ColumnReader(column_id, column_unique_id), - _eof(false), - _values(nullptr), - _data_reader(nullptr) {} - -TinyColumnReader::~TinyColumnReader() { - SAFE_DELETE(_data_reader); -} - -Status TinyColumnReader::init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats) { - if (nullptr == streams) { - LOG(WARNING) << "input streams is nullptr"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - ColumnReader::init(streams, size, mem_pool, stats); - ReadOnlyFileStream* data_stream = - extract_stream(_column_unique_id, StreamInfoMessage::DATA, streams); - - if (nullptr == data_stream) { - LOG(WARNING) << "specified stream not exist"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - _values = reinterpret_cast(mem_pool->allocate(size)); - _data_reader = new (std::nothrow) RunLengthByteReader(data_stream); - - if (nullptr == _data_reader) { - LOG(WARNING) << "malloc data reader failed"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - return Status::OK(); -} - -Status TinyColumnReader::seek(PositionProvider* positions) { - Status res; - if (nullptr == _present_reader) { - res = _data_reader->seek(positions); - if (!res.ok()) { - return res; - } - } else { - res = ColumnReader::seek(positions); - if (!res.ok()) { - return res; - } - res = _data_reader->seek(positions); - if (!res.ok() && OLAP_ERR_COLUMN_STREAM_EOF != res.precise_code()) { - LOG(WARNING) << "fail to seek tinyint stream. res = " << res; - return res; - } - } - - return Status::OK(); -} - -Status TinyColumnReader::skip(uint64_t row_count) { - // count_none_nulls 其实就是columnReader的跳过函数。 - return _data_reader->skip(_count_none_nulls(row_count)); -} - -Status TinyColumnReader::next_vector(ColumnVector* column_vector, uint32_t size, - MemPool* mem_pool) { - Status res = ColumnReader::next_vector(column_vector, size, mem_pool); - if (!res.ok()) { - if (OLAP_ERR_DATA_EOF == res.precise_code()) { - _eof = true; - } - return res; - } - - bool* is_null = column_vector->is_null(); - column_vector->set_col_data(_values); - if (column_vector->no_nulls()) { - for (uint32_t i = 0; i < size; ++i) { - res = _data_reader->next(_values + i); - if (!res.ok()) { - break; - } - } - } else { - for (uint32_t i = 0; i < size; ++i) { - if (!is_null[i]) { - res = _data_reader->next(_values + i); - if (!res.ok()) { - break; - } - } - } - } - _stats->bytes_read += size; - - if (OLAP_ERR_DATA_EOF == res.precise_code()) { - _eof = true; - } - - return res; -} - -DecimalColumnReader::DecimalColumnReader(uint32_t column_id, uint32_t column_unique_id) - : ColumnReader(column_id, column_unique_id), - _eof(false), - _values(nullptr), - _int_reader(nullptr), - _frac_reader(nullptr) {} - -DecimalColumnReader::~DecimalColumnReader() { - SAFE_DELETE(_int_reader); - SAFE_DELETE(_frac_reader); -} - -Status DecimalColumnReader::init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats) { - if (nullptr == streams) { - LOG(WARNING) << "input streams is nullptr"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - // reset stream and reader - ColumnReader::init(streams, size, mem_pool, stats); - - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(decimal12_t))); - - // 从map中找到需要的流，StringColumnReader的数据应该由一条DATA流和一条LENGTH流组成 - ReadOnlyFileStream* int_stream = - extract_stream(_column_unique_id, StreamInfoMessage::DATA, streams); - - if (nullptr == int_stream) { - LOG(WARNING) << "specified stream not found. [unique_id = " << _column_unique_id << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - ReadOnlyFileStream* frac_stream = - extract_stream(_column_unique_id, StreamInfoMessage::SECONDARY, streams); - - if (nullptr == frac_stream) { - LOG(WARNING) << "specified stream not found. [unique_id = " << _column_unique_id << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - _int_reader = new (std::nothrow) RunLengthIntegerReader(int_stream, true); - - if (nullptr == _int_reader) { - LOG(WARNING) << "fail to malloc RunLengthIntegerReader"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _frac_reader = new (std::nothrow) RunLengthIntegerReader(frac_stream, true); - - if (nullptr == _frac_reader) { - LOG(WARNING) << "fail to malloc RunLengthIntegerReader"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - return Status::OK(); -} - -Status DecimalColumnReader::seek(PositionProvider* positions) { - Status res; - if (nullptr == _present_reader) { - res = _int_reader->seek(positions); - if (!res.ok()) { - return res; - } - - res = _frac_reader->seek(positions); - - if (!res.ok()) { - return res; - } - } else { - //all field in the segment can be nullptr, so the data stream is EOF - res = ColumnReader::seek(positions); - if (!res.ok()) { - return res; - } - res = _int_reader->seek(positions); - if (!res.ok() && OLAP_ERR_COLUMN_STREAM_EOF != res.precise_code()) { - LOG(WARNING) << "fail to seek int stream of decimal. res = " << res; - return res; - } - - res = _frac_reader->seek(positions); - if (!res.ok() && OLAP_ERR_COLUMN_STREAM_EOF != res.precise_code()) { - LOG(WARNING) << "fail to seek frac stream of decimal. res = " << res; - return res; - } - } - - return Status::OK(); -} - -Status DecimalColumnReader::skip(uint64_t row_count) { - Status res = _int_reader->skip(row_count); - - if (!res.ok()) { - LOG(WARNING) << "fail to create int part reader"; - return res; - } - - res = _frac_reader->skip(row_count); - - if (!res.ok()) { - LOG(WARNING) << "fail to create frac part reader"; - return res; - } - - return Status::OK(); -} - -Status DecimalColumnReader::next_vector(ColumnVector* column_vector, uint32_t size, - MemPool* mem_pool) { - Status res = ColumnReader::next_vector(column_vector, size, mem_pool); - if (!res.ok()) { - if (OLAP_ERR_DATA_EOF == res.precise_code()) { - _eof = true; - } - return res; - } - - bool* is_null = column_vector->is_null(); - column_vector->set_col_data(_values); - - if (column_vector->no_nulls()) { - for (uint32_t i = 0; i < size; ++i) { - int64_t value = 0; - Status res = _int_reader->next(&value); - if (!res.ok()) { - LOG(WARNING) << "fail to read decimal int part"; - break; - } - _values[i].integer = value; - - res = _frac_reader->next(&value); - if (!res.ok()) { - LOG(WARNING) << "fail to read decimal frac part"; - break; - } - _values[i].fraction = value; - } - } else { - for (uint32_t i = 0; i < size; ++i) { - int64_t value = 0; - if (!is_null[i]) { - Status res = _int_reader->next(&value); - if (!res.ok()) { - LOG(WARNING) << "fail to read decimal int part"; - break; - } - _values[i].integer = value; - - res = _frac_reader->next(&value); - if (!res.ok()) { - LOG(WARNING) << "fail to read decimal frac part"; - break; - } - _values[i].fraction = value; - } - } - } - _stats->bytes_read += sizeof(decimal12_t) * size; - - return res; -} - -LargeIntColumnReader::LargeIntColumnReader(uint32_t column_id, uint32_t column_unique_id) - : ColumnReader(column_id, column_unique_id), - _eof(false), - _values(nullptr), - _high_reader(nullptr), - _low_reader(nullptr) {} - -LargeIntColumnReader::~LargeIntColumnReader() { - SAFE_DELETE(_high_reader); - SAFE_DELETE(_low_reader); -} - -Status LargeIntColumnReader::init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats) { - if (nullptr == streams) { - LOG(WARNING) << "input streams is nullptr"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - // reset stream and reader - ColumnReader::init(streams, size, mem_pool, stats); - - _values = reinterpret_cast( - mem_pool->try_allocate_aligned(size * sizeof(int128_t), alignof(int128_t))); - - // 从map中找到需要的流，LargeIntColumnReader的数据应该由一条DATA流组成 - ReadOnlyFileStream* high_stream = - extract_stream(_column_unique_id, StreamInfoMessage::DATA, streams); - if (nullptr == high_stream) { - LOG(WARNING) << "specified stream not found. [unique_id = " << _column_unique_id << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - ReadOnlyFileStream* low_stream = - extract_stream(_column_unique_id, StreamInfoMessage::SECONDARY, streams); - if (nullptr == low_stream) { - LOG(WARNING) << "specified stream not found. [unique_id = " << _column_unique_id << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - _high_reader = new (std::nothrow) RunLengthIntegerReader(high_stream, true); - if (nullptr == _high_reader) { - LOG(WARNING) << "fail to malloc RunLengthIntegerReader."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _low_reader = new (std::nothrow) RunLengthIntegerReader(low_stream, true); - if (nullptr == _low_reader) { - LOG(WARNING) << "fail to malloc RunLengthIntegerReader."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - return Status::OK(); -} - -Status LargeIntColumnReader::seek(PositionProvider* positions) { - Status res; - if (nullptr == _present_reader) { - res = _high_reader->seek(positions); - if (!res.ok()) { - return res; - } - - res = _low_reader->seek(positions); - if (!res.ok()) { - return res; - } - } else { - //all field in the segment can be nullptr, so the data stream is EOF - res = ColumnReader::seek(positions); - if (!res.ok()) { - LOG(WARNING) << "fail to seek null stream of largeint"; - return res; - } - - res = _high_reader->seek(positions); - if (!res.ok() && OLAP_ERR_COLUMN_STREAM_EOF != res.precise_code()) { - LOG(WARNING) << "fail to seek high int stream of largeint. res = " << res; - return res; - } - - res = _low_reader->seek(positions); - if (!res.ok() && OLAP_ERR_COLUMN_STREAM_EOF != res.precise_code()) { - LOG(WARNING) << "fail to seek low int stream of largeint. res = " << res; - return res; - } - } - - return Status::OK(); -} - -Status LargeIntColumnReader::skip(uint64_t row_count) { - Status res = _high_reader->skip(row_count); - if (!res.ok()) { - LOG(WARNING) << "fail to skip large int high part. res = " << res; - return res; - } - - res = _low_reader->skip(row_count); - if (!res.ok()) { - LOG(WARNING) << "fail to skip large int low part reader. res = " << res; - return res; - } - - return Status::OK(); -} - -Status LargeIntColumnReader::next_vector(ColumnVector* column_vector, uint32_t size, - MemPool* mem_pool) { - Status res = ColumnReader::next_vector(column_vector, size, mem_pool); - if (!res.ok()) { - if (OLAP_ERR_DATA_EOF == res.precise_code()) { - _eof = true; - } - return res; - } - - bool* is_null = column_vector->is_null(); - column_vector->set_col_data(_values); - - if (column_vector->no_nulls()) { - for (uint32_t i = 0; i < size; ++i) { - int64_t* value = nullptr; - value = (int64_t*)(_values + i); - res = _high_reader->next(value); - if (!res.ok()) { - LOG(WARNING) << "fail to read decimal int part"; - break; - } - - res = _low_reader->next(++value); - if (!res.ok()) { - LOG(WARNING) << "fail to read decimal frac part"; - break; - } - } - } else { - for (uint32_t i = 0; i < size; ++i) { - int64_t* value = nullptr; - if (!is_null[i]) { - value = (int64_t*)(_values + i); - res = _high_reader->next(value); - if (!res.ok()) { - LOG(WARNING) << "fail to read decimal int part"; - break; - } - - res = _low_reader->next(++value); - if (!res.ok()) { - LOG(WARNING) << "fail to read decimal frac part"; - break; - } - } - } - } - _stats->bytes_read += 16 * size; - - return res; -} - -} // namespace doris diff --git a/be/src/olap/rowset/column_reader.h b/be/src/olap/rowset/column_reader.h deleted file mode 100644 index 2880cef335..0000000000 --- a/be/src/olap/rowset/column_reader.h +++ /dev/null @@ -1,841 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_COLUMN_READER_H -#define DORIS_BE_SRC_OLAP_ROWSET_COLUMN_READER_H - -#include "olap/file_stream.h" -#include "olap/olap_common.h" -#include "olap/rowset/run_length_byte_reader.h" -#include "olap/rowset/run_length_integer_reader.h" -#include "olap/stream_name.h" -#include "runtime/large_int_value.h" -#include "runtime/vectorized_row_batch.h" -#include "util/date_func.h" - -namespace doris { - -class StreamName; -class ReadOnlyFileStream; -class BitFieldReader; -class RowIndexEntryMessage; -class ColumnEncodingMessage; - -// Solution flow -inline ReadOnlyFileStream* extract_stream(uint32_t column_unique_id, StreamInfoMessage::Kind kind, - std::map* streams) { - StreamName stream_name(column_unique_id, kind); - std::map::iterator it = streams->find(stream_name); - - if (streams->end() != it) { - return (*it).second; - } - - return nullptr; -} - -// Unique id -> PositionProvider -typedef std::unordered_map UniqueIdPositionProviderMap; -// Unique id -> ColumnEncodingMessage -typedef std::map UniqueIdEncodingMap; - -// Readers for Integer and String. -// Although these readers are also named Reader, note that they do not inherit from ColumnReader -// Therefore, the case of null values is not considered. - -// For SHORT/INT/LONG type data, use int64 as the stored data uniformly -// Due to the use of variable length coding, it will not cause waste -// IntegerColumnReader is a reader that reads the int64 data of the bottom layer, when the upper layer returns -// Use IntColumnReaderWrapper to convert to a specific data type -// -// NOTE. Since the RLE reader can only read int64, which is different from java, java shaping does not consider symbols -// Then this thing actually seems to be � -// �� method supports unsigned integer shaping, you need to pay attention to whether to modify RLEReader in the future -class IntegerColumnReader { -public: - IntegerColumnReader(uint32_t column_unique_id); - ~IntegerColumnReader(); - /** - * Initialize the Integer column reader - * @param streams contains the map of the required stream - * @param is_sign whether the number read has a sign - * @return [description] - */ - Status init(std::map* streams, bool is_sign); - // Position the internal pointer to positions - Status seek(PositionProvider* positions); - // Move the internal pointer back row_count rows - Status skip(uint64_t row_count); - // Return the data of the current row by moving the internal pointer to the next row - Status next(int64_t* value); - bool eof() { return _eof; } - -private: - bool _eof; - uint32_t _column_unique_id; - RunLengthIntegerReader* _data_reader; -}; - -// For readers of string columns encoded in Direct mode -// Direct method of String can be read directly -class StringColumnDirectReader { -public: - StringColumnDirectReader(uint32_t column_unique_id, uint32_t dictionary_size); - ~StringColumnDirectReader(); - - Status init(std::map* streams, int size, MemPool* mem_pool); - Status seek(PositionProvider* positions); - Status skip(uint64_t row_count); - // Return the data of the current row and move the internal pointer backward - // buffer - the buffer of the returned data - // length - the size of the buffer area when input, and the size of the string when returning - Status next(char* buffer, uint32_t* length); - Status next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool, - int64_t* read_bytes); - - size_t get_buffer_size() { return sizeof(RunLengthByteReader); } - -private: - uint32_t _column_unique_id; - Slice* _values; - ReadOnlyFileStream* _data_stream; - RunLengthIntegerReader* _length_reader; -}; - -// For readers using dictionary-encoded string columns -// The interface is the same as StringColumnDirectReader -// Reading process: -// 1. Read all the dictionary data and save it in the whole buffer -// 2. Read the length data, construct the offset dictionary, the offset is � -// ��The beginning of each string, combined with 1 can read data -// 3. Read the actual saved data when needed. -// �� is an int). Find the offset according to this int, and then read the dictionary data according to the offset -class StringColumnDictionaryReader { -public: - StringColumnDictionaryReader(uint32_t column_unique_id, uint32_t dictionary_size); - ~StringColumnDictionaryReader(); - Status init(std::map* streams, int size, MemPool* mem_pool); - Status seek(PositionProvider* positions); - Status skip(uint64_t row_count); - Status next(char* buffer, uint32_t* length); - Status next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool, - int64_t* read_bytes); - - size_t get_buffer_size() { return sizeof(RunLengthByteReader) + _dictionary_size; } - -private: - bool _eof; - uint32_t _dictionary_size; - uint32_t _column_unique_id; - Slice* _values; - char* _read_buffer; - //uint64_t _dictionary_size; - //uint64_t* _offset_dictionary; // The offset corresponding to the number used to find the response data - //StorageByteBuffer* _dictionary_data_buffer; // Save dict data - std::vector _dictionary; - // Used to read the actual data (represented by an integer) - RunLengthIntegerReader* _data_reader; -}; - -// ColumnReader is used to read a column and is the base class of other XXXColumnReader -// ColumnReader maintains the NULL feature of the column through the bit field presented -class ColumnReader { -public: - // Factory method, create ColumnReader, if the column has sub-columns, recursively create a sub reader - // If the column to be read does not exist in segment_columns, then: - // 1.If the column allows Null values, create a NullValueReader - // 2.If the column does not allow Null values, but has a default value, create a DefaultValueReader - // 3.Otherwise the creation fails - // Input: - // column_id - the position of the column to be created in the columns - // columns - the schema of the table - // included - column to be created, if the unique id of a column is included in included - // segment_columns - a collection of unique ids of all columns in the segment - // encodings - column encoding information, use encodings[_column_unique_id] to access - static ColumnReader* create(uint32_t column_id, const TabletSchema& schema, - const UniqueIdToColumnIdMap& included, - UniqueIdToColumnIdMap& segment_included, - const UniqueIdEncodingMap& encodings); - - static ColumnReader* create(uint32_t column_id, const std::vector& schema, - const UniqueIdToColumnIdMap& included, - UniqueIdToColumnIdMap& segment_included, - const UniqueIdEncodingMap& encodings); - - ColumnReader(uint32_t column_id, uint32_t column_unique_id); - virtual ~ColumnReader(); - - // Use streams to initialize Reader - // ColumnReader is initialized only once, and a new object is allocated each time it is used. - // Input: - // streams-input stream - virtual Status init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats); - - // Set the position of the next returned data - // positions are the positions where each column needs to seek, ColumnReader passes (*positions)[_column_unique_id] - // Get the seek position of this column - virtual Status seek(PositionProvider* positions); - - virtual Status skip(uint64_t row_count); - - virtual Status next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool); - - uint32_t column_unique_id() { return _column_unique_id; } - - uint32_t column_id() { return _column_id; } - - virtual size_t get_buffer_size() { return 0; } - -protected: - // NOTE. Count the non-blank rows in rows. This is because "blank lines" do not exist in actual storage. - // So for upper-level fields that may be empty (such as integer), the caller wants to skip 10 lines, - // but actually for - uint64_t _count_none_nulls(uint64_t rows); - - bool _value_present; - bool* _is_null; - uint32_t _column_id; // The id of the column in the schema - uint32_t _column_unique_id; // the unique id of the column - BitFieldReader* _present_reader; // NULL value of NULLabel field - std::vector _sub_readers; - OlapReaderStatistics* _stats = nullptr; -}; - -class DefaultValueReader : public ColumnReader { -public: - DefaultValueReader(uint32_t column_id, uint32_t column_unique_id, std::string default_value, - FieldType type, int length) - : ColumnReader(column_id, column_unique_id), - _default_value(default_value), - _values(nullptr), - _type(type), - _length(length) {} - - virtual ~DefaultValueReader() {} - - virtual Status init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats) { - switch (_type) { - case OLAP_FIELD_TYPE_TINYINT: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(int8_t))); - int32_t value = 0; - std::stringstream ss(_default_value); - ss >> value; - for (int i = 0; i < size; ++i) { - ((int8_t*)_values)[i] = value; - } - break; - } - case OLAP_FIELD_TYPE_SMALLINT: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(int16_t))); - int16_t value = 0; - std::stringstream ss(_default_value); - ss >> value; - for (int i = 0; i < size; ++i) { - ((int16_t*)_values)[i] = value; - } - break; - } - case OLAP_FIELD_TYPE_INT: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(int32_t))); - int32_t value = 0; - std::stringstream ss(_default_value); - ss >> value; - for (int i = 0; i < size; ++i) { - ((int32_t*)_values)[i] = value; - } - break; - } - case OLAP_FIELD_TYPE_BIGINT: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(int64_t))); - int64_t value = 0; - std::stringstream ss(_default_value); - ss >> value; - for (int i = 0; i < size; ++i) { - ((int64_t*)_values)[i] = value; - } - break; - } - case OLAP_FIELD_TYPE_LARGEINT: { - _values = reinterpret_cast( - mem_pool->try_allocate_aligned(size * sizeof(int128_t), alignof(int128_t))); - int128_t value = 0; - std::stringstream ss(_default_value); - ss >> value; - for (int i = 0; i < size; ++i) { - ((int128_t*)_values)[i] = value; - } - break; - } - case OLAP_FIELD_TYPE_FLOAT: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(float))); - float value = 0; - std::stringstream ss(_default_value); - ss >> value; - for (int i = 0; i < size; ++i) { - ((float*)_values)[i] = value; - } - break; - } - case OLAP_FIELD_TYPE_DOUBLE: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(double))); - double value = 0; - std::stringstream ss(_default_value); - ss >> value; - for (int i = 0; i < size; ++i) { - ((double*)_values)[i] = value; - } - break; - } - case OLAP_FIELD_TYPE_DECIMAL: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(decimal12_t))); - decimal12_t value = {0, 0}; - value.from_string(_default_value); - for (int i = 0; i < size; ++i) { - ((decimal12_t*)_values)[i] = value; - } - break; - } - case OLAP_FIELD_TYPE_CHAR: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(Slice))); - int32_t length = _length; - char* string_buffer = reinterpret_cast(mem_pool->allocate(size * length)); - memset(string_buffer, 0, size * length); - for (int i = 0; i < size; ++i) { - memory_copy(string_buffer, _default_value.c_str(), _default_value.length()); - ((Slice*)_values)[i].size = length; - ((Slice*)_values)[i].data = string_buffer; - string_buffer += length; - } - break; - } - case OLAP_FIELD_TYPE_VARCHAR: - case OLAP_FIELD_TYPE_OBJECT: - case OLAP_FIELD_TYPE_HLL: - case OLAP_FIELD_TYPE_STRING: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(Slice))); - int32_t length = _default_value.length(); - char* string_buffer = reinterpret_cast(mem_pool->allocate(size * length)); - for (int i = 0; i < size; ++i) { - memory_copy(string_buffer, _default_value.c_str(), length); - ((Slice*)_values)[i].size = length; - ((Slice*)_values)[i].data = string_buffer; - string_buffer += length; - } - break; - } - case OLAP_FIELD_TYPE_DATE: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(uint24_t))); - uint24_t value = timestamp_from_date(_default_value); - for (int i = 0; i < size; ++i) { - ((uint24_t*)_values)[i] = value; - } - break; - } - case OLAP_FIELD_TYPE_DATEV2: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(uint32_t))); - uint32_t value = 0; - std::stringstream ss(_default_value); - ss >> value; - for (int i = 0; i < size; ++i) { - ((uint32_t*)_values)[i] = value; - } - break; - } - case OLAP_FIELD_TYPE_DATETIME: { - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(uint64_t))); - uint64_t value = timestamp_from_datetime(_default_value); - for (int i = 0; i < size; ++i) { - ((uint64_t*)_values)[i] = value; - } - break; - } - default: - break; - } - _stats = stats; - return Status::OK(); - } - virtual Status seek(PositionProvider* positions) { return Status::OK(); } - virtual Status skip(uint64_t row_count) { return Status::OK(); } - - virtual Status next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool) { - column_vector->set_no_nulls(true); - column_vector->set_col_data(_values); - _stats->bytes_read += _length * size; - return Status::OK(); - } - -private: - std::string _default_value; - void* _values; - FieldType _type; - int32_t _length; -}; - -class NullValueReader : public ColumnReader { -public: - NullValueReader(uint32_t column_id, uint32_t column_unique_id) - : ColumnReader(column_id, column_unique_id) {} - Status init(std::map* streams, int size, MemPool* mem_pool, - OlapReaderStatistics* stats) override { - _is_null = reinterpret_cast(mem_pool->allocate(size)); - memset(_is_null, 1, size); - _stats = stats; - return Status::OK(); - } - virtual Status seek(PositionProvider* positions) override { return Status::OK(); } - virtual Status skip(uint64_t row_count) override { return Status::OK(); } - virtual Status next_vector(ColumnVector* column_vector, uint32_t size, - MemPool* mem_pool) override { - column_vector->set_no_nulls(false); - column_vector->set_is_null(_is_null); - _stats->bytes_read += size; - return Status::OK(); - } -}; - -// 对于Tiny类型, 使用Byte作为存储 -class TinyColumnReader : public ColumnReader { -public: - TinyColumnReader(uint32_t column_id, uint32_t column_unique_id); - virtual ~TinyColumnReader(); - - virtual Status init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats); - virtual Status seek(PositionProvider* positions); - virtual Status skip(uint64_t row_count); - virtual Status next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool); - - virtual size_t get_buffer_size() { return sizeof(RunLengthByteReader); } - -private: - bool _eof; - char* _values; - RunLengthByteReader* _data_reader; -}; - -// A wrapper for IntColumnReader, which implements the interface to ColumnReader -template -class IntegerColumnReaderWrapper : public ColumnReader { -public: - IntegerColumnReaderWrapper(uint32_t column_id, uint32_t column_unique_id) - : ColumnReader(column_id, column_unique_id), - _reader(column_unique_id), - _values(nullptr), - _eof(false) {} - - virtual ~IntegerColumnReaderWrapper() {} - - virtual Status init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats) { - Status res = ColumnReader::init(streams, size, mem_pool, stats); - - if (res.ok()) { - res = _reader.init(streams, is_sign); - } - - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(T))); - - return res; - } - virtual Status seek(PositionProvider* positions) { - Status res; - if (nullptr == _present_reader) { - res = _reader.seek(positions); - if (!res.ok()) { - return res; - } - } else { - //all field in the segment can be nullptr, so the data stream is EOF - res = ColumnReader::seek(positions); - if (!res.ok()) { - return res; - } - res = _reader.seek(positions); - if (!res.ok() && OLAP_ERR_COLUMN_STREAM_EOF != res.precise_code()) { - LOG(WARNING) << "fail to seek int stream. res = " << res; - return res; - } - } - - return Status::OK(); - } - virtual Status skip(uint64_t row_count) { return _reader.skip(_count_none_nulls(row_count)); } - - virtual Status next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool) { - Status res = ColumnReader::next_vector(column_vector, size, mem_pool); - if (!res.ok()) { - if (OLAP_ERR_DATA_EOF == res.precise_code()) { - _eof = true; - } - return res; - } - - column_vector->set_col_data(_values); - if (column_vector->no_nulls()) { - for (uint32_t i = 0; i < size; ++i) { - int64_t value = 0; - res = _reader.next(&value); - if (!res.ok()) { - break; - } - _values[i] = value; - } - } else { - bool* is_null = column_vector->is_null(); - for (uint32_t i = 0; i < size; ++i) { - int64_t value = 0; - if (!is_null[i]) { - res = _reader.next(&value); - if (!res.ok()) { - break; - } - } - _values[i] = value; - } - } - _stats->bytes_read += sizeof(T) * size; - - if (OLAP_ERR_DATA_EOF == res.precise_code()) { - _eof = true; - } - return res; - } - - virtual size_t get_buffer_size() { return sizeof(RunLengthIntegerReader); } - -private: - IntegerColumnReader _reader; // Wrapped real reader - T* _values; - bool _eof; -}; - -// There are two types of strings in OLAP Engine, fixed-length strings and variable-length strings, using two wrappers respectively -// class handles the return format of these two strings -// FixLengthStringColumnReader handles fixed-length strings, the feature is that the part of insufficient length should be filled with 0 -template -class FixLengthStringColumnReader : public ColumnReader { -public: - FixLengthStringColumnReader(uint32_t column_id, uint32_t column_unique_id, - uint32_t string_length, uint32_t dictionary_size) - : ColumnReader(column_id, column_unique_id), - _eof(false), - _reader(column_unique_id, dictionary_size), - _string_length(string_length) {} - virtual ~FixLengthStringColumnReader() {} - - virtual Status init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats) { - Status res = ColumnReader::init(streams, size, mem_pool, stats); - - if (res.ok()) { - res = _reader.init(streams, size, mem_pool); - } - - return res; - } - - virtual Status seek(PositionProvider* positions) { - Status res; - if (nullptr == _present_reader) { - res = _reader.seek(positions); - if (!res.ok()) { - return res; - } - } else { - //all field in the segment can be nullptr, so the data stream is EOF - res = ColumnReader::seek(positions); - if (!res.ok()) { - return res; - } - res = _reader.seek(positions); - if (!res.ok() && OLAP_ERR_COLUMN_STREAM_EOF != res.precise_code()) { - LOG(WARNING) << "fail to read fixed string stream. res = " << res; - return res; - } - } - - return Status::OK(); - } - virtual Status skip(uint64_t row_count) { return _reader.skip(_count_none_nulls(row_count)); } - virtual Status next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool) { - Status res = ColumnReader::next_vector(column_vector, size, mem_pool); - if (!res.ok()) { - if (OLAP_ERR_DATA_EOF == res.precise_code()) { - _eof = true; - } - return res; - } - - return _reader.next_vector(column_vector, size, mem_pool, &_stats->bytes_read); - } - - virtual size_t get_buffer_size() { return _reader.get_buffer_size() + _string_length; } - -private: - bool _eof; - ReaderClass _reader; - uint32_t _string_length; -}; - -// VarStringColumnReader handles variable length strings, characterized by using uint16 in the data header to indicate the length -template -class VarStringColumnReader : public ColumnReader { -public: - VarStringColumnReader(uint32_t column_id, uint32_t column_unique_id, uint32_t max_length, - uint32_t dictionary_size) - : ColumnReader(column_id, column_unique_id), - _eof(false), - _reader(column_unique_id, dictionary_size), - _max_length(max_length) {} - virtual ~VarStringColumnReader() {} - virtual Status init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats) { - Status res = ColumnReader::init(streams, size, mem_pool, stats); - if (res.ok()) { - res = _reader.init(streams, size, mem_pool); - } - - return res; - } - - virtual Status seek(PositionProvider* position) { - Status res; - if (nullptr == _present_reader) { - res = _reader.seek(position); - if (!res.ok()) { - return res; - } - } else { - //all field in the segment can be nullptr, so the data stream is EOF - res = ColumnReader::seek(position); - if (!res.ok()) { - return res; - } - res = _reader.seek(position); - if (!res.ok() && OLAP_ERR_COLUMN_STREAM_EOF != res.precise_code()) { - LOG(WARNING) << "fail to seek varchar stream. res = " << res; - return res; - } - } - - return Status::OK(); - } - virtual Status skip(uint64_t row_count) { return _reader.skip(_count_none_nulls(row_count)); } - - virtual Status next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool) { - Status res = ColumnReader::next_vector(column_vector, size, mem_pool); - if (!res.ok()) { - if (OLAP_ERR_DATA_EOF == res.precise_code()) { - _eof = true; - } - return res; - } - - return _reader.next_vector(column_vector, size, mem_pool, &_stats->bytes_read); - } - - virtual size_t get_buffer_size() { return _reader.get_buffer_size() + _max_length; } - -private: - bool _eof; - ReaderClass _reader; - uint32_t _max_length; -}; - -template -class FloatintPointColumnReader : public ColumnReader { -public: - FloatintPointColumnReader(uint32_t column_id, uint32_t column_unique_id) - : ColumnReader(column_id, column_unique_id), - _eof(false), - _data_stream(nullptr), - _values(nullptr) {} - - virtual ~FloatintPointColumnReader() {} - - virtual Status init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats) { - if (nullptr == streams) { - LOG(WARNING) << "input streams is nullptr"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - // reset stream and reader - ColumnReader::init(streams, size, mem_pool, stats); - _data_stream = extract_stream(_column_unique_id, StreamInfoMessage::DATA, streams); - - if (nullptr == _data_stream) { - LOG(WARNING) << "specified stream not exist"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_STREAM_NOT_EXIST); - } - - _values = reinterpret_cast(mem_pool->allocate(size * sizeof(FLOAT_TYPE))); - return Status::OK(); - } - virtual Status seek(PositionProvider* position) { - if (nullptr == position) { - LOG(WARNING) << "input positions is nullptr"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - if (nullptr == _data_stream) { - LOG(WARNING) << "reader not init."; - return Status::OLAPInternalError(OLAP_ERR_NOT_INITED); - } - - Status res; - if (nullptr == _present_reader) { - res = _data_stream->seek(position); - if (!res.ok()) { - return res; - } - } else { - //all field in the segment can be nullptr, so the data stream is EOF - res = ColumnReader::seek(position); - if (!res.ok()) { - return res; - } - res = _data_stream->seek(position); - if (!res.ok() && OLAP_ERR_COLUMN_STREAM_EOF != res.precise_code()) { - LOG(WARNING) << "fail to seek float stream. res = " << res; - return res; - } - } - - return Status::OK(); - } - virtual Status skip(uint64_t row_count) { - if (nullptr == _data_stream) { - LOG(WARNING) << "reader not init."; - return Status::OLAPInternalError(OLAP_ERR_NOT_INITED); - } - - uint64_t skip_values_count = _count_none_nulls(row_count); - return _data_stream->skip(skip_values_count * sizeof(FLOAT_TYPE)); - } - - virtual Status next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool) { - if (nullptr == _data_stream) { - LOG(WARNING) << "reader not init."; - return Status::OLAPInternalError(OLAP_ERR_NOT_INITED); - } - - Status res = ColumnReader::next_vector(column_vector, size, mem_pool); - if (!res.ok()) { - if (OLAP_ERR_DATA_EOF == res.precise_code()) { - _eof = true; - } - return res; - } - - bool* is_null = column_vector->is_null(); - column_vector->set_col_data(_values); - size_t length = sizeof(FLOAT_TYPE); - if (column_vector->no_nulls()) { - for (uint32_t i = 0; i < size; ++i) { - FLOAT_TYPE value = 0.0; - res = _data_stream->read(reinterpret_cast(&value), &length); - if (!res.ok()) { - break; - } - _values[i] = value; - } - } else { - for (uint32_t i = 0; i < size; ++i) { - FLOAT_TYPE value = 0.0; - if (!is_null[i]) { - res = _data_stream->read(reinterpret_cast(&value), &length); - if (!res.ok()) { - break; - } - } - _values[i] = value; - } - } - _stats->bytes_read += sizeof(FLOAT_TYPE) * size; - - if (OLAP_ERR_DATA_EOF == res.precise_code()) { - _eof = true; - } - - return res; - } - -protected: - bool _eof; - ReadOnlyFileStream* _data_stream; - FLOAT_TYPE* _values; -}; - -class DecimalColumnReader : public ColumnReader { -public: - DecimalColumnReader(uint32_t column_id, uint32_t column_unique_id); - virtual ~DecimalColumnReader(); - Status init(std::map* streams, int size, MemPool* mem_pool, - OlapReaderStatistics* stats) override; - virtual Status seek(PositionProvider* positions) override; - virtual Status skip(uint64_t row_count) override; - virtual Status next_vector(ColumnVector* column_vector, uint32_t size, - MemPool* mem_pool) override; - - virtual size_t get_buffer_size() override { return sizeof(RunLengthByteReader) * 2; } - -private: - bool _eof; - decimal12_t* _values; - RunLengthIntegerReader* _int_reader; - RunLengthIntegerReader* _frac_reader; -}; - -class LargeIntColumnReader : public ColumnReader { -public: - LargeIntColumnReader(uint32_t column_id, uint32_t column_unique_id); - virtual ~LargeIntColumnReader(); - virtual Status init(std::map* streams, int size, - MemPool* mem_pool, OlapReaderStatistics* stats); - virtual Status seek(PositionProvider* positions); - virtual Status skip(uint64_t row_count); - virtual Status next_vector(ColumnVector* column_vector, uint32_t size, MemPool* mem_pool); - - virtual size_t get_buffer_size() { return sizeof(RunLengthByteReader) * 2; } - -private: - bool _eof; - int128_t* _values; - RunLengthIntegerReader* _high_reader; - RunLengthIntegerReader* _low_reader; -}; - -typedef FloatintPointColumnReader FloatColumnReader; -typedef FloatintPointColumnReader DoubleColumnReader; -typedef IntegerColumnReaderWrapper DiscreteDoubleColumnReader; - -// Use 3 bytes to store the date -// Use IntegerColumnReader, truncated to 3 bytes length when returning data -typedef IntegerColumnReaderWrapper DateColumnReader; - -// Internal use LONG implementation -typedef IntegerColumnReaderWrapper DateTimeColumnReader; - -typedef IntegerColumnReaderWrapper DateV2ColumnReader; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_COLUMN_READER_H diff --git a/be/src/olap/rowset/column_writer.cpp b/be/src/olap/rowset/column_writer.cpp deleted file mode 100644 index af212a6927..0000000000 --- a/be/src/olap/rowset/column_writer.cpp +++ /dev/null @@ -1,873 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/column_writer.h" - -#include "olap/file_helper.h" -#include "olap/rowset/bit_field_writer.h" - -namespace doris { - -ColumnWriter* ColumnWriter::create(uint32_t column_id, const TabletSchema& schema, - OutStreamFactory* stream_factory, size_t num_rows_per_row_block, - double bf_fpp) { - ColumnWriter* column_writer = nullptr; - const TabletColumn& column = schema.column(column_id); - - switch (column.type()) { - case OLAP_FIELD_TYPE_BOOL: - case OLAP_FIELD_TYPE_TINYINT: - case OLAP_FIELD_TYPE_UNSIGNED_TINYINT: { - column_writer = new (std::nothrow) - ByteColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_SMALLINT: { - column_writer = new (std::nothrow) IntegerColumnWriterWrapper( - column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_UNSIGNED_SMALLINT: { - column_writer = new (std::nothrow) IntegerColumnWriterWrapper( - column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_INT: { - column_writer = new (std::nothrow) IntegerColumnWriterWrapper( - column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_UNSIGNED_INT: { - column_writer = new (std::nothrow) IntegerColumnWriterWrapper( - column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_BIGINT: { - column_writer = new (std::nothrow) IntegerColumnWriterWrapper( - column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_UNSIGNED_BIGINT: { - column_writer = new (std::nothrow) IntegerColumnWriterWrapper( - column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_FLOAT: { - column_writer = new (std::nothrow) FloatColumnWriter(column_id, stream_factory, column, - num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_DOUBLE: { - column_writer = new (std::nothrow) DoubleColumnWriter(column_id, stream_factory, column, - num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_DISCRETE_DOUBLE: { - column_writer = new (std::nothrow) DiscreteDoubleColumnWriter( - column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_CHAR: { - column_writer = new (std::nothrow) FixLengthStringColumnWriter( - column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_DATETIME: { - column_writer = new (std::nothrow) DateTimeColumnWriter(column_id, stream_factory, column, - num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_DATE: { - column_writer = new (std::nothrow) - DateColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_DATEV2: { - column_writer = new (std::nothrow) DateV2ColumnWriter(column_id, stream_factory, column, - num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_DECIMAL: { - column_writer = new (std::nothrow) DecimalColumnWriter(column_id, stream_factory, column, - num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_LARGEINT: { - column_writer = new (std::nothrow) LargeIntColumnWriter(column_id, stream_factory, column, - num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_VARCHAR: - case OLAP_FIELD_TYPE_OBJECT: - case OLAP_FIELD_TYPE_HLL: - case OLAP_FIELD_TYPE_STRING: { - column_writer = new (std::nothrow) VarStringColumnWriter(column_id, stream_factory, column, - num_rows_per_row_block, bf_fpp); - break; - } - case OLAP_FIELD_TYPE_STRUCT: - case OLAP_FIELD_TYPE_ARRAY: - case OLAP_FIELD_TYPE_MAP: - default: { - LOG(WARNING) << "Unsupported field type. field=" << column.name() - << ", type=" << column.type(); - break; - } - } - - return column_writer; -} - -ColumnWriter::ColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, size_t num_rows_per_row_block, double bf_fpp) - : _column_id(column_id), - _column(column), - _stream_factory(stream_factory), - _index(column.type()), - _is_present(nullptr), - _is_present_stream(nullptr), - _index_stream(nullptr), - _is_found_nulls(false), - _bf(nullptr), - _num_rows_per_row_block(num_rows_per_row_block), - _bf_fpp(bf_fpp) {} - -ColumnWriter::~ColumnWriter() { - SAFE_DELETE(_is_present); - SAFE_DELETE(_bf); - - for (std::vector::iterator it = _sub_writers.begin(); it != _sub_writers.end(); - ++it) { - SAFE_DELETE(*it); - } -} - -Status ColumnWriter::init() { - if (_column.is_nullable()) { - _is_present_stream = - _stream_factory->create_stream(unique_column_id(), StreamInfoMessage::PRESENT); - - if (nullptr == _is_present_stream) { - LOG(WARNING) << "fail to allocate IS PRESENT STREAM"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _is_present = new (std::nothrow) BitFieldWriter(_is_present_stream); - - if (nullptr == _is_present) { - LOG(WARNING) << "fail to allocate IS PRESENT Writer"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - if (!_is_present->init()) { - LOG(WARNING) << "fail to init IS PRESENT Writer"; - return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); - } - } - - Status res = _block_statistics.init(_column.type(), true); - - if (!res.ok()) { - LOG(WARNING) << "init block statistic failed"; - return res; - } - - res = _segment_statistics.init(_column.type(), true); - - if (!res.ok()) { - LOG(WARNING) << "init segment statistic failed"; - return res; - } - - _index_stream = - _stream_factory->create_stream(unique_column_id(), StreamInfoMessage::ROW_INDEX); - - if (nullptr == _index_stream) { - LOG(WARNING) << "fail to allocate Index STREAM"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - // bloom filter index - if (is_bf_column()) { - _bf_index_stream = - _stream_factory->create_stream(unique_column_id(), StreamInfoMessage::BLOOM_FILTER); - if (nullptr == _bf_index_stream) { - LOG(WARNING) << "fail to allocate bloom filter index stream"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _bf = new (std::nothrow) BloomFilter(); - if (nullptr == _bf) { - LOG(WARNING) << "fail to allocate bloom filter"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - if (!_bf->init(_num_rows_per_row_block, _bf_fpp)) { - LOG(WARNING) << "fail to init bloom filter. num rows: " << _num_rows_per_row_block - << ", fpp: " << _bf_fpp; - return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); - } - } - - return Status::OK(); -} - -Status ColumnWriter::write(RowCursor* row_cursor) { - Status res = Status::OK(); - - bool is_null = row_cursor->is_null(_column_id); - char* buf = row_cursor->cell_ptr(_column_id); - if (_is_present) { - res = _is_present->write(is_null); - - if (is_null) { - _is_found_nulls = true; - } - } - - if (is_bf_column()) { - if (!is_null) { - if (_column.type() == OLAP_FIELD_TYPE_CHAR || - _column.type() == OLAP_FIELD_TYPE_VARCHAR || - _column.type() == OLAP_FIELD_TYPE_HLL || _column.type() == OLAP_FIELD_TYPE_STRING) { - Slice* slice = reinterpret_cast(buf); - _bf->add_bytes(slice->data, slice->size); - } else { - _bf->add_bytes(buf, row_cursor->column_size(_column_id)); - } - } else { - _bf->add_bytes(nullptr, 0); - } - } - - return res; -} - -Status ColumnWriter::flush() { - return _is_present->flush(); -} - -Status ColumnWriter::create_row_index_entry() { - Status res = Status::OK(); - segment_statistics()->merge(&_block_statistics); - _index_entry.set_statistic(&_block_statistics); - _index.add_index_entry(_index_entry); - _index_entry.reset_write_offset(); - _block_statistics.reset(); - record_position(); - - if (is_bf_column()) { - _bf_index.add_bloom_filter(_bf); - - _bf = new (std::nothrow) BloomFilter(); - if (nullptr == _bf) { - LOG(WARNING) << "fail to allocate bloom filter"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - if (!_bf->init(_num_rows_per_row_block, _bf_fpp)) { - LOG(WARNING) << "fail to init bloom filter. num rows: " << _num_rows_per_row_block - << ", fpp: " << _bf_fpp; - return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); - } - } - - for (std::vector::iterator it = _sub_writers.begin(); it != _sub_writers.end(); - ++it) { - if (!(res = (*it)->create_row_index_entry())) { - LOG(WARNING) << "fail to create sub column's index."; - return res; - } - } - - return res; -} - -uint64_t ColumnWriter::estimate_buffered_memory() { - uint64_t result = 0; - - // bloom filter - if (is_bf_column()) { - result += _bf_index.estimate_buffered_memory(); - } - - for (std::vector::iterator it = _sub_writers.begin(); it != _sub_writers.end(); - ++it) { - result += (*it)->estimate_buffered_memory(); - } - - return result; -} - -// Delete the positions used by is_present_stream: -// * OutStream uses 2 -// * ByteRunLength uses 1 -// * BitRunLength uses 1 -// Delete 4 in total -void ColumnWriter::_remove_is_present_positions() { - for (uint32_t i = 0; i < _index.entry_size(); i++) { - PositionEntryWriter* entry = _index.mutable_entry(i); - entry->remove_written_position(0, 4); - } -} - -Status ColumnWriter::finalize(ColumnDataHeaderMessage* header) { - Status res = Status::OK(); - - if (nullptr != _is_present) { - if (!(res = _is_present->flush())) { - return res; - } - - if (!_is_found_nulls) { - _is_present_stream->suppress(); - _remove_is_present_positions(); - } - } - - char* index_buf = nullptr; - // char* index_statistic_buf = NULL; - // Write index pb - size_t pb_size = _index.output_size(); - index_buf = new (std::nothrow) char[pb_size]; - ColumnMessage* column = nullptr; - - if (!_index.write_to_buffer(index_buf, pb_size)) { - LOG(WARNING) << "fail to serialize index"; - res = Status::OLAPInternalError(OLAP_ERR_SERIALIZE_PROTOBUF_ERROR); - goto FINALIZE_EXIT; - } - - res = _index_stream->write(index_buf, pb_size); - - if (!res.ok()) { - LOG(WARNING) << "fail to write index to stream"; - goto FINALIZE_EXIT; - } - - res = _index_stream->flush(); - - if (!res.ok()) { - LOG(WARNING) << "fail to flush index stream"; - goto FINALIZE_EXIT; - } - - // write bloom filter index - if (is_bf_column()) { - res = _bf_index.write_to_buffer(_bf_index_stream); - if (!res.ok()) { - LOG(WARNING) << "fail to write bloom filter stream"; - OLAP_GOTO(FINALIZE_EXIT); - } - - res = _bf_index_stream->flush(); - if (!res.ok()) { - LOG(WARNING) << "fail to flush bloom filter stream"; - OLAP_GOTO(FINALIZE_EXIT); - } - } - - // Record a Schema information in the Segment header - // This makes it not affect the reading of the data in the existing segment after modifying the schema of the table - column = header->add_column(); - column->set_name(_column.name()); - column->set_type(TabletColumn::get_string_by_field_type(_column.type())); - column->set_aggregation(TabletColumn::get_string_by_aggregation_type(_column.aggregation())); - column->set_length(_column.length()); - column->set_is_key(_column.is_key()); - column->set_precision(_column.precision()); - column->set_frac(_column.frac()); - column->set_unique_id(_column.unique_id()); - column->set_is_bf_column(is_bf_column()); - - save_encoding(header->add_column_encoding()); - -FINALIZE_EXIT: - SAFE_DELETE_ARRAY(index_buf); - // SAFE_DELETE_ARRAY(index_statistic_buf); - return res; -} - -void ColumnWriter::record_position() { - if (nullptr != _is_present) { - _is_present->get_position(&_index_entry); - } -} - -// The default returns DIRECT, String type may return Dict -void ColumnWriter::save_encoding(ColumnEncodingMessage* encoding) { - encoding->set_kind(ColumnEncodingMessage::DIRECT); -} - -void ColumnWriter::get_bloom_filter_info(bool* has_bf_column, uint32_t* bf_hash_function_num, - uint32_t* bf_bit_num) { - if (is_bf_column()) { - *has_bf_column = true; - *bf_hash_function_num = _bf->hash_function_num(); - *bf_bit_num = _bf->bit_num(); - return; - } - - for (std::vector::iterator it = _sub_writers.begin(); it != _sub_writers.end(); - ++it) { - (*it)->get_bloom_filter_info(has_bf_column, bf_hash_function_num, bf_bit_num); - if (*has_bf_column) { - return; - } - } -} - -//////////////////////////////////////////////////////////////////////////////// -ByteColumnWriter::ByteColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, size_t num_rows_per_row_block, - double bf_fpp) - : ColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp), - _writer(nullptr) {} - -ByteColumnWriter::~ByteColumnWriter() { - SAFE_DELETE(_writer); -} - -Status ByteColumnWriter::init() { - Status res = Status::OK(); - - if (!(res = ColumnWriter::init())) { - return res; - } - - OutStreamFactory* factory = stream_factory(); - OutStream* stream = factory->create_stream(unique_column_id(), StreamInfoMessage::DATA); - - if (nullptr == stream) { - LOG(WARNING) << "fail to allocate DATA STREAM"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _writer = new (std::nothrow) RunLengthByteWriter(stream); - - if (nullptr == _writer) { - LOG(WARNING) << "fail to allocate RunLengthByteWriter"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - record_position(); - return Status::OK(); -} - -Status ByteColumnWriter::finalize(ColumnDataHeaderMessage* header) { - Status res = Status::OK(); - - if (!(res = ColumnWriter::finalize(header))) { - LOG(WARNING) << "fail to finalize ColumnWriter."; - return res; - } - - if (!(res = _writer->flush())) { - LOG(WARNING) << "fail to flush."; - return res; - } - - return Status::OK(); -} - -void ByteColumnWriter::record_position() { - ColumnWriter::record_position(); - _writer->get_position(index_entry()); -} - -//////////////////////////////////////////////////////////////////////////////// - -IntegerColumnWriter::IntegerColumnWriter(uint32_t column_id, uint32_t unique_column_id, - OutStreamFactory* stream_factory, bool is_singed) - : _unique_column_id(unique_column_id), - _stream_factory(stream_factory), - _writer(nullptr), - _is_signed(is_singed) {} - -IntegerColumnWriter::~IntegerColumnWriter() { - SAFE_DELETE(_writer); -} - -Status IntegerColumnWriter::init() { - OutStream* stream = _stream_factory->create_stream(_unique_column_id, StreamInfoMessage::DATA); - - if (nullptr == stream) { - LOG(WARNING) << "fail to allocate DATA STREAM"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _writer = new (std::nothrow) RunLengthIntegerWriter(stream, _is_signed); - - if (nullptr == _writer) { - LOG(WARNING) << "fail to allocate RunLengthIntegerWriter"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - return Status::OK(); -} - -//////////////////////////////////////////////////////////////////////////////// - -VarStringColumnWriter::VarStringColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, - size_t num_rows_per_row_block, double bf_fpp) - : ColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp), - _use_dictionary_encoding(false), - _dict_total_size(0), - _dict_stream(nullptr), - _length_writer(nullptr), - _data_stream(nullptr), - _id_writer(nullptr) {} - -VarStringColumnWriter::~VarStringColumnWriter() { - SAFE_DELETE(_length_writer); - SAFE_DELETE(_id_writer); -} - -Status VarStringColumnWriter::init() { - Status res = Status::OK(); - - if (!(res = ColumnWriter::init())) { - return res; - } - - _dict_stream = - stream_factory()->create_stream(unique_column_id(), StreamInfoMessage::DICTIONARY_DATA); - _data_stream = stream_factory()->create_stream(unique_column_id(), StreamInfoMessage::DATA); - OutStream* length_stream = - stream_factory()->create_stream(unique_column_id(), StreamInfoMessage::LENGTH); - - if (nullptr == _dict_stream || nullptr == length_stream || nullptr == _data_stream) { - LOG(WARNING) << "fail to create stream."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _length_writer = new (std::nothrow) RunLengthIntegerWriter(length_stream, false); - _id_writer = new (std::nothrow) RunLengthIntegerWriter(_data_stream, false); - - if (nullptr == _length_writer || nullptr == _id_writer) { - LOG(WARNING) << "fail to create writer."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - record_position(); - return Status::OK(); -} - -Status VarStringColumnWriter::write(const char* str, uint32_t len) { - Status res = Status::OK(); - // zdb shield the dictionary coding - //std::string key(str, len); - - if (!(res = _data_stream->write(str, len))) { - LOG(WARNING) << "fail to write string content."; - return res; - } - - if (!(res = _length_writer->write(len))) { - LOG(WARNING) << "fail to write string length."; - return res; - } - - return Status::OK(); -} - -uint64_t VarStringColumnWriter::estimate_buffered_memory() { - // the length of _string_id is short after RLE - return _dict_total_size; -} - -Status VarStringColumnWriter::_finalize_dict_encoding() { - Status res = Status::OK(); - std::vector dump_order; - uint32_t current_id = 0; - - dump_order.resize(_string_keys.size()); - - for (StringDict::iterator it = _string_dict.begin(); it != _string_dict.end(); ++it) { - dump_order[it->second] = current_id; - current_id++; - const std::string& key = it->first.get(); - res = _dict_stream->write(key.c_str(), key.length()); - - if (!res.ok()) { - LOG(WARNING) << "fail to write string dict to stream."; - return res; - } - - if (!(res = _length_writer->write(key.length()))) { - LOG(WARNING) << "fail to write string length to stream."; - return res; - } - } - - uint32_t block_id = 0; - - // Suppose there are n ids in total. (total) - for (uint32_t i = 0; i <= _string_id.size(); i++) { - while (block_id < _block_row_count.size() - 1 && i == _block_row_count[block_id]) { - _id_writer->get_position(index()->mutable_entry(block_id), false); - block_id++; - } - - if (i != _string_id.size()) { - res = _id_writer->write(dump_order[_string_id[i]]); - - if (!res.ok()) { - LOG(WARNING) << "fail to write string id to stream."; - return res; - } - } - } - - return Status::OK(); -} - -Status VarStringColumnWriter::_finalize_direct_encoding() { - //Status res = Status::OK(); - //uint32_t block_id = 0; - - _dict_stream->suppress(); - return Status::OK(); -} - -Status VarStringColumnWriter::finalize(ColumnDataHeaderMessage* header) { - Status res = Status::OK(); - uint64_t ratio_threshold = config::column_dictionary_key_ratio_threshold; - uint64_t size_threshold = config::column_dictionary_key_size_threshold; - - // the dictionary condition:1 key size < size threshold; 2 key ratio < ratio threshold - _use_dictionary_encoding = (_string_keys.size() < size_threshold) && - (_string_keys.size() * 100UL < _string_id.size() * ratio_threshold); - - if (_use_dictionary_encoding) { - res = _finalize_dict_encoding(); - if (!res.ok()) { - LOG(WARNING) << "fail to finalize dict encoding."; - return res; - } - } else { - res = _finalize_direct_encoding(); - if (!res.ok()) { - LOG(WARNING) << "fail to finalize direct encoding."; - return res; - } - } - - // The index's supplementary writing has been completed, ColumnWriter::finalize will write the header - res = ColumnWriter::finalize(header); - if (!res.ok()) { - LOG(WARNING) << "fail to finalize ColumnWriter."; - return res; - } - - // id_writer is practical to data_stream, it doesn't matter if you repeat flush - if (!_length_writer->flush() || !_id_writer->flush() || !_dict_stream->flush() || - !_data_stream->flush()) { - LOG(WARNING) << "fail to flush stream."; - return Status::OLAPInternalError(OLAP_ERR_WRITER_DATA_WRITE_ERROR); - } - - _string_keys.clear(); - _string_dict.clear(); - _string_id.clear(); - _block_row_count.clear(); - _dict_total_size = 0; - return Status::OK(); -} - -void VarStringColumnWriter::save_encoding(ColumnEncodingMessage* encoding) { - if (_use_dictionary_encoding) { - encoding->set_kind(ColumnEncodingMessage::DICTIONARY); - encoding->set_dictionary_size(_string_keys.size()); - } else { - encoding->set_kind(ColumnEncodingMessage::DIRECT); - } -} - -// Unlike other Writer, data is written to Stream only when it is finalized. -// So it is impossible to record the position of the stream. For this reason, record the number of data written in each block, and when finalize -// Use this information to add stream location information to Index -void VarStringColumnWriter::record_position() { - ColumnWriter::record_position(); - _block_row_count.push_back(_string_id.size()); - //zdb shield dictionary coding - _data_stream->get_position(index_entry()); - _length_writer->get_position(index_entry(), false); -} - -//////////////////////////////////////////////////////////////////////////////// - -FixLengthStringColumnWriter::FixLengthStringColumnWriter(uint32_t column_id, - OutStreamFactory* stream_factory, - const TabletColumn& column, - size_t num_rows_per_row_block, - double bf_fpp) - : VarStringColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp), - _length(column.length()) {} - -FixLengthStringColumnWriter::~FixLengthStringColumnWriter() {} - -//////////////////////////////////////////////////////////////////////////////// - -DecimalColumnWriter::DecimalColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, size_t num_rows_per_row_block, - double bf_fpp) - : ColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp), - _int_writer(nullptr), - _frac_writer(nullptr) {} - -DecimalColumnWriter::~DecimalColumnWriter() { - SAFE_DELETE(_int_writer); - SAFE_DELETE(_frac_writer); -} - -Status DecimalColumnWriter::init() { - Status res = Status::OK(); - - res = ColumnWriter::init(); - if (!res.ok()) { - return res; - } - - OutStream* int_stream = - stream_factory()->create_stream(unique_column_id(), StreamInfoMessage::DATA); - OutStream* frac_stream = - stream_factory()->create_stream(unique_column_id(), StreamInfoMessage::SECONDARY); - - if (nullptr == int_stream || nullptr == frac_stream) { - LOG(WARNING) << "fail to create stream."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _int_writer = new (std::nothrow) RunLengthIntegerWriter(int_stream, true); - _frac_writer = new (std::nothrow) RunLengthIntegerWriter(frac_stream, true); - - if (nullptr == _int_writer || nullptr == _frac_writer) { - LOG(WARNING) << "fail to create writer."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - record_position(); - return Status::OK(); -} - -Status DecimalColumnWriter::finalize(ColumnDataHeaderMessage* header) { - Status res; - - res = ColumnWriter::finalize(header); - if (!res.ok()) { - LOG(WARNING) << "fail to finalize ColumnWriter."; - return res; - } - - res = _int_writer->flush(); - if (!res.ok()) { - LOG(WARNING) << "fail to flush integer writer."; - return res; - } - - res = _frac_writer->flush(); - if (!res.ok()) { - LOG(WARNING) << "fail to flush fraction writer."; - return res; - } - - return Status::OK(); -} - -void DecimalColumnWriter::record_position() { - ColumnWriter::record_position(); - _int_writer->get_position(index_entry(), false); - _frac_writer->get_position(index_entry(), false); -} - -//////////////////////////////////////////////////////////////////////////////// - -LargeIntColumnWriter::LargeIntColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, - size_t num_rows_per_row_block, double bf_fpp) - : ColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp), - _high_writer(nullptr), - _low_writer(nullptr) {} - -LargeIntColumnWriter::~LargeIntColumnWriter() { - SAFE_DELETE(_high_writer); - SAFE_DELETE(_low_writer); -} - -Status LargeIntColumnWriter::init() { - Status res = Status::OK(); - - res = ColumnWriter::init(); - if (!res.ok()) { - return res; - } - - OutStream* high_stream = - stream_factory()->create_stream(unique_column_id(), StreamInfoMessage::DATA); - OutStream* low_stream = - stream_factory()->create_stream(unique_column_id(), StreamInfoMessage::SECONDARY); - - if (nullptr == high_stream || nullptr == low_stream) { - LOG(WARNING) << "fail to create stream."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - _high_writer = new (std::nothrow) RunLengthIntegerWriter(high_stream, true); - _low_writer = new (std::nothrow) RunLengthIntegerWriter(low_stream, true); - - if (nullptr == _high_writer || nullptr == _low_writer) { - LOG(WARNING) << "fail to create writer."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - record_position(); - return Status::OK(); -} - -Status LargeIntColumnWriter::finalize(ColumnDataHeaderMessage* header) { - Status res; - - res = ColumnWriter::finalize(header); - if (!res.ok()) { - LOG(WARNING) << "fail to finalize ColumnWriter."; - return res; - } - - res = _high_writer->flush(); - if (!res.ok()) { - LOG(WARNING) << "fail to flush integer writer."; - return res; - } - - res = _low_writer->flush(); - if (!res.ok()) { - LOG(WARNING) << "fail to flush fraction writer."; - return res; - } - - return Status::OK(); -} - -void LargeIntColumnWriter::record_position() { - ColumnWriter::record_position(); - _high_writer->get_position(index_entry(), false); - _low_writer->get_position(index_entry(), false); -} - -} // namespace doris diff --git a/be/src/olap/rowset/column_writer.h b/be/src/olap/rowset/column_writer.h deleted file mode 100644 index 7771189e74..0000000000 --- a/be/src/olap/rowset/column_writer.h +++ /dev/null @@ -1,587 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_COLUMN_WRITER_H -#define DORIS_BE_SRC_OLAP_ROWSET_COLUMN_WRITER_H - -#include - -#include - -#include "olap/bloom_filter.hpp" -#include "olap/bloom_filter_writer.h" -#include "olap/field.h" -#include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/out_stream.h" -#include "olap/row_block.h" -#include "olap/row_cursor.h" -#include "olap/rowset/run_length_byte_writer.h" -#include "olap/rowset/run_length_integer_writer.h" -#include "olap/stream_index_writer.h" - -namespace doris { - -class OutStream; -class OutStreamFactory; -class ColumnStatistics; -class BitFieldWriter; -class RunLengthByteWriter; -class RunLengthIntegerWriter; - -class ColumnWriter { -public: - // Create a ColumnWriter, the lifetime of the object after creation is owned by the caller - // That is, the caller is responsible for calling delete to destruct the ColumnWriter - // Args: - // column_id: the position of the created column in columns - // columns: all column information of the table - // stream_factory: The factory object used to create the output stream, the lifetime of the object is owned by the caller - static ColumnWriter* create(uint32_t column_id, const TabletSchema& schema, - OutStreamFactory* stream_factory, size_t num_rows_per_row_block, - double bf_fpp); - - virtual ~ColumnWriter(); - virtual Status init(); - - Status write(RowCursor* cursor); - - virtual Status write_batch(RowBlock* block, RowCursor* cursor) = 0; - - // Write the previously recorded block location information and current statistical information into a new index entry - Status create_row_index_entry(); - // Estimate the current cache memory size, excluding the memory that has been output to OutStream - virtual uint64_t estimate_buffered_memory(); - virtual Status flush(); - // End the segment, flush stream and update the header: - // * column_unique_id - // * column_type - // * column_encoding - // * zone_maps - virtual Status finalize(ColumnDataHeaderMessage* header); - virtual void save_encoding(ColumnEncodingMessage* encoding); - uint32_t column_id() const { return _column_id; } - - uint32_t unique_column_id() const { return _column.unique_id(); } - - virtual void get_bloom_filter_info(bool* has_bf_column, uint32_t* bf_hash_function_num, - uint32_t* bf_bit_num); - - ColumnStatistics* segment_statistics() { return &_segment_statistics; } - - ColumnStatistics* block_statistics() { return &_block_statistics; } - -protected: - ColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, const TabletColumn& column, - size_t num_rows_per_row_block, double bf_fpp); - - OutStreamFactory* stream_factory() { return _stream_factory; } - PositionEntryWriter* index_entry() { return &_index_entry; } - StreamIndexWriter* index() { return &_index; } - // Record the position of the current Stream, which is used to generate index entries - virtual void record_position(); - -protected: - ColumnStatistics _block_statistics; - ColumnStatistics _segment_statistics; - -private: - void _remove_is_present_positions(); - - bool is_bf_column() { return _column.is_bf_column(); } - - uint32_t _column_id; - const TabletColumn& _column; - OutStreamFactory* _stream_factory; // The object is owned by the external caller - std::vector _sub_writers; // Writer to save the sub-column - PositionEntryWriter _index_entry; - StreamIndexWriter _index; - BitFieldWriter* _is_present; //Record NULL Bits for columns that allow NULL - OutStream* _is_present_stream; - OutStream* _index_stream; // Note that the ownership of the object is _stream_factory - bool _is_found_nulls; - BloomFilter* _bf; - BloomFilterIndexWriter _bf_index; - OutStream* _bf_index_stream; - size_t _num_rows_per_row_block; - double _bf_fpp; - - DISALLOW_COPY_AND_ASSIGN(ColumnWriter); -}; - -class ByteColumnWriter : public ColumnWriter { -public: - ByteColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, size_t num_rows_per_row_block, double bf_fpp); - virtual ~ByteColumnWriter(); - virtual Status init() override; - - Status write_batch(RowBlock* block, RowCursor* cursor) override { - for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { - block->get_row(i, cursor); - - Status res = ColumnWriter::write(cursor); - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write ColumnWriter."; - return res; - } - - auto cell = cursor->cell(column_id()); - _block_statistics.add(cell); - if (!cell.is_null()) { - char value = *reinterpret_cast(cell.cell_ptr()); - res = _writer->write(value); - if (!res.ok()) { - LOG(WARNING) << "fail to write double, res=" << res; - return res; - } - } - } - return Status::OK(); - } - - virtual Status finalize(ColumnDataHeaderMessage* header) override; - virtual void record_position() override; - virtual Status flush() override { return Status::OK(); } - -private: - RunLengthByteWriter* _writer; - - DISALLOW_COPY_AND_ASSIGN(ByteColumnWriter); -}; - -// For SHORT/INT/LONG type data, use int64 as the stored data uniformly -class IntegerColumnWriter { -public: - IntegerColumnWriter(uint32_t column_id, uint32_t unique_column_id, - OutStreamFactory* stream_factory, bool is_singed); - ~IntegerColumnWriter(); - Status init(); - Status write(int64_t data) { return _writer->write(data); } - Status finalize(ColumnDataHeaderMessage* header) { return _writer->flush(); } - void record_position(PositionEntryWriter* index_entry) { - _writer->get_position(index_entry, false); - } - Status flush() { return _writer->flush(); } - -private: - uint32_t _unique_column_id; - OutStreamFactory* _stream_factory; - RunLengthIntegerWriter* _writer; - bool _is_signed; - - DISALLOW_COPY_AND_ASSIGN(IntegerColumnWriter); -}; - -template -class IntegerColumnWriterWrapper : public ColumnWriter { -public: - IntegerColumnWriterWrapper(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, size_t num_rows_per_row_block, - double bf_fpp) - : ColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp), - _writer(column_id, column.unique_id(), stream_factory, is_singed) {} - - virtual ~IntegerColumnWriterWrapper() {} - - virtual Status init() override { - Status res = Status::OK(); - - res = ColumnWriter::init(); - if (!res.ok()) { - LOG(WARNING) << "fail to init ColumnWriter. res = " << res; - return res; - } - - res = _writer.init(); - if (!res.ok()) { - LOG(WARNING) << "fail to init IntegerColumnWriter. res = " << res; - return res; - } - - record_position(); - return Status::OK(); - } - - Status write_batch(RowBlock* block, RowCursor* cursor) override { - for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { - block->get_row(i, cursor); - Status res = ColumnWriter::write(cursor); - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write ColumnWriter. res = " << res; - return res; - } - - auto cell = cursor->cell(column_id()); - _block_statistics.add(cell); - if (!cell.is_null()) { - T value = *reinterpret_cast(cell.cell_ptr()); - res = _writer.write(static_cast(value)); - if (!res.ok()) { - LOG(WARNING) << "fail to write integer, res=" << res; - return res; - } - } - } - return Status::OK(); - } - - virtual Status flush() override { - Status res = ColumnWriter::flush(); - - if (!res.ok()) { - LOG(WARNING) << "fail to flush column_writer. res = " << res; - return res; - } - - res = _writer.flush(); - - if (!res.ok()) { - LOG(WARNING) << "fail to flush integer_writer. res = " << res; - return res; - } - - return res; - } - - virtual Status finalize(ColumnDataHeaderMessage* header) override { - Status res = ColumnWriter::finalize(header); - - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to finalize ColumnWriter. res = " << res; - return res; - } - - return _writer.finalize(header); - } - - virtual void record_position() override { - ColumnWriter::record_position(); - _writer.record_position(index_entry()); - } - -private: - IntegerColumnWriter _writer; - - DISALLOW_COPY_AND_ASSIGN(IntegerColumnWriterWrapper); -}; - -template -class DoubleColumnWriterBase : public ColumnWriter { -public: - DoubleColumnWriterBase(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, size_t num_rows_per_row_block, double bf_fpp) - : ColumnWriter(column_id, stream_factory, column, num_rows_per_row_block, bf_fpp), - _stream(nullptr) {} - - virtual ~DoubleColumnWriterBase() {} - - virtual Status init() override { - Status res = Status::OK(); - - res = ColumnWriter::init(); - if (!res.ok()) { - return res; - } - - OutStreamFactory* factory = stream_factory(); - _stream = factory->create_stream(unique_column_id(), StreamInfoMessage::DATA); - - if (nullptr == _stream) { - LOG(WARNING) << "fail to allocate DATA STREAM"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - record_position(); - return Status::OK(); - } - - virtual Status flush() override { return Status::OK(); } - - Status write_batch(RowBlock* block, RowCursor* cursor) override { - for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { - block->get_row(i, cursor); - Status res = ColumnWriter::write(cursor); - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write ColumnWriter. res = " << res; - return res; - } - - auto cell = cursor->cell(column_id()); - _block_statistics.add(cell); - if (!cell.is_null()) { - const T* value = reinterpret_cast(cell.cell_ptr()); - res = _stream->write(reinterpret_cast(value), sizeof(T)); - if (!res.ok()) { - LOG(WARNING) << "fail to write double, res=" << res; - return res; - } - } - } - return Status::OK(); - } - - virtual Status finalize(ColumnDataHeaderMessage* header) override { - Status res = Status::OK(); - - res = ColumnWriter::finalize(header); - if (!res.ok()) { - LOG(WARNING) << "fail to finalize ColumnWriter. res = " << res; - return res; - } - - res = _stream->flush(); - if (!res.ok()) { - LOG(WARNING) << "fail to flush. res = " << res; - return res; - } - - return Status::OK(); - } - - virtual void record_position() override { - ColumnWriter::record_position(); - _stream->get_position(index_entry()); - } - -private: - OutStream* _stream; - - DISALLOW_COPY_AND_ASSIGN(DoubleColumnWriterBase); -}; - -typedef DoubleColumnWriterBase DoubleColumnWriter; -typedef DoubleColumnWriterBase FloatColumnWriter; -typedef IntegerColumnWriterWrapper DiscreteDoubleColumnWriter; - -// VarString and String are used as variable length types to write using StringColumnWriter -class VarStringColumnWriter : public ColumnWriter { -public: - VarStringColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, size_t num_rows_per_row_block, double bf_fpp); - virtual ~VarStringColumnWriter(); - virtual Status init() override; - - Status write_batch(RowBlock* block, RowCursor* cursor) override { - for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { - block->get_row(i, cursor); - Status res = ColumnWriter::write(cursor); - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write ColumnWriter."; - return res; - } - bool is_null = cursor->is_null(column_id()); - if (!is_null) { - char* buf = cursor->cell_ptr(column_id()); - Slice* slice = reinterpret_cast(buf); - res = write(slice->data, slice->size); - if (!res.ok()) { - LOG(WARNING) << "fail to write varchar, res=" << res; - return res; - } - } - } - return Status::OK(); - } - - virtual uint64_t estimate_buffered_memory() override; - virtual Status finalize(ColumnDataHeaderMessage* header) override; - virtual void save_encoding(ColumnEncodingMessage* encoding) override; - virtual void record_position() override; - virtual Status flush() override { return Status::OK(); } - -protected: - //Write a piece of data directly without using cursor - Status write(const char* str, uint32_t length); - -private: - // You can use references as keys in the map - class DictKey { - public: - explicit DictKey(const std::string& str_ref) : _str_ref(str_ref) {} - bool operator<(const DictKey& other) const { return _str_ref < other._str_ref; } - bool operator==(const DictKey& other) const { return _str_ref == other._str_ref; } - const std::string& get() const { return _str_ref; } - - private: - const std::string _str_ref; - }; - typedef std::map StringDict; - -private: - Status _finalize_dict_encoding(); - Status _finalize_direct_encoding(); - -private: - bool _use_dictionary_encoding; - std::vector _string_id; - std::vector _string_keys; - StringDict _string_dict; - uint64_t _dict_total_size; - OutStream* _dict_stream; - RunLengthIntegerWriter* _length_writer; - OutStream* _data_stream; - RunLengthIntegerWriter* _id_writer; - std::vector _block_row_count; - - DISALLOW_COPY_AND_ASSIGN(VarStringColumnWriter); -}; - -// Specialize the VarStringColumnWriter, extract the data and write it when writing -class FixLengthStringColumnWriter : public VarStringColumnWriter { -public: - FixLengthStringColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, size_t num_rows_per_row_block, - double bf_fpp); - virtual ~FixLengthStringColumnWriter(); - - Status write_batch(RowBlock* block, RowCursor* cursor) override { - for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { - block->get_row(i, cursor); - - Status res = ColumnWriter::write(cursor); - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write ColumnWriter."; - return res; - } - - bool is_null = cursor->is_null(column_id()); - char* buf = cursor->cell_ptr(column_id()); - - if (!is_null) { - //const char* str = reinterpret_cast(buf); - Slice* slice = reinterpret_cast(buf); - res = VarStringColumnWriter::write(slice->data, slice->size); - if (!res.ok()) { - LOG(WARNING) << "fail to write fix-length string, res=" << res; - return res; - } - } - } - return Status::OK(); - } - - virtual Status flush() override { return Status::OK(); } - -private: - uint32_t _length; - - DISALLOW_COPY_AND_ASSIGN(FixLengthStringColumnWriter); -}; - -//Date is a three-byte integer -typedef IntegerColumnWriterWrapper DateColumnWriter; - -// DateTime is implemented with int64 -typedef IntegerColumnWriterWrapper DateTimeColumnWriter; - -// DateTime is implemented with int64 -typedef IntegerColumnWriterWrapper DateV2ColumnWriter; - -class DecimalColumnWriter : public ColumnWriter { -public: - DecimalColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, size_t num_rows_per_row_block, double bf_fpp); - virtual ~DecimalColumnWriter(); - virtual Status init() override; - - Status write_batch(RowBlock* block, RowCursor* cursor) override { - for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { - block->get_row(i, cursor); - Status res = ColumnWriter::write(cursor); - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write ColumnWriter."; - return res; - } - - auto cell = cursor->cell(column_id()); - _block_statistics.add(cell); - if (!cell.is_null()) { - decimal12_t value = *reinterpret_cast(cell.cell_ptr()); - res = _int_writer->write(value.integer); - if (!res.ok()) { - LOG(WARNING) << "fail to write integer of Decimal."; - return res; - } - res = _frac_writer->write(value.fraction); - if (!res.ok()) { - LOG(WARNING) << "fail to write fraction of Decimal."; - return res; - } - } - } - return Status::OK(); - } - - virtual Status finalize(ColumnDataHeaderMessage* header) override; - virtual void record_position() override; - virtual Status flush() override { return Status::OK(); } - -private: - RunLengthIntegerWriter* _int_writer; - RunLengthIntegerWriter* _frac_writer; - - DISALLOW_COPY_AND_ASSIGN(DecimalColumnWriter); -}; - -class LargeIntColumnWriter : public ColumnWriter { -public: - LargeIntColumnWriter(uint32_t column_id, OutStreamFactory* stream_factory, - const TabletColumn& column, size_t num_rows_per_row_block, double bf_fpp); - virtual ~LargeIntColumnWriter(); - virtual Status init() override; - - Status write_batch(RowBlock* block, RowCursor* cursor) override { - for (uint32_t i = 0; i < block->row_block_info().row_num; i++) { - block->get_row(i, cursor); - Status res = ColumnWriter::write(cursor); - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write ColumnWriter."; - return res; - } - auto cell = cursor->cell(column_id()); - _block_statistics.add(cell); - if (!cell.is_null()) { - const int64_t* value = reinterpret_cast(cell.cell_ptr()); - res = _high_writer->write(*value); - if (!res.ok()) { - LOG(WARNING) << "fail to write integer of LargeInt."; - return res; - } - res = _low_writer->write(*(++value)); - if (!res.ok()) { - LOG(WARNING) << "fail to write fraction of LargeInt."; - return res; - } - } - } - return Status::OK(); - } - - virtual Status finalize(ColumnDataHeaderMessage* header) override; - virtual void record_position() override; - virtual Status flush() override { return Status::OK(); } - -private: - RunLengthIntegerWriter* _high_writer; - RunLengthIntegerWriter* _low_writer; - - DISALLOW_COPY_AND_ASSIGN(LargeIntColumnWriter); -}; - -} // namespace doris -#endif // DORIS_BE_SRC_OLAP_ROWSET_COLUMN_WRITER_H diff --git a/be/src/olap/rowset/rowset_factory.cpp b/be/src/olap/rowset/rowset_factory.cpp index 361290fe51..9072e6c122 100644 --- a/be/src/olap/rowset/rowset_factory.cpp +++ b/be/src/olap/rowset/rowset_factory.cpp @@ -21,8 +21,6 @@ #include "beta_rowset.h" #include "gen_cpp/olap_file.pb.h" -#include "olap/rowset/alpha_rowset.h" -#include "olap/rowset/alpha_rowset_writer.h" #include "olap/rowset/beta_rowset_writer.h" #include "olap/rowset/rowset_writer.h" @@ -32,8 +30,7 @@ Status RowsetFactory::create_rowset(const TabletSchema* schema, const FilePathDesc& rowset_path_desc, RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr* rowset) { if (rowset_meta->rowset_type() == ALPHA_ROWSET) { - rowset->reset(new AlphaRowset(schema, rowset_path_desc, rowset_meta)); - return (*rowset)->init(); + return Status::OLAPInternalError(OLAP_ERR_ROWSET_INVALID); } if (rowset_meta->rowset_type() == BETA_ROWSET) { rowset->reset(new BetaRowset(schema, rowset_path_desc, rowset_meta)); @@ -45,8 +42,7 @@ Status RowsetFactory::create_rowset(const TabletSchema* schema, Status RowsetFactory::create_rowset_writer(const RowsetWriterContext& context, std::unique_ptr* output) { if (context.rowset_type == ALPHA_ROWSET) { - output->reset(new AlphaRowsetWriter); - return (*output)->init(context); + return Status::OLAPInternalError(OLAP_ERR_ROWSET_INVALID); } if (context.rowset_type == BETA_ROWSET) { output->reset(new BetaRowsetWriter); diff --git a/be/src/olap/rowset/run_length_byte_reader.cpp b/be/src/olap/rowset/run_length_byte_reader.cpp deleted file mode 100644 index b29f921de7..0000000000 --- a/be/src/olap/rowset/run_length_byte_reader.cpp +++ /dev/null @@ -1,145 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/run_length_byte_reader.h" - -#include "olap/in_stream.h" -#include "olap/rowset/column_reader.h" - -namespace doris { - -RunLengthByteReader::RunLengthByteReader(ReadOnlyFileStream* input) - : _input(input), _num_literals(0), _used(0), _repeat(false) {} - -Status RunLengthByteReader::_read_values() { - Status res = Status::OK(); - _used = 0; - char control_byte = 0; - - res = _input->read(&control_byte); - if (!res.ok()) { - LOG(WARNING) << "fail to read control byte.res = " << res; - return res; - } - - if (control_byte > -1) { - _repeat = true; - _num_literals = control_byte + RunLengthByteWriter::MIN_REPEAT_SIZE; - - res = _input->read(&_literals[0]); - if (!res.ok()) { - LOG(WARNING) << "fail to read value byte.res = " << res; - return res; - } - } else { - _repeat = false; - _num_literals = -control_byte; - uint64_t bytes = 0; - - while (bytes < static_cast(_num_literals)) { - uint64_t to_read = _num_literals - bytes; - - res = _input->read(&_literals[bytes], &to_read); - if (!res.ok()) { - LOG(WARNING) << "fail to read value byte.res = " << res; - return res; - } - - bytes += to_read; - } - } - - return res; -} - -bool RunLengthByteReader::has_next() const { - return _used != _num_literals || !_input->eof(); -} - -Status RunLengthByteReader::next(char* value) { - Status res = Status::OK(); - - if (_used == _num_literals) { - res = _read_values(); - if (!res.ok()) { - LOG(WARNING) << "fail to read values.res = " << res; - return res; - } - } - - if (_repeat) { - _used += 1; - *value = _literals[0]; - } else { - *value = _literals[_used++]; - } - - return res; -} - -Status RunLengthByteReader::seek(PositionProvider* position) { - Status res = Status::OK(); - - res = _input->seek(position); - if (!res.ok()) { - VLOG_TRACE << "fail to ReadOnlyFileStream seek. res = " << res; - return res; - } - - int64_t consumed = position->get_next(); - - if (consumed != 0) { - // a loop is required for cases where we break the run into two parts - while (consumed > 0) { - res = _read_values(); - if (!res.ok()) { - LOG(WARNING) << "fail to read values.res = " << res; - break; - } - - _used = consumed; - consumed -= _num_literals; - } - } else { - _used = 0; - _num_literals = 0; - } - - return res; -} - -Status RunLengthByteReader::skip(uint64_t num_values) { - Status res = Status::OK(); - - while (num_values > 0) { - if (_used == _num_literals) { - res = _read_values(); - if (!res.ok()) { - LOG(WARNING) << "fail to read values.res = " << res; - break; - } - } - - uint64_t consume = std::min(num_values, static_cast(_num_literals - _used)); - _used += consume; - num_values -= consume; - } - - return res; -} - -} // namespace doris \ No newline at end of file diff --git a/be/src/olap/rowset/run_length_byte_reader.h b/be/src/olap/rowset/run_length_byte_reader.h deleted file mode 100644 index f80bd1b451..0000000000 --- a/be/src/olap/rowset/run_length_byte_reader.h +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_BYTE_READER_H -#define DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_BYTE_READER_H - -#include "olap/file_stream.h" -#include "olap/olap_define.h" -#include "olap/rowset/run_length_byte_writer.h" - -namespace doris { - -class ReadOnlyFileStream; -class PositionProvider; - -// A reader that reads a sequence of bytes. A control byte is read before -// each run with positive values 0 to 127 meaning 3 to 130 repetitions. If the -// byte is -1 to -128, 1 to 128 literal byte values follow. -class RunLengthByteReader { -public: - explicit RunLengthByteReader(ReadOnlyFileStream* input); - ~RunLengthByteReader() {} - bool has_next() const; - // Gets the next piece of data, or if there is no more, returns Status::OLAPInternalError(OLAP_ERR_DATA_EOF) - Status next(char* value); - Status seek(PositionProvider* position); - Status skip(uint64_t num_values); - -private: - Status _read_values(); - - ReadOnlyFileStream* _input; - char _literals[RunLengthByteWriter::MAX_LITERAL_SIZE]; - int32_t _num_literals; - int32_t _used; - bool _repeat; - - DISALLOW_COPY_AND_ASSIGN(RunLengthByteReader); -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_BYTE_READER_H \ No newline at end of file diff --git a/be/src/olap/rowset/run_length_byte_writer.cpp b/be/src/olap/rowset/run_length_byte_writer.cpp deleted file mode 100644 index b1009c7846..0000000000 --- a/be/src/olap/rowset/run_length_byte_writer.cpp +++ /dev/null @@ -1,142 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/run_length_byte_writer.h" - -#include "olap/out_stream.h" - -namespace doris { - -const int32_t RunLengthByteWriter::MIN_REPEAT_SIZE; -const int32_t RunLengthByteWriter::MAX_LITERAL_SIZE; -const int32_t RunLengthByteWriter::MAX_REPEAT_SIZE; - -RunLengthByteWriter::RunLengthByteWriter(OutStream* output) - : _output(output), _num_literals(0), _repeat(false), _tail_run_length(0) {} - -Status RunLengthByteWriter::_write_values() { - Status res = Status::OK(); - - if (_num_literals != 0) { - if (_repeat) { - res = _output->write(_num_literals - MIN_REPEAT_SIZE); - if (!res.ok()) { - LOG(WARNING) << "fail to write control byte."; - return res; - } - - res = _output->write(_literals[0]); - if (!res.ok()) { - LOG(WARNING) << "fail to write repeat byte"; - return res; - } - } else { - res = _output->write(-_num_literals); - if (!res.ok()) { - LOG(WARNING) << "fail to write control byte."; - return res; - } - - res = _output->write(_literals, _num_literals); - if (!res.ok()) { - LOG(WARNING) << "fail to write literals bytes."; - return res; - } - } - - _repeat = false; - _tail_run_length = 0; - _num_literals = 0; - } - - return res; -} - -Status RunLengthByteWriter::write(char value) { - Status res = Status::OK(); - - if (0 == _num_literals) { - _literals[0] = value; - _num_literals = 1; - _tail_run_length = 1; - } else if (_repeat) { - if (value == _literals[0]) { - _num_literals++; - - if (_num_literals == MAX_REPEAT_SIZE) { - res = _write_values(); - } - } else { - res = _write_values(); - - if (res.ok()) { - _literals[0] = value; - _num_literals = 1; - _tail_run_length = 1; - } - } - } else { - if (value == _literals[_num_literals - 1]) { - _tail_run_length++; - } else { - _tail_run_length = 1; - } - - if (_tail_run_length == MIN_REPEAT_SIZE) { - if (_num_literals + 1 == MIN_REPEAT_SIZE) { - _repeat = true; - _num_literals++; - } else { - _num_literals -= MIN_REPEAT_SIZE - 1; - res = _write_values(); - - if (res.ok()) { - _literals[0] = value; - _repeat = true; - _num_literals = MIN_REPEAT_SIZE; - } - } - } else { - _literals[_num_literals] = value; - _num_literals++; - - if (_num_literals == MAX_LITERAL_SIZE) { - res = _write_values(); - } - } - } - - return res; -} - -Status RunLengthByteWriter::flush() { - Status res; - - res = _write_values(); - if (!res.ok()) { - return res; - } - - return _output->flush(); -} - -void RunLengthByteWriter::get_position(PositionEntryWriter* index_entry) const { - _output->get_position(index_entry); - index_entry->add_position(_num_literals); -} - -} // namespace doris \ No newline at end of file diff --git a/be/src/olap/rowset/run_length_byte_writer.h b/be/src/olap/rowset/run_length_byte_writer.h deleted file mode 100644 index 90d5046d83..0000000000 --- a/be/src/olap/rowset/run_length_byte_writer.h +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_BYTE_WRITER_H -#define DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_BYTE_WRITER_H - -#include "olap/olap_define.h" -#include "olap/stream_index_writer.h" - -namespace doris { - -class OutStream; -class RowIndexEntryMessage; - -// A writer that writes a sequence of bytes. A control byte is written before -// each run with positive values 0 to 127 meaning 3 to 130 repetitions. If the -// byte is -1 to -128, 1 to 128 literal byte values follow. -class RunLengthByteWriter { -public: - explicit RunLengthByteWriter(OutStream* output); - ~RunLengthByteWriter() {} - Status write(char byte); - Status flush(); - void get_position(PositionEntryWriter* index_entry) const; - static const int32_t MIN_REPEAT_SIZE = 3; - static const int32_t MAX_LITERAL_SIZE = 128; - static const int32_t MAX_REPEAT_SIZE = 127 + MIN_REPEAT_SIZE; - -private: - Status _write_values(); - - OutStream* _output; - char _literals[MAX_LITERAL_SIZE]; - int32_t _num_literals; - bool _repeat; - int32_t _tail_run_length; - - DISALLOW_COPY_AND_ASSIGN(RunLengthByteWriter); -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_BYTE_WRITER_H \ No newline at end of file diff --git a/be/src/olap/rowset/run_length_integer_reader.cpp b/be/src/olap/rowset/run_length_integer_reader.cpp deleted file mode 100644 index 4fcd6019d4..0000000000 --- a/be/src/olap/rowset/run_length_integer_reader.cpp +++ /dev/null @@ -1,429 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/run_length_integer_reader.h" - -#include "olap/in_stream.h" -#include "olap/rowset/column_reader.h" -#include "olap/serialize.h" - -namespace doris { - -RunLengthIntegerReader::RunLengthIntegerReader(ReadOnlyFileStream* input, bool is_singed) - : _input(input), _signed(is_singed), _num_literals(0), _used(0) {} - -Status RunLengthIntegerReader::_read_values() { - Status res = Status::OK(); - - // read the first 2 bits and determine the encoding type - uint8_t first_byte = 0; - - res = _input->read((char*)&first_byte); - if (!res.ok()) { - LOG(WARNING) << "fail to read first byte.res = " << res; - return res; - } else { - int enc = (first_byte >> 6) & 0x03; - - if (RunLengthIntegerWriter::SHORT_REPEAT == enc) { - res = _read_short_repeat_values(first_byte); - } else if (RunLengthIntegerWriter::DIRECT == enc) { - res = _read_direct_values(first_byte); - } else if (RunLengthIntegerWriter::PATCHED_BASE == enc) { - res = _read_patched_base_values(first_byte); - } else { - res = _read_delta_values(first_byte); - } - } - - return res; -} - -Status RunLengthIntegerReader::_read_delta_values(uint8_t first_byte) { - Status res = Status::OK(); - - // extract the number of fixed bits - uint32_t fb = (first_byte >> 1) & 0x1f; - - if (fb != 0) { - fb = ser::decode_bit_width(fb); - } - - // extract the blob run length - int32_t len = (first_byte & 0x01) << 8; - uint8_t byte = 0; - - res = _input->read((char*)&byte); - if (!res.ok()) { - return res; - } - - len |= byte; - - // read the first value stored as vint - int64_t first_val = 0; - - if (_signed) { - res = ser::read_var_signed(_input, &first_val); - if (!res.ok()) { - LOG(WARNING) << "fail to read var signed.res = " << res; - return res; - } - } else { - res = ser::read_var_unsigned(_input, &first_val); - if (!res.ok()) { - LOG(WARNING) << "fail to read var unsigned.res = " << res; - return res; - } - } - - // store first value to result buffer - int64_t prev_val = first_val; - _literals[_num_literals++] = first_val; - - // if fixed bits is 0 then all values have fixed delta - if (fb == 0) { - // read the fixed delta value stored as vint (deltas can be negative even - // if all number are positive) - int64_t fd = 0; - - res = ser::read_var_signed(_input, &fd); - if (!res.ok()) { - LOG(WARNING) << "fail to read var signed.res = " << res; - return res; - } - - // add fixed deltas to adjacent values - for (int i = 0; i < len; i++) { - //_literals[_num_literals++] = _literals[_num_literals - 2] + fd; - _literals[_num_literals] = _literals[_num_literals - 1] + fd; - _num_literals++; - } - } else { - int64_t delta_base = 0; - - res = ser::read_var_signed(_input, &delta_base); - if (!res.ok()) { - LOG(WARNING) << "fail to read var signed.res = " << res; - return res; - } - - // add delta base and first value - _literals[_num_literals++] = first_val + delta_base; - prev_val = _literals[_num_literals - 1]; - len -= 1; - - // write the unpacked values, add it to previous value and store final - // value to result buffer. if the delta base value is negative then it - // is a decreasing sequence else an increasing sequence - res = ser::read_ints(_input, &_literals[_num_literals], len, fb); - if (!res.ok()) { - LOG(WARNING) << "fail to read ints.res = " << res; - return res; - } - - while (len > 0) { - if (delta_base < 0) { - _literals[_num_literals] = prev_val - _literals[_num_literals]; - } else { - _literals[_num_literals] = prev_val + _literals[_num_literals]; - } - - prev_val = _literals[_num_literals]; - --len; - ++_num_literals; - } - } - - return res; -} - -Status RunLengthIntegerReader::_read_patched_base_values(uint8_t first_byte) { - Status res = Status::OK(); - - // extract the number of fixed bits - int32_t fbo = (first_byte >> 1) & 0x1f; - int32_t fb = ser::decode_bit_width(fbo); - - // extract the run length of data blob - int32_t len = (first_byte & 0x01) << 8; - uint8_t byte = 0; - - res = _input->read((char*)&byte); - if (!res.ok()) { - return res; - } - - len |= byte; - // runs are always one off - len += 1; - - // extract the number of bytes occupied by base - char third_byte = '\0'; - - res = _input->read(&third_byte); - if (!res.ok()) { - LOG(WARNING) << "fail to read byte from in_stream.res = " << res; - return res; - } - - int32_t bw = ((uint8_t)third_byte >> 5) & 0x07; - // base width is one off - bw += 1; - - // extract patch width - uint32_t pwo = third_byte & 0x1f; - uint32_t pw = ser::decode_bit_width(pwo); - - // read fourth byte and extract patch gap width - char four_byte = '\0'; - - res = _input->read(&four_byte); - if (!res.ok()) { - LOG(WARNING) << "fail to read byte from in_straem.res = " << res; - return res; - } - - int32_t pgw = ((uint8_t)four_byte >> 5) & 0x07; - // patch gap width is one off - pgw += 1; - - // extract the length of the patch list - int32_t pl = four_byte & 0x1f; - - // read the next base width number of bytes to extract base value - int64_t base = 0; - - res = ser::bytes_to_long_be(_input, bw, &base); - if (!res.ok()) { - LOG(WARNING) << "fail to bytes to long be.res = " << res; - return res; - } - - int64_t mask = (1L << ((bw * 8) - 1)); - - // if MSB of base value is 1 then base is negative value else positive - // TODO(lijiao): Why is zig_zag not used here? - if ((base & mask) != 0) { - base = base & ~mask; - base = -base; - } - - // unpack the data blob - int64_t unpacked[len]; - - res = ser::read_ints(_input, unpacked, len, fb); - if (!res.ok()) { - return res; - } - - // unpack the patch blob - int64_t unpacked_patch[pl]; - uint32_t bit_width = ser::get_closet_fixed_bits(pw + pgw); - - res = ser::read_ints(_input, unpacked_patch, pl, bit_width); - if (!res.ok()) { - return res; - } - - // apply the patch directly when decoding the packed data - int32_t patch_idx = 0; - int64_t curr_gap = 0; - int64_t curr_patch = 0; - curr_gap = (uint64_t)unpacked_patch[patch_idx] >> pw; - curr_patch = unpacked_patch[patch_idx] & ((1L << pw) - 1); - int64_t actual_gap = 0; - - // special case: gap is >255 then patch value will be 0. - // if gap is <=255 then patch value cannot be 0 - while (curr_gap == 255 && curr_patch == 0) { - actual_gap += 255; - ++patch_idx; - curr_gap = (uint64_t)unpacked_patch[patch_idx] >> pw; - curr_patch = unpacked_patch[patch_idx] & ((1L << pw) - 1); - } - - // add the left over gap - actual_gap += curr_gap; - - // unpack data blob, patch it (if required), add base to get final result - for (int32_t i = 0; i < len; i++) { - if (i == actual_gap) { - // extract the patch value - int64_t patched_val = unpacked[i] | (curr_patch << fb); - - // add base to patched value - _literals[_num_literals++] = base + patched_val; - - // increment the patch to point to next entry in patch list - ++patch_idx; - - if (patch_idx < pl) { - // read the next gap and patch - curr_gap = (uint64_t)unpacked_patch[patch_idx] >> pw; - curr_patch = unpacked_patch[patch_idx] & ((1L << pw) - 1); - actual_gap = 0; - - // special case: gap is >255 then patch will be 0. if gap is - // <=255 then patch cannot be 0 - while (curr_gap == 255 && curr_patch == 0) { - actual_gap += 255; - ++patch_idx; - curr_gap = (uint64_t)unpacked_patch[patch_idx] >> pw; - curr_patch = unpacked_patch[patch_idx] & ((1L << pw) - 1); - } - - // add the left over gap - actual_gap += curr_gap; - - // next gap is relative to the current gap - actual_gap += i; - } - } else { - // no patching required. add base to unpacked value to get final value - _literals[_num_literals++] = base + unpacked[i]; - } - } - - return res; -} - -Status RunLengthIntegerReader::_read_direct_values(uint8_t first_byte) { - Status res = Status::OK(); - - // extract the number of fixed bits - uint32_t fbo = (first_byte >> 1) & 0x1f; - uint32_t fb = ser::decode_bit_width(fbo); - - // extract the run length - int32_t len = (first_byte & 0x01) << 8; - uint8_t byte = 0; - - res = _input->read((char*)&byte); - if (!res.ok()) { - return res; - } - - len |= byte; - // runs are one off - len += 1; - - // write the unpacked values and zigzag decode to result buffer - res = ser::read_ints(_input, _literals, len, fb); - if (!res.ok()) { - return res; - } - - if (_signed) { - for (int32_t i = 0; i < len; ++i) { - _literals[_num_literals] = ser::zig_zag_decode(_literals[_num_literals]); - ++_num_literals; - } - } else { - _num_literals += len; - } - - return res; -} - -Status RunLengthIntegerReader::_read_short_repeat_values(uint8_t first_byte) { - Status res = Status::OK(); - - // read the number of bytes occupied by the value - int32_t size = (first_byte >> 3) & 0x07; - // #bytes are one off - size += 1; - - // read the run length - int32_t len = first_byte & 0x07; - // run lengths values are stored only after MIN_REPEAT value is met - len += RunLengthIntegerWriter::MIN_REPEAT; - - // read the repeated value which is store using fixed bytes - int64_t val = 0; - - res = ser::bytes_to_long_be(_input, size, &val); - if (!res.ok()) { - return res; - } - - if (_signed) { - val = ser::zig_zag_decode(val); - } - - // repeat the value for length times - for (int32_t i = 0; i < len; i++) { - _literals[_num_literals++] = val; - } - - return res; -} - -Status RunLengthIntegerReader::seek(PositionProvider* position) { - Status res = Status::OK(); - - if (!(res = _input->seek(position))) { - return res; - } - - int32_t consumed = static_cast(position->get_next()); - - if (consumed != 0) { - // a loop is required for cases where we break the run into two parts - while (consumed > 0) { - _num_literals = 0; - - res = _read_values(); - if (!res.ok()) { - return res; - } - - _used = consumed; - consumed -= _num_literals; - } - } else { - _used = 0; - _num_literals = 0; - } - - return res; -} - -Status RunLengthIntegerReader::skip(uint64_t num_values) { - Status res = Status::OK(); - - while (num_values > 0) { - if (_used == _num_literals) { - _num_literals = 0; - _used = 0; - - res = _read_values(); - if (!res.ok()) { - LOG(WARNING) << "fail to read values.res = " << res; - return res; - } - } - - int64_t consume = std::min(num_values, static_cast(_num_literals - _used)); - _used += consume; - num_values -= consume; - } - - return res; -} - -} // namespace doris diff --git a/be/src/olap/rowset/run_length_integer_reader.h b/be/src/olap/rowset/run_length_integer_reader.h deleted file mode 100644 index 95f1c9429d..0000000000 --- a/be/src/olap/rowset/run_length_integer_reader.h +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_INTEGER_READER_H -#define DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_INTEGER_READER_H - -#include "olap/file_stream.h" -#include "olap/olap_define.h" -#include "olap/rowset/run_length_integer_writer.h" -#include "olap/stream_index_reader.h" -#include "util/runtime_profile.h" - -namespace doris { - -class ReadOnlyFileStream; -class PositionProvider; - -class RunLengthIntegerReader { -public: - explicit RunLengthIntegerReader(ReadOnlyFileStream* input, bool is_singed); - ~RunLengthIntegerReader() {} - bool has_next() const { return _used != _num_literals || !_input->eof(); } - // 获取下一条数据, 如果没有更多的数据了, 返回Status::OLAPInternalError(OLAP_ERR_DATA_EOF) - Status next(int64_t* value) { - Status res = Status::OK(); - - if (OLAP_UNLIKELY(_used == _num_literals)) { - _num_literals = 0; - _used = 0; - - res = _read_values(); - if (!res.ok()) { - return res; - } - } - - *value = _literals[_used++]; - return res; - } - Status seek(PositionProvider* position); - Status skip(uint64_t num_values); - -private: - Status _read_values(); - Status _read_delta_values(uint8_t first_byte); - Status _read_patched_base_values(uint8_t first_byte); - Status _read_direct_values(uint8_t first_byte); - Status _read_short_repeat_values(uint8_t first_byte); - - ReadOnlyFileStream* _input; - bool _signed; - int64_t _literals[RunLengthIntegerWriter::MAX_SCOPE]; - int32_t _num_literals; - int32_t _used; - - DISALLOW_COPY_AND_ASSIGN(RunLengthIntegerReader); -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_INTEGER_READER_H \ No newline at end of file diff --git a/be/src/olap/rowset/run_length_integer_writer.cpp b/be/src/olap/rowset/run_length_integer_writer.cpp deleted file mode 100644 index 270f246930..0000000000 --- a/be/src/olap/rowset/run_length_integer_writer.cpp +++ /dev/null @@ -1,734 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/run_length_integer_writer.h" - -#include - -#include "olap/out_stream.h" -#include "olap/serialize.h" - -namespace doris { - -RunLengthIntegerWriter::RunLengthIntegerWriter(OutStream* output, bool is_singed) - : _output(output), _is_signed(is_singed), _fixed_run_length(0), _var_run_length(0) { - _clear(); -} - -// NOTE _fixed_run_length 和 _var_run_length没有修改 -void RunLengthIntegerWriter::_clear() { - _num_literals = 0; - _encoding = NONE_ENCODING; - _prev_delta = 0; - _fixed_delta = 0; - _zz_bits_90p = 0; - _zz_bits_100p = 0; - _br_bits_95p = 0; - _br_bits_100p = 0; - _bits_delta_max = 0; - _patch_gap_width = 0; - _patch_length = 0; - _patch_width = 0; - _min = 0; - _is_fixed_delta = false; -} - -void RunLengthIntegerWriter::_init_literals(int64_t value) { - _literals[0] = value; - _num_literals = 1; - _fixed_run_length = 1; - _var_run_length = 1; -} - -void RunLengthIntegerWriter::_determined_encoding() { - // for identifying monotonic sequences - bool is_increasing = false; - uint32_t increasing_count = 1; - bool is_decreasing = false; - uint32_t decreasing_count = 1; - - // for identifying type of delta encoding - _min = _literals[0]; - int64_t max = _literals[0]; - _is_fixed_delta = true; - int64_t current_delta = 0; - int64_t delta_max = 0; - - // populate all variables to identify the encoding type - if (_num_literals >= 1) { - if (_num_literals > 1) { - current_delta = _literals[1] - _literals[0]; - } - - for (uint32_t i = 0; i < _num_literals; i++) { - if (i > 0 && _literals[i] >= max) { - max = _literals[i]; - ++increasing_count; - } - - if (i > 0 && _literals[i] <= _min) { - _min = _literals[i]; - //decreasing_count--; - ++decreasing_count; - } - - //if (i > 0 and _is_fixed_delta) { - if (i > 0 && _is_fixed_delta) { - if (_literals[i] - _literals[i - 1] != current_delta) { - _is_fixed_delta = false; - } - - _fixed_delta = current_delta; - } - - // populate zigzag encoded literals - if (_is_signed) { - _zig_zag_literals[i] = ser::zig_zag_encode(_literals[i]); - } else { - _zig_zag_literals[i] = _literals[i]; - } - - // max delta value is required for computing the fixed bits - // required for delta blob in delta encoding - if (i > 0) { - if (i == 1) { - // first value preserve the sign - _adj_deltas[0] = _literals[1] - _literals[0]; - } else { - _adj_deltas[i - 1] = std::abs(_literals[i] - _literals[i - 1]); - - if (_adj_deltas[i - 1] > delta_max) { - delta_max = _adj_deltas[i - 1]; - } - } - } - } - - // stores the number of bits required for packing delta blob in delta - // encoding - _bits_delta_max = ser::find_closet_num_bits(delta_max); - - // if decreasing count equals total number of literals then the sequence - // is monotonically decreasing - if (increasing_count == 1 && decreasing_count == _num_literals) { - is_decreasing = true; - } - - // if increasing count equals total number of literals then the sequence - // is monotonically increasing - if (decreasing_count == 1 && increasing_count == _num_literals) { - is_increasing = true; - } - } - - // use DIRECT for delta overflows - uint16_t hists[65]; - ser::compute_hists(_zig_zag_literals, _num_literals, hists); - - _zz_bits_90p = ser::percentile_bits_with_hist(hists, _num_literals, 0.9); - _zz_bits_100p = ser::percentile_bits_with_hist(hists, _num_literals, 1.0); - if (!ser::is_safe_subtract(max, _min)) { - _encoding = DIRECT; - return; - } - - // if the sequence is both increasing and decreasing then it is not monotonic - if (is_decreasing && is_increasing) { - is_decreasing = false; - is_increasing = false; - } - - // fixed delta condition - if (!is_increasing && !is_decreasing && _is_fixed_delta) { - _encoding = DELTA; - return; - } - - // monotonic condition - //if (is_increasing && is_decreasing) { - if (is_increasing || is_decreasing) { - _encoding = DELTA; - return; - } - - // percentile values are computed for the zigzag encoded values. if the - // number of bit requirement between 90th and 100th percentile varies - // beyond a threshold then we need to patch the values. if the variation - // is not significant then we can use direct or delta encoding - uint32_t diff_bits = _zz_bits_100p - _zz_bits_90p; - - // if the difference between 90th percentile and 100th percentile fixed - // bits is > 1 then we need patch the values - if (!is_increasing && !is_decreasing && diff_bits > 1 && !_is_fixed_delta) { - // patching is done only on base reduced values. - // remove base from literals - for (uint32_t i = 0; i < _num_literals; i++) { - _base_reduced_literals[i] = _literals[i] - _min; - } - - ser::compute_hists(_base_reduced_literals, _num_literals, hists); - // 95th percentile width is used to determine max allowed value - // after which patching will be done - _br_bits_95p = ser::percentile_bits_with_hist(hists, _num_literals, 0.95); - - // 100th percentile is used to compute the max patch width - _br_bits_100p = ser::percentile_bits_with_hist(hists, _num_literals, 1.0); - - // after base reducing the values, if the difference in bits between - // 95th percentile and 100th percentile value is zero then there - // is no point in patching the values, in which case we will - // fallback to DIRECT encoding. - // The decision to use patched base was based on zigzag values, but the - // actual patching is done on base reduced literals. - if ((_br_bits_100p - _br_bits_95p) != 0) { - _encoding = PATCHED_BASE; - _prepare_patched_blob(); - return; - } else { - _encoding = DIRECT; - return; - } - } - - // if difference in bits between 95th percentile and 100th percentile is - // 0, then patch length will become 0. Hence we will fallback to direct - if (!is_increasing && !is_decreasing && diff_bits <= 1 && !_is_fixed_delta) { - _encoding = DIRECT; - return; - } - - // never happen - LOG(FATAL) << "ops: fail to determine encoding type."; -} - -void RunLengthIntegerWriter::_prepare_patched_blob() { - // mask will be max value beyond which patch will be generated - int64_t mask = (1L << _br_bits_95p) - 1; - - int32_t gap_list[MAX_PATCH_LIST]; - int64_t patch_list[MAX_PATCH_LIST]; - - // #bit for patch - _patch_width = ser::get_closet_fixed_bits(_br_bits_100p - _br_bits_95p); - - // if patch bit requirement is 64 then it will not possible to pack - // gap and patch together in a long. To make sure gap and patch can be - // packed together adjust the patch width - if (_patch_width == 64) { - _patch_width = 56; - _br_bits_95p = 8; - mask = (1L << _br_bits_95p) - 1; - } - - uint32_t gap_idx = 0; - uint32_t patch_idx = 0; - uint32_t prev = 0; - uint32_t gap = 0; - uint32_t max_gap = 0; - - for (uint32_t i = 0; i < _num_literals; i++) { - // if value is above mask then create the patch and record the gap - if (((uint64_t)_base_reduced_literals[i]) > (uint64_t)mask) { - gap = i - prev; - - if (gap > max_gap) { - max_gap = gap; - } - - // gaps are relative, so store the previous patched value index - prev = i; - gap_list[gap_idx++] = gap; - - // extract the most significant bits that are over mask bits - int64_t patch = ((uint64_t)_base_reduced_literals[i]) >> _br_bits_95p; - patch_list[patch_idx++] = patch; - - // strip off the MSB to enable safe bit packing - _base_reduced_literals[i] &= mask; - } - } - - _patch_length = gap_idx; - - // if the element to be patched is the first and only element then - // max gap will be 0, but to store the gap as 0 we need at least 1 bit - if (max_gap == 0 && _patch_length != 0) { - _patch_gap_width = 1; - } else { - _patch_gap_width = ser::find_closet_num_bits(max_gap); - } - - // special case: if the patch gap width is greater than 256, then - // we need 9 bits to encode the gap width. But we only have 3 bits in - // header to record the gap width. To deal with this case, we will save - // two entries in patch list in the following way - // 256 gap width => 0 for patch value - // actual gap - 256 => actual patch value - // We will do the same for gap width = 511. If the element to be patched is - // the last element in the scope then gap width will be 511. In this case we - // will have 3 entries in the patch list in the following way - // 255 gap width => 0 for patch value - // 255 gap width => 0 for patch value - // 1 gap width => actual patch value - if (_patch_gap_width > 8) { - _patch_gap_width = 8; - - // for gap = 511, we need two additional entries in patch list - if (max_gap == 511) { - _patch_length += 2; - } else { - _patch_length += 1; - } - } - - // create gap vs patch list - gap_idx = 0; - patch_idx = 0; - - for (uint32_t i = 0; i < _patch_length; i++) { - int64_t g = gap_list[gap_idx++]; - int64_t p = patch_list[patch_idx++]; - - while (g > 255) { - _gap_vs_patch_list[i++] = (255L << _patch_width); - g -= 255; - } - - // store patch value in LSBs and gap in MSBs - _gap_vs_patch_list[i] = (g << _patch_width) | p; - } -} - -Status RunLengthIntegerWriter::_write_short_repeat_values() { - Status res = Status::OK(); - int64_t repeat_value; - - repeat_value = _is_signed ? ser::zig_zag_encode(_literals[0]) : _literals[0]; - - uint32_t num_bits_repeat_value = ser::find_closet_num_bits(repeat_value); - uint32_t num_bytes_repeat_value = num_bits_repeat_value % 8 == 0 - ? num_bits_repeat_value >> 3 - : (num_bits_repeat_value >> 3) + 1; - - ShortRepeatHead head; - head.type = SHORT_REPEAT; - head.value_bytes = num_bytes_repeat_value - 1; - head.run_length = _fixed_run_length - MIN_REPEAT; - - res = _output->write((char*)&head, sizeof(head)); - if (!res.ok()) { - LOG(WARNING) << "fail to write SHORT_REPEAT head."; - return res; - } - - // write the repeating value in big endian byte order - for (int32_t i = num_bytes_repeat_value - 1; i >= 0; i--) { - char byte = (char)(((uint64_t)repeat_value >> (i * 8)) & 0xff); - - if (!(res = _output->write(byte))) { - LOG(WARNING) << "fail to write SHORT_REPEAT data."; - return res; - } - } - - _fixed_run_length = 0; - return Status::OK(); -} - -Status RunLengthIntegerWriter::_write_direct_values() { - Status res = Status::OK(); - DirectHead head; - - head.type = DIRECT; - head.bit_width = ser::encode_bit_width(_zz_bits_100p); - head.set_length(_var_run_length - 1); - - res = _output->write((char*)&head, sizeof(head)); - if (!res.ok()) { - LOG(WARNING) << "fail to write DIRECT head."; - return res; - } - - res = ser::write_ints(_output, _zig_zag_literals, _num_literals, _zz_bits_100p); - if (!res.ok()) { - LOG(WARNING) << "fail to write DIRECT data."; - return res; - } - - _var_run_length = 0; - return Status::OK(); -} - -Status RunLengthIntegerWriter::_write_patched_base_values() { - Status res = Status::OK(); - uint32_t bit_width = 0; - PatchedBaseHead head; - - head.type = PATCHED_BASE; - head.bit_width = ser::encode_bit_width(_br_bits_95p); - head.set_length(_var_run_length - 1); - - // if the min value is negative toggle the sign - bool is_negative = _min < 0; - - if (is_negative) { - _min = -_min; - } - - // find the number of bytes required for base. - // The additional bit is used to store the sign of the base value - uint32_t base_width = ser::find_closet_num_bits(_min) + 1; - if (base_width > 64) { - base_width = 64; - } - uint32_t base_bytes = base_width % 8 == 0 ? base_width / 8 : (base_width / 8 + 1); - - if (is_negative) { - _min |= (1L << (base_bytes * 8 - 1)); - } - - head.base_bytes = base_bytes - 1; - head.patch_width = ser::encode_bit_width(_patch_width); - head.gap_width = _patch_gap_width - 1; - head.patch_length = _patch_length; - - // write header - res = _output->write((char*)&head, sizeof(head)); - - if (!res.ok()) { - LOG(WARNING) << "fail to write data.res = " << res; - return res; - } - - // write the base value using fixed bytes in big endian order - for (int32_t i = base_bytes - 1; i >= 0; i--) { - char byte = ((uint64_t)_min >> (i * 8)) & 0xff; - res = _output->write(byte); - - if (!res.ok()) { - LOG(WARNING) << "fail to write data.res = " << res; - return res; - } - } - - // write reduced literals - bit_width = ser::get_closet_fixed_bits(_br_bits_95p); - res = ser::write_ints(_output, _base_reduced_literals, _num_literals, bit_width); - - if (!res.ok()) { - LOG(WARNING) << "fail to write data.res = " << res; - return res; - } - - // write patch list - bit_width = ser::get_closet_fixed_bits(_patch_gap_width + _patch_width); - res = ser::write_ints(_output, _gap_vs_patch_list, _patch_length, bit_width); - - if (!res.ok()) { - LOG(WARNING) << "fail to write data.res = " << res; - return res; - } - - _var_run_length = 0; - return Status::OK(); -} - -Status RunLengthIntegerWriter::_write_delta_values() { - Status res = Status::OK(); - uint32_t len = 0; - uint32_t bit_width = _bits_delta_max; - uint32_t encoded_bit_width = 0; - DeltaHead head; - - if (_is_fixed_delta) { - // if fixed run length is greater than threshold then it will be fixed - // delta sequence with delta value 0 else fixed delta sequence with - // non-zero delta value - if (_fixed_run_length > MIN_REPEAT) { - // ex. sequence: 2 2 2 2 2 - len = _fixed_run_length - 1; - _fixed_run_length = 0; - } else { - // ex. sequence: 4 6 8 10 12 14 16 - len = _var_run_length - 1; - _var_run_length = 0; - } - } else { - // fixed width 0 is used for long repeating values. - // sequences that require only 1 bit to encode will have an additional bit - if (bit_width == 1) { - bit_width = 2; - } - - encoded_bit_width = ser::encode_bit_width(bit_width); - len = _var_run_length - 1; - _var_run_length = 0; - } - - head.type = DELTA; - head.bit_width = encoded_bit_width; - head.set_length(len); - - // write header - res = _output->write((char*)&head, sizeof(head)); - - if (!res.ok()) { - LOG(WARNING) << "fail to write data.res = " << res; - return res; - } - - // store the first value from zigzag literal array - if (_is_signed) { - res = ser::write_var_signed(_output, _literals[0]); - } else { - res = ser::write_var_unsigned(_output, _literals[0]); - } - - if (!res.ok()) { - LOG(WARNING) << "fail to write data.res = " << res; - return res; - } - - if (_is_fixed_delta) { - // if delta is fixed then we don't need to store delta blob - res = ser::write_var_signed(_output, _fixed_delta); - - if (!res.ok()) { - LOG(WARNING) << "fail to write data.res = " << res; - return res; - } - } else { - // store the first value as delta value using zigzag encoding - res = ser::write_var_signed(_output, _adj_deltas[0]); - - if (!res.ok()) { - LOG(WARNING) << "fail to write data.res = " << res; - return res; - } - - // adjacent delta values are bit packed - res = ser::write_ints(_output, &_adj_deltas[1], _num_literals - 2, bit_width); - - if (!res.ok()) { - LOG(WARNING) << "fail to write data.res = " << res; - return res; - } - } - - return Status::OK(); -} - -Status RunLengthIntegerWriter::_write_values() { - Status res = Status::OK(); - - if (_num_literals > 0) { - switch (_encoding) { - case SHORT_REPEAT: - if (!(res = _write_short_repeat_values())) { - LOG(WARNING) << "fail to write short repeat value.res = " << res; - return res; - } - - break; - - case DIRECT: - if (!(res = _write_direct_values())) { - LOG(WARNING) << "fail to write direct value.res = " << res; - return res; - } - - break; - - case PATCHED_BASE: - if (!(res = _write_patched_base_values())) { - LOG(WARNING) << "fail to write patched base value.res = " << res; - return res; - } - - break; - - case DELTA: - if (!(res = _write_delta_values())) { - LOG(WARNING) << "fail to write delta value.res = " << res; - return res; - } - - break; - - default: - LOG(WARNING) << "Unknown encoding encoding=" << _encoding; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - _clear(); - } - - return Status::OK(); -} - -Status RunLengthIntegerWriter::write(int64_t value) { - Status res = Status::OK(); - - if (_num_literals == 0) { - _init_literals(value); - } else if (_num_literals == 1) { - _prev_delta = value - _literals[0]; - _literals[1] = value; - _num_literals = 2; - - if (value == _literals[0]) { - _fixed_run_length = 2; - _var_run_length = 0; - } else { - _fixed_run_length = 0; - _var_run_length = 2; - } - } else { // _num_literals >= 2 - long current_delta = value - _literals[_num_literals - 1]; - - if (_prev_delta == 0 && current_delta == 0) { - // fixed delta run - _literals[_num_literals++] = value; - - // if variable run is non-zero then we are seeing repeating - // values at the end of variable run in which case keep - // updating variable and fixed runs - if (_var_run_length > 0) { - _fixed_run_length = 2; - } - - _fixed_run_length++; - - // if fixed run met the minimum condition and if variable - // run is non-zero then flush the variable run and shift the - // tail fixed runs to start of the buffer - if (_fixed_run_length >= MIN_REPEAT && _var_run_length > 0) { - _num_literals -= MIN_REPEAT; - _var_run_length -= MIN_REPEAT - 1; - uint32_t tail_pos = _num_literals; - _determined_encoding(); - - if (!(res = _write_values())) { - LOG(WARNING) << "fail to write values."; - return res; - } - - // shift tail fixed runs to beginning of the buffer - for (uint32_t i = 0; i < MIN_REPEAT; i++) { - _literals[_num_literals++] = _literals[tail_pos + i]; - } - } - - if (_fixed_run_length == MAX_SCOPE) { - _determined_encoding(); - - if (!(res = _write_values())) { - LOG(WARNING) << "fail to write values."; - return res; - } - } - } else { - // variable delta run - - // if fixed run length is non-zero and if it satisfies the - // short repeat conditions then write the values as short repeats - // else use delta encoding - if (_fixed_run_length >= MIN_REPEAT) { - if (_fixed_run_length <= MAX_SHORT_REPEAT_LENGTH) { - _encoding = SHORT_REPEAT; - } else { - _encoding = DELTA; - _is_fixed_delta = true; - } - - if (!(res = _write_values())) { - LOG(WARNING) << "fail to write values."; - return res; - } - } else if (_fixed_run_length > 0) { - if (value != _literals[_num_literals - 1]) { - _var_run_length = _fixed_run_length; - _fixed_run_length = 0; - } - } - - // after writing values re-initialize - if (_num_literals == 0) { - _init_literals(value); - } else { - // keep updating variable run length - _prev_delta = value - _literals[_num_literals - 1]; - _literals[_num_literals++] = value; - _var_run_length++; - - if (_var_run_length == MAX_SCOPE) { - _determined_encoding(); - - if (!(res = _write_values())) { - LOG(WARNING) << "fail to write values."; - return res; - } - } - } - } - } - - return res; -} - -Status RunLengthIntegerWriter::flush() { - Status res = Status::OK(); - - if (_num_literals != 0) { - if (_var_run_length != 0) { - _determined_encoding(); - } else if (_fixed_run_length != 0) { - if (_fixed_run_length < MIN_REPEAT) { - _var_run_length = _fixed_run_length; - _fixed_run_length = 0; - _determined_encoding(); - } else if (_fixed_run_length <= MAX_SHORT_REPEAT_LENGTH) { - _encoding = SHORT_REPEAT; - } else { - _encoding = DELTA; - _is_fixed_delta = true; - } - } - - if (!(res = _write_values())) { - LOG(WARNING) << "fail to write values."; - return res; - } - } - - // 这里的output是不是需要flush？ - // 如果flush了。那么流中会吧这一块单独搞成一个buffer段。比如写10行，也会单独成段 - // 如果不flush，会接着写，但我认为是没问题的 - return _output->flush(); -} - -void RunLengthIntegerWriter::get_position(PositionEntryWriter* index_entry, bool print) const { - _output->get_position(index_entry); - index_entry->add_position(_num_literals); - - if (print) { - _output->print_position_debug_info(); - VLOG_TRACE << "literals=" << _num_literals; - } -} - -} // namespace doris diff --git a/be/src/olap/rowset/run_length_integer_writer.h b/be/src/olap/rowset/run_length_integer_writer.h deleted file mode 100644 index f4008d2816..0000000000 --- a/be/src/olap/rowset/run_length_integer_writer.h +++ /dev/null @@ -1,283 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_INTEGER_WRITER_H -#define DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_INTEGER_WRITER_H - -#include - -#include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/out_stream.h" - -namespace doris { - -class OutStream; -class RowIndexEntryMessage; - -// 以下注释来自Hive orc -/* -A writer that performs light weight compression over sequence of integers. - -There are four types of lightweight integer compression: - - SHORT_REPEAT - - DIRECT - - PATCHED_BASE - - DELTA - -The description and format for these types are as below: - -|SHORT_REPEAT|: Used for short repeated integer sequences. - # 1 byte header - - 2 bits for encoding type - - 3 bits for bytes required for repeating value - - 3 bits for repeat count (MIN_REPEAT + run length) - # Blob - repeat value (fixed bytes) - -|DIRECT|: Used for random integer sequences whose number of bit requirement -doesn't vary a lot. - # 2 bytes header - - 1st byte - > 2 bits for encoding type - > 5 bits for fixed bit width of values in blob - > 1 bit for storing MSB of run length - - 2nd byte - > 8 bits for lower run length bits - # Blob - stores the direct values using fixed bit width. The length of the - data blob is (fixed width * run length) bits long - -|PATCHED_BASE|: Used for random integer sequences whose number of bit -requirement varies beyond a threshold. - - # 4 bytes header - - 1st byte - > 2 bits for encoding type - > 5 bits for fixed bit width of values in blob - > 1 bit for storing MSB of run length - - 2nd byte - > 8 bits for lower run length bits - - 3rd byte - > 3 bits for bytes required to encode base value - > 5 bits for patch width - - 4th byte - > 3 bits for patch gap width - > 5 bits for patch length - # Base value - Stored using fixed number of bytes. If MSB is set, base - value is negative else positive. Length of base value is (base width * 8) - bits. - # Data blob - Base reduced values as stored using fixed bit width. Length - of data blob is (fixed width * run length) bits. - # Patch blob - Patch blob is a list of gap and patch value. Each entry in - the patch list is (patch width + patch gap width) bits long. Gap between the - subsequent elements to be patched are stored in upper part of entry whereas - patch values are stored in lower part of entry. Length of patch blob is - ((patch width + patch gap width) * patch length) bits. - -|DELTA|: Used for monotonically increasing or decreasing sequences, sequences -with fixed delta values or long repeated sequences. - # 2 bytes header - - 1st byte - > 2 bits for encoding type - > 5 bits for fixed bit width of values in blob - > 1 bit for storing MSB of run length - - 2nd byte - > 8 bits for lower run length bits - # Base value - encoded as varint - # Delta base - encoded as varint - # Delta blob - only positive values. monotonicity and orderness are decided - based on the sign of the base value and delta base - */ -class RunLengthIntegerWriter { -public: - // 如果RunLengthIntegerWriter用于输出带符号的数据(int8, int16, int32, int64), - // 则设置is_singed为true, 此时会使用ZigZag - // 编码以减少存储负数时使用的比特数 - explicit RunLengthIntegerWriter(OutStream* output, bool is_signed); - ~RunLengthIntegerWriter() {} - Status write(int64_t value); - Status flush(); - void get_position(PositionEntryWriter* index_entry, bool print) const; - - void print_position_debug_info() { - _output->print_position_debug_info(); - VLOG_TRACE << "_num_literals=" << _num_literals; - } - -private: - friend class RunLengthIntegerReader; - // 定义头部, 利用C的bitfield功能 - // Note: 百度的机器通常是LITTLE_ENDIAN的, ARM的机器未知, - // 这里处理LITTLE_ENDIAN, 如果遇到了BIG_ENDIAN请自行修改 - // - // 补充说明 - // 请注意： - // C++位域是不可移植的,位域在一个字节中的排列方向与CPU/OS的大小端关系没有明确的文档说明 - // - // A bit-field may have type int, unsigned int, or signed int. - // Whether the high-order bit position of a “plain” int bit-field is treated as - // a sign bit is implementation-defined. - // A bit-field is interpreted as an integral type consisting of the specified number of bits. - // - // An implementation may allocate any addressable storage unit large enough to - // hold a bit-field. If enough space remains, a bit-field that immediately follows - // another bit-field in a structure shall be packed into adjacent bits of the same unit. - // If insufficient space remains, whether a bit-field that does not fit is put into the - // next unit or overlaps adjacent units is implementation-defined. - // The order of allocation of bit-fields within a unit (high-order to low-order or - // low-order to high-order) is implementation-defined. - // The alignment of the addressable storage unit is unspecified. - // - // 即使在小端机器上，以下排列顺序也只是一种可能性较大的实现 - // 文档对位域的排列顺序说明，几乎都是implementation-defined或unspecified - // 对于实际的排列顺序，未来可以写个小函数来判断 -#if __BYTE_ORDER == __LITTLE_ENDIAN - struct ShortRepeatHead { - uint8_t run_length : 3, value_bytes : 3, type : 2; - char blob[]; - }; - - struct DirectHead { - uint8_t length_msb : 1, bit_width : 5, type : 2; - uint8_t length_low : 8; - char blob[]; - - uint16_t length() const { return (((uint16_t)length_msb) << 8) | length_low; } - void set_length(uint16_t length) { - length_msb = (length >> 8) & 0x01; - length_low = length & 0xff; - } - }; - - struct PatchedBaseHead { - uint8_t length_msb : 1, bit_width : 5, type : 2; - uint8_t length_low : 8; - uint8_t patch_width : 5, base_bytes : 3; - uint8_t patch_length : 5, gap_width : 3; - - char blob[]; - uint16_t length() const { return (((uint16_t)length_msb) << 8) | length_low; } - void set_length(uint16_t length) { - length_msb = (length >> 8) & 0x01; - length_low = length & 0xff; - } - }; - - struct DeltaHead { - uint8_t length_msb : 1, bit_width : 5, type : 2; - uint8_t length_low : 8; - char blob[]; - - uint16_t length() const { return (((uint16_t)length_msb) << 8) | length_low; } - void set_length(uint16_t length) { - length_msb = (length >> 8) & 0x01; - length_low = length & 0xff; - } - }; -#elif __BYTE_ORDER == __BIG_ENDIAN -#error "Target machine is big endian." - // 大端的时候, 可以使用下面的代码, 请自行测试 - - /* - struct ShortRepeatHead { - uint8_t type: 2, - value_bytes: 3, - repeat: 3; - char blob[]; - }; - - struct DirectHead { - uint16_t type: 2, - bit_width: 5, - length: 9; - char blob[]; - }; - - struct PatchedBase { - uint32_t type: 2, - bit_width: 5, - length: 9, - base_bytes: 3, - patch_width: 5, - gap_width: 3, - patch_length: 5; - char blob[]; - }; - struct DeltaHead { - uint16_t type: 2, - bit_width: 5, - length: 9; - char blob[]; - }; - */ -#else -#error "Endian order can not be determined." -#endif // DORIS_BE_SRC_OLAP_COLUMN_FILE_RUN_LENGTH_INTEGER_WRITER_H - enum EncodingType { - SHORT_REPEAT = 0, - DIRECT = 1, - PATCHED_BASE = 2, - DELTA = 3, - NONE_ENCODING = 4 - }; - -private: - void _clear(); - void _determined_encoding(); - void _init_literals(int64_t value); - void _prepare_patched_blob(); - Status _write_values(); - Status _write_short_repeat_values(); - Status _write_direct_values(); - Status _write_patched_base_values(); - Status _write_delta_values(); - - static const uint16_t MAX_SCOPE = 512; - static const uint16_t MIN_REPEAT = 3; // NOTE 不要修改这个值, 否则程序出错 - static const uint16_t MAX_SHORT_REPEAT_LENGTH = 10; - // MAX_PATCH_LIST原本只需要0.05*MAX_SCOPE, 然而为了防止percentile_bits()里 - // 与这里的浮点计算产生的误差, 这里直接放大两倍, 请不要修改这个值 - static const uint16_t MAX_PATCH_LIST = uint16_t(MAX_SCOPE * 0.1); - OutStream* _output; - bool _is_signed; - uint32_t _fixed_run_length; - uint32_t _var_run_length; - int64_t _prev_delta; - int64_t _literals[MAX_SCOPE]; - EncodingType _encoding; - uint16_t _num_literals; - int64_t _zig_zag_literals[MAX_SCOPE]; // for direct encoding - int64_t _base_reduced_literals[MAX_SCOPE]; // for patched base encoding - int64_t _adj_deltas[MAX_SCOPE - 1]; // for delta encoding - int64_t _fixed_delta; - uint32_t _zz_bits_90p; - uint32_t _zz_bits_100p; - uint32_t _br_bits_95p; - uint32_t _br_bits_100p; - uint32_t _bits_delta_max; - uint32_t _patch_width; - uint32_t _patch_gap_width; - uint32_t _patch_length; - int64_t _gap_vs_patch_list[MAX_PATCH_LIST]; - int64_t _min; - bool _is_fixed_delta; - - DISALLOW_COPY_AND_ASSIGN(RunLengthIntegerWriter); -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_RUN_LENGTH_INTEGER_WRITER_H \ No newline at end of file diff --git a/be/src/olap/rowset/segment_group.cpp b/be/src/olap/rowset/segment_group.cpp deleted file mode 100644 index d2eb084424..0000000000 --- a/be/src/olap/rowset/segment_group.cpp +++ /dev/null @@ -1,1029 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/segment_group.h" - -#include -#include -#include -#include -#include - -#include "olap/column_mapping.h" -#include "olap/data_dir.h" -#include "olap/row_block.h" -#include "olap/row_cursor.h" -#include "olap/rowset/column_data.h" -#include "olap/schema.h" -#include "olap/storage_engine.h" -#include "olap/utils.h" -#include "olap/wrapper_field.h" -#include "util/file_utils.h" -#include "util/stack_util.h" - -using std::ifstream; -using std::string; -using std::vector; - -namespace doris { - -#define SEGMENT_GROUP_PARAM_VALIDATE() \ - do { \ - if (!_index_loaded) { \ - LOG(WARNING) << "fail to find, index is not loaded. [segment_group_id=" \ - << _segment_group_id << "]"; \ - return Status::OLAPInternalError(OLAP_ERR_NOT_INITED); \ - } \ - } while (0); - -#define POS_PARAM_VALIDATE(pos) \ - do { \ - if (nullptr == pos) { \ - LOG(WARNING) << "fail to find, nullptr position parameter."; \ - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); \ - } \ - } while (0); - -#define SLICE_PARAM_VALIDATE(slice) \ - do { \ - if (nullptr == slice) { \ - LOG(WARNING) << "fail to find, nullptr slice parameter."; \ - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); \ - } \ - } while (0); - -SegmentGroup::SegmentGroup(int64_t tablet_id, const RowsetId& rowset_id, const TabletSchema* schema, - const std::string& rowset_path_prefix, Version version, bool delete_flag, - int32_t segment_group_id, int32_t num_segments) - : _tablet_id(tablet_id), - _rowset_id(rowset_id), - _schema(schema), - _rowset_path_prefix(rowset_path_prefix), - _version(version), - _delete_flag(delete_flag), - _segment_group_id(segment_group_id), - _num_segments(num_segments) { - _index_loaded = false; - _ref_count = 0; - _is_pending = false; - _partition_id = 0; - _txn_id = 0; - _short_key_length = 0; - _new_short_key_length = 0; - _short_key_buf = nullptr; - _new_segment_created = false; - _empty = false; - - for (size_t i = 0; i < _schema->num_short_key_columns(); ++i) { - const TabletColumn& column = _schema->column(i); - _short_key_columns.push_back(column); - _short_key_length += column.index_length() + 1; // 1 for null byte - if (column.type() == OLAP_FIELD_TYPE_CHAR || column.type() == OLAP_FIELD_TYPE_VARCHAR || - column.type() == OLAP_FIELD_TYPE_STRING) { - _new_short_key_length += sizeof(Slice) + 1; - } else { - _new_short_key_length += column.index_length() + 1; - } - } -} - -SegmentGroup::SegmentGroup(int64_t tablet_id, const RowsetId& rowset_id, const TabletSchema* schema, - const std::string& rowset_path_prefix, bool delete_flag, - int32_t segment_group_id, int32_t num_segments, bool is_pending, - TPartitionId partition_id, TTransactionId transaction_id) - : _tablet_id(tablet_id), - _rowset_id(rowset_id), - _schema(schema), - _rowset_path_prefix(rowset_path_prefix), - _delete_flag(delete_flag), - _segment_group_id(segment_group_id), - _num_segments(num_segments), - _is_pending(is_pending), - _partition_id(partition_id), - _txn_id(transaction_id) { - _version = {-1, -1}; - _load_id.set_hi(0); - _load_id.set_lo(0); - _index_loaded = false; - _ref_count = 0; - _short_key_length = 0; - _new_short_key_length = 0; - _short_key_buf = nullptr; - _new_segment_created = false; - _empty = false; - - for (size_t i = 0; i < _schema->num_short_key_columns(); ++i) { - const TabletColumn& column = _schema->column(i); - _short_key_columns.push_back(column); - _short_key_length += column.index_length() + 1; // 1 for null byte - if (column.type() == OLAP_FIELD_TYPE_CHAR || column.type() == OLAP_FIELD_TYPE_VARCHAR || - column.type() == OLAP_FIELD_TYPE_STRING) { - _new_short_key_length += sizeof(Slice) + 1; - } else { - _new_short_key_length += column.index_length() + 1; - } - } -} - -SegmentGroup::~SegmentGroup() { - delete[] _short_key_buf; - _current_file_handler.close(); - - for (size_t i = 0; i < _zone_maps.size(); ++i) { - SAFE_DELETE(_zone_maps[i].first); - SAFE_DELETE(_zone_maps[i].second); - } - _seg_pb_map.clear(); -} - -std::string SegmentGroup::_construct_file_name(int32_t segment_id, const string& suffix) const { - // during convert from old files, the segment group id == -1, but we want to convert - // it to 0 - int32_t tmp_sg_id = 0; - if (_segment_group_id > 0) { - tmp_sg_id = _segment_group_id; - } - std::string file_name = _rowset_id.to_string() + "_" + std::to_string(tmp_sg_id) + "_" + - std::to_string(segment_id) + suffix; - return file_name; -} - -std::string SegmentGroup::_construct_file_name(const RowsetId& rowset_id, int32_t segment_id, - const string& suffix) const { - std::string file_name = rowset_id.to_string() + "_" + std::to_string(_segment_group_id) + "_" + - std::to_string(segment_id) + suffix; - return file_name; -} - -std::string SegmentGroup::construct_index_file_path(const std::string& snapshot_path, - int32_t segment_id) const { - std::string file_path = snapshot_path; - file_path.append("/"); - file_path.append(_construct_file_name(segment_id, ".idx")); - return file_path; -} - -std::string SegmentGroup::construct_index_file_path(int32_t segment_id) const { - return construct_index_file_path(_rowset_path_prefix, segment_id); -} - -std::string SegmentGroup::construct_data_file_path(const std::string& snapshot_path, - int32_t segment_id) const { - std::string file_path = snapshot_path; - file_path.append("/"); - file_path.append(_construct_file_name(segment_id, ".dat")); - return file_path; -} - -std::string SegmentGroup::construct_data_file_path(int32_t segment_id) const { - return construct_data_file_path(_rowset_path_prefix, segment_id); -} - -void SegmentGroup::acquire() { - ++_ref_count; -} - -int64_t SegmentGroup::ref_count() { - return _ref_count; -} - -void SegmentGroup::release() { - --_ref_count; -} - -bool SegmentGroup::is_in_use() { - return _ref_count > 0; -} - -// you can not use SegmentGroup after delete_all_files(), or else unknown behavior occurs. -bool SegmentGroup::delete_all_files() { - bool success = true; - if (_empty) { - return success; - } - for (uint32_t seg_id = 0; seg_id < _num_segments; ++seg_id) { - // get full path for one segment - string index_path = construct_index_file_path(seg_id); - string data_path = construct_data_file_path(seg_id); - - VLOG_NOTICE << "delete index file. path=" << index_path; - if (remove(index_path.c_str()) != 0) { - // if the errno is not ENOENT, log the error msg. - // ENOENT stands for 'No such file or directory' - if (errno != ENOENT) { - char errmsg[64]; - LOG(WARNING) << "fail to delete index file. err=" << strerror_r(errno, errmsg, 64) - << ", path=" << index_path; - success = false; - } - } - - VLOG_NOTICE << "delete data file. path=" << data_path; - if (remove(data_path.c_str()) != 0) { - if (errno != ENOENT) { - char errmsg[64]; - LOG(WARNING) << "fail to delete data file. err=" << strerror_r(errno, errmsg, 64) - << ", path=" << data_path; - success = false; - } - } - } - return success; -} - -Status SegmentGroup::add_zone_maps_for_linked_schema_change( - const std::vector>& zone_map_fields, - const SchemaMapping& schema_mapping) { - //When add rollup tablet, the base tablet index maybe empty - if (zone_map_fields.size() == 0) { - return Status::OK(); - } - - // 1. rollup tablet get_num_zone_map_columns() will less than base tablet zone_map_fields.size(). - // For LinkedSchemaChange, the rollup tablet keys order is the same as base tablet - // 2. adding column to existed table, get_num_zone_map_columns() will larger than - // zone_map_fields.size() - int zonemap_col_num = get_num_zone_map_columns(); - CHECK(zonemap_col_num <= schema_mapping.size()) - << zonemap_col_num << " vs. " << schema_mapping.size(); - - for (size_t i = 0; i < zonemap_col_num; ++i) { - // in duplicate/unique table update from 0.11 to 0.12, zone map index may be missed and may not a new column. - if (_schema->keys_type() != AGG_KEYS && schema_mapping[i].ref_column != -1 && - schema_mapping[i].ref_column >= zone_map_fields.size()) { - // the sequence of columns in _zone_maps and _schema must be consistent, so here - // process should not add missed zonemap and we break the loop. - break; - } - const TabletColumn& column = _schema->column(i); - - // nullptr is checked in olap_cond.cpp eval, note: When we apply column statistic, Field can be nullptr when type is Varchar. - WrapperField* first = nullptr; - WrapperField* second = nullptr; - - // when this is no ref_column (add new column), fill default value - if (schema_mapping[i].ref_column == -1 || - schema_mapping[i].ref_column >= zone_map_fields.size()) { - // ref_column == -1 means this is a new column. - // for new column, use default value to fill into column_statistics - if (schema_mapping[i].default_value != nullptr) { - first = WrapperField::create(column); - DCHECK(first != nullptr) << "failed to allocate memory for field: " << i; - first->copy(schema_mapping[i].default_value); - second = WrapperField::create(column); - DCHECK(second != nullptr) << "failed to allocate memory for field: " << i; - second->copy(schema_mapping[i].default_value); - } - } else { - WrapperField* wfirst = zone_map_fields[schema_mapping[i].ref_column].first; - WrapperField* wsecond = zone_map_fields[schema_mapping[i].ref_column].second; - - if (wfirst != nullptr) { - first = WrapperField::create(column); - DCHECK(first != nullptr) << "failed to allocate memory for field: " << i; - first->copy(wfirst); - } - - if (wsecond != nullptr) { - second = WrapperField::create(column); - DCHECK(second != nullptr) << "failed to allocate memory for field: " << i; - second->copy(wsecond); - } - } - - // first and second can be nullptr, because when type is Varchar then default_value and zone_map_fields in old column - // can be nullptr, and it is checked in olap_cond.cpp eval function. - _zone_maps.push_back(std::make_pair(first, second)); - } - - return Status::OK(); -} - -Status SegmentGroup::add_zone_maps( - const std::vector>& zone_map_fields) { - DCHECK(_empty || zone_map_fields.size() == get_num_zone_map_columns()); - for (size_t i = 0; i < zone_map_fields.size(); ++i) { - const TabletColumn& column = _schema->column(i); - WrapperField* first = WrapperField::create(column); - DCHECK(first != nullptr) << "failed to allocate memory for field: " << i; - first->copy(zone_map_fields[i].first); - - WrapperField* second = WrapperField::create(column); - DCHECK(second != nullptr) << "failed to allocate memory for field: " << i; - second->copy(zone_map_fields[i].second); - - _zone_maps.push_back(std::make_pair(first, second)); - } - return Status::OK(); -} - -Status SegmentGroup::add_zone_maps( - std::vector>& zone_map_strings, - std::vector& null_vec) { - DCHECK(_empty || zone_map_strings.size() <= get_num_zone_map_columns()); - for (size_t i = 0; i < zone_map_strings.size(); ++i) { - const TabletColumn& column = _schema->column(i); - WrapperField* first = WrapperField::create(column); - DCHECK(first != nullptr) << "failed to allocate memory for field: " << i; - RETURN_NOT_OK(first->from_string(zone_map_strings[i].first)); - if (null_vec[i]) { - //[min, max] -> [nullptr, max] - first->set_null(); - } - WrapperField* second = WrapperField::create(column); - DCHECK(first != nullptr) << "failed to allocate memory for field: " << i; - RETURN_NOT_OK(second->from_string(zone_map_strings[i].second)); - _zone_maps.push_back(std::make_pair(first, second)); - } - return Status::OK(); -} - -Status SegmentGroup::load(bool use_cache) { - if (_empty) { - _index_loaded = true; - return Status::OK(); - } - Status res = Status::OLAPInternalError(OLAP_ERR_INDEX_LOAD_ERROR); - std::lock_guard guard(_index_load_lock); - - if (_index_loaded) { - return Status::OK(); - } - - if (_num_segments == 0) { - LOG(WARNING) << "fail to load index, segments number is 0."; - return res; - } - - if (_index.init(_short_key_length, _new_short_key_length, _schema->num_short_key_columns(), - &_short_key_columns) != Status::OK()) { - LOG(WARNING) << "fail to create MemIndex. num_segment=" << _num_segments; - return res; - } - - // for each segment - for (uint32_t seg_id = 0; seg_id < _num_segments; ++seg_id) { - string seg_path = construct_data_file_path(seg_id); - if (!(res = load_pb(seg_path.c_str(), seg_id))) { - LOG(WARNING) << "failed to load pb structures. [seg_path='" << seg_path << "']"; - - return res; - } - - // get full path for one segment - std::string path = construct_index_file_path(seg_id); - if ((res = _index.load_segment(path.c_str(), &_current_num_rows_per_row_block, - use_cache)) != Status::OK()) { - LOG(WARNING) << "fail to load segment. [path='" << path << "']"; - - return res; - } - } - - _delete_flag = _index.delete_flag(); - _index_loaded = true; - - return Status::OK(); -} - -Status SegmentGroup::load_pb(const char* file, uint32_t seg_id) { - Status res = Status::OK(); - - FileHeader seg_file_header; - FileHandler seg_file_handler; - res = seg_file_handler.open(file, O_RDONLY); - if (!res.ok()) { - LOG(WARNING) << "failed to open segment file. err=" << res << ", file=" << file; - return res; - } - - res = seg_file_header.unserialize(&seg_file_handler); - if (!res.ok()) { - seg_file_handler.close(); - LOG(WARNING) << "fail to unserialize header. err=" << res << ", path=" << file; - return res; - } - - _seg_pb_map[seg_id] = seg_file_header; - seg_file_handler.close(); - return Status::OK(); -} - -bool SegmentGroup::index_loaded() { - return _index_loaded; -} - -Status SegmentGroup::validate() { - if (_empty) { - return Status::OK(); - } - - Status res = Status::OK(); - for (uint32_t seg_id = 0; seg_id < _num_segments; ++seg_id) { - FileHeader index_file_header; - FileHeader data_file_header; - - // get full path for one segment - string index_path = construct_index_file_path(seg_id); - string data_path = construct_data_file_path(seg_id); - - // 检查index文件头 - if ((res = index_file_header.validate(index_path)) != Status::OK()) { - LOG(WARNING) << "validate index file error. [file='" << index_path << "']"; - return res; - } - - // 检查data文件头 - if ((res = data_file_header.validate(data_path)) != Status::OK()) { - LOG(WARNING) << "validate data file error. [file='" << data_path << "']"; - return res; - } - } - - return Status::OK(); -} - -bool SegmentGroup::check() { - // if the segment group is converted from old files, _empty == false but _num_segments == 0 - if (_empty && (_num_segments > 0 || !zero_num_rows())) { - LOG(WARNING) << "invalid num segments for empty segment group, _num_segments:" - << _num_segments << ",num rows:" << num_rows(); - return false; - } - return true; -} - -Status SegmentGroup::find_short_key(const RowCursor& key, RowCursor* helper_cursor, bool find_last, - RowBlockPosition* pos) const { - SEGMENT_GROUP_PARAM_VALIDATE(); - POS_PARAM_VALIDATE(pos); - - // 由于find会从前一个segment找起，如果前一个segment中恰好没有该key， - // 就用前移后移来移动segment的位置. - OLAPIndexOffset offset = _index.find(key, helper_cursor, find_last); - if (offset.offset > 0) { - offset.offset = offset.offset - 1; - - OLAPIndexOffset next_offset = _index.next(offset); - if (!(next_offset == _index.end())) { - offset = next_offset; - } - } - - VLOG_NOTICE << "seg=" << offset.segment << ", offset=" << offset.offset; - return _index.get_row_block_position(offset, pos); -} - -Status SegmentGroup::get_row_block_entry(const RowBlockPosition& pos, EntrySlice* entry) const { - SEGMENT_GROUP_PARAM_VALIDATE(); - SLICE_PARAM_VALIDATE(entry); - - return _index.get_entry(_index.get_offset(pos), entry); -} - -Status SegmentGroup::find_first_row_block(RowBlockPosition* position) const { - SEGMENT_GROUP_PARAM_VALIDATE(); - POS_PARAM_VALIDATE(position); - - return _index.get_row_block_position(_index.find_first(), position); -} - -Status SegmentGroup::find_last_row_block(RowBlockPosition* position) const { - SEGMENT_GROUP_PARAM_VALIDATE(); - POS_PARAM_VALIDATE(position); - - return _index.get_row_block_position(_index.find_last(), position); -} - -Status SegmentGroup::find_next_row_block(RowBlockPosition* pos, bool* eof) const { - SEGMENT_GROUP_PARAM_VALIDATE(); - POS_PARAM_VALIDATE(pos); - POS_PARAM_VALIDATE(eof); - - OLAPIndexOffset current = _index.get_offset(*pos); - *eof = false; - - OLAPIndexOffset next = _index.next(current); - if (next == _index.end()) { - *eof = true; - return Status::OLAPInternalError(OLAP_ERR_INDEX_EOF); - } - - return _index.get_row_block_position(next, pos); -} - -Status SegmentGroup::find_mid_point(const RowBlockPosition& low, const RowBlockPosition& high, - RowBlockPosition* output, uint32_t* dis) const { - *dis = compute_distance(low, high); - if (*dis >= _index.count()) { - return Status::OLAPInternalError(OLAP_ERR_INDEX_EOF); - } else { - *output = low; - if (advance_row_block(*dis / 2, output) != Status::OK()) { - return Status::OLAPInternalError(OLAP_ERR_INDEX_EOF); - } - - return Status::OK(); - } -} - -Status SegmentGroup::find_prev_point(const RowBlockPosition& current, - RowBlockPosition* prev) const { - OLAPIndexOffset current_offset = _index.get_offset(current); - OLAPIndexOffset prev_offset = _index.prev(current_offset); - - return _index.get_row_block_position(prev_offset, prev); -} - -Status SegmentGroup::advance_row_block(int64_t num_row_blocks, RowBlockPosition* position) const { - SEGMENT_GROUP_PARAM_VALIDATE(); - POS_PARAM_VALIDATE(position); - - OLAPIndexOffset off = _index.get_offset(*position); - iterator_offset_t absolute_offset = _index.get_absolute_offset(off) + num_row_blocks; - if (absolute_offset >= _index.count()) { - return Status::OLAPInternalError(OLAP_ERR_INDEX_EOF); - } - - return _index.get_row_block_position(_index.get_relative_offset(absolute_offset), position); -} - -// PRECONDITION position1 < position2 -uint32_t SegmentGroup::compute_distance(const RowBlockPosition& position1, - const RowBlockPosition& position2) const { - iterator_offset_t offset1 = _index.get_absolute_offset(_index.get_offset(position1)); - iterator_offset_t offset2 = _index.get_absolute_offset(_index.get_offset(position2)); - - return offset2 > offset1 ? offset2 - offset1 : 0; -} - -Status SegmentGroup::add_segment() { - // 打开文件 - ++_num_segments; - - OLAPIndexHeaderMessage* index_header = nullptr; - // 构造Proto格式的Header - index_header = _file_header.mutable_message(); - index_header->set_start_version(_version.first); - index_header->set_end_version(_version.second); - // Version hash is useless but it is a required field in header message pb - index_header->set_cumulative_version_hash(0); - index_header->set_segment(_num_segments - 1); - index_header->set_num_rows_per_block(_schema->num_rows_per_row_block()); - index_header->set_delete_flag(_delete_flag); - index_header->set_null_supported(true); - - // 分配一段存储short key的内存, 初始化index_row - if (_short_key_buf == nullptr) { - _short_key_buf = new (std::nothrow) char[_short_key_length]; - if (_short_key_buf == nullptr) { - LOG(WARNING) << "malloc short_key_buf error."; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - memset(_short_key_buf, 0, _short_key_length); - if (_current_index_row.init(*_schema) != Status::OK()) { - LOG(WARNING) << "init _current_index_row fail."; - return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); - } - } - - // 初始化checksum - _checksum = ADLER32_INIT; - return Status::OK(); -} - -Status SegmentGroup::add_row_block(const RowBlock& row_block, const uint32_t data_offset) { - // get first row of the row_block to distill index item. - row_block.get_row(0, &_current_index_row); - return add_short_key(_current_index_row, data_offset); -} - -Status SegmentGroup::add_short_key(const RowCursor& short_key, const uint32_t data_offset) { - Status res = Status::OK(); - if (!_new_segment_created) { - string file_path = construct_index_file_path(_num_segments - 1); - StorageEngine* engine = StorageEngine::instance(); - if (engine != nullptr) { - std::filesystem::path tablet_path(_rowset_path_prefix); - std::filesystem::path data_dir_path = - tablet_path.parent_path().parent_path().parent_path().parent_path(); - std::string data_dir_string = data_dir_path.string(); - DataDir* data_dir = engine->get_store(data_dir_string); - data_dir->add_pending_ids(ROWSET_ID_PREFIX + _rowset_id.to_string()); - } - res = _current_file_handler.open_with_mode(file_path.c_str(), O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR); - if (!res.ok()) { - char errmsg[64]; - LOG(WARNING) << "can not create file. file_path=" << file_path << ", err='" - << strerror_r(errno, errmsg, 64); - return res; - } - _new_segment_created = true; - - // 准备FileHeader - if ((res = _file_header.prepare(&_current_file_handler)) != Status::OK()) { - LOG(WARNING) << "write file header error. [err=" << Errno::str() << "]"; - return res; - } - - // 跳过FileHeader - if (_current_file_handler.seek(_file_header.size(), SEEK_SET) == -1) { - LOG(WARNING) << "lseek header file error. [err=" << Errno::str() << "]"; - res = Status::OLAPInternalError(OLAP_ERR_IO_ERROR); - return res; - } - } - - // 将short key的内容写入_short_key_buf - size_t offset = 0; - - //short_key.write_null_array(_short_key_buf); - //offset += short_key.get_num_null_byte(); - for (size_t i = 0; i < _short_key_columns.size(); i++) { - short_key.write_index_by_index(i, _short_key_buf + offset); - offset += short_key.get_index_size(i) + 1; - } - - // 写入Short Key对应的数据 - if ((res = _current_file_handler.write(_short_key_buf, _short_key_length)) != Status::OK()) { - LOG(WARNING) << "write short key failed. [err=" << Errno::str() << "]"; - - return res; - } - - // 写入对应的数据文件偏移量 - if ((res = _current_file_handler.write(&data_offset, sizeof(data_offset))) != Status::OK()) { - LOG(WARNING) << "write data_offset failed. [err=" << Errno::str() << "]"; - return res; - } - - _checksum = olap_adler32(_checksum, _short_key_buf, _short_key_length); - _checksum = olap_adler32(_checksum, reinterpret_cast(&data_offset), - sizeof(data_offset)); - return Status::OK(); -} - -Status SegmentGroup::finalize_segment(uint32_t data_segment_size, int64_t num_rows) { - // 准备FileHeader - Status res = Status::OK(); - - int file_length = _current_file_handler.tell(); - if (file_length == -1) { - LOG(WARNING) << "get file_length error. err=" << Errno::no() - << ", _new_segment_created=" << _new_segment_created; - return Status::OLAPInternalError(OLAP_ERR_IO_ERROR); - } - - _file_header.set_file_length(file_length); - _file_header.set_checksum(_checksum); - _file_header.mutable_extra()->data_length = data_segment_size; - _file_header.mutable_extra()->num_rows = num_rows; - - // 写入更新之后的FileHeader - if ((res = _file_header.serialize(&_current_file_handler)) != Status::OK()) { - LOG(WARNING) << "write file header error. [err=" << Errno::str() << "]"; - - return res; - } - - VLOG_NOTICE << "finalize_segment. file_name=" << _current_file_handler.file_name() - << ", file_length=" << file_length; - - if ((res = _current_file_handler.close()) != Status::OK()) { - LOG(WARNING) << "close file error. [err=" << Errno::str() << "]"; - - return res; - } - - _new_segment_created = false; - return Status::OK(); -} - -uint64_t SegmentGroup::num_index_entries() const { - return _index.count(); -} - -size_t SegmentGroup::current_num_rows_per_row_block() const { - return _current_num_rows_per_row_block; -} - -const TabletSchema& SegmentGroup::get_tablet_schema() { - return *_schema; -} - -int SegmentGroup::get_num_zone_map_columns() { - return _schema->num_key_columns(); -} - -int SegmentGroup::get_num_key_columns() { - return _schema->num_key_columns(); -} - -int SegmentGroup::get_num_short_key_columns() { - return _schema->num_short_key_columns(); -} - -size_t SegmentGroup::get_num_rows_per_row_block() { - return _schema->num_rows_per_row_block(); -} - -std::string SegmentGroup::rowset_path_prefix() { - return _rowset_path_prefix; -} - -int64_t SegmentGroup::get_tablet_id() { - return _tablet_id; -} - -Status SegmentGroup::copy_files_to(const std::string& dir) { - if (_empty) { - return Status::OK(); - } - for (int segment_id = 0; segment_id < _num_segments; segment_id++) { - std::string dest_data_file = construct_data_file_path(dir, segment_id); - if (FileUtils::check_exist(dest_data_file)) { - LOG(WARNING) << "file already exists:" << dest_data_file; - return Status::OLAPInternalError(OLAP_ERR_FILE_ALREADY_EXIST); - } - std::string data_file_to_copy = construct_data_file_path(segment_id); - if (!FileUtils::copy_file(data_file_to_copy, dest_data_file).ok()) { - LOG(WARNING) << "fail to copy data file. from=" << data_file_to_copy - << ", to=" << dest_data_file << ", errno=" << Errno::no(); - return Status::OLAPInternalError(OLAP_ERR_OS_ERROR); - } - std::string dest_index_file = construct_index_file_path(dir, segment_id); - if (FileUtils::check_exist(dest_index_file)) { - LOG(WARNING) << "file already exists:" << dest_index_file; - return Status::OLAPInternalError(OLAP_ERR_FILE_ALREADY_EXIST); - } - std::string index_file_to_copy = construct_index_file_path(segment_id); - if (!FileUtils::copy_file(index_file_to_copy, dest_index_file).ok()) { - LOG(WARNING) << "fail to copy index file. from=" << index_file_to_copy - << ", to=" << dest_index_file << ", errno=" << Errno::no(); - return Status::OLAPInternalError(OLAP_ERR_OS_ERROR); - } - } - return Status::OK(); -} - -// when convert from old files, remove existing files -// convert from old files in 2 cases: -// case 1: clone from old version be -// case 2: upgrade to new version be -Status SegmentGroup::convert_from_old_files(const std::string& snapshot_path, - std::vector* success_links) { - if (_empty) { - // the segment group is empty, it does not have files, just return - return Status::OK(); - } - for (int segment_id = 0; segment_id < _num_segments; segment_id++) { - std::string new_data_file_name = construct_data_file_path(_rowset_path_prefix, segment_id); - // if file exist should remove it because same file name does not mean same data - if (FileUtils::check_exist(new_data_file_name)) { - LOG(INFO) << "file already exist, remove it. file=" << new_data_file_name; - RETURN_WITH_WARN_IF_ERROR(FileUtils::remove(new_data_file_name), - Status::OLAPInternalError(OLAP_ERR_CANNOT_CREATE_DIR), - "remove path failed. path=" + new_data_file_name); - } - std::string old_data_file_name = construct_old_data_file_path(snapshot_path, segment_id); - if (link(old_data_file_name.c_str(), new_data_file_name.c_str()) != 0) { - LOG(WARNING) << "fail to create hard link. from=" << old_data_file_name - << ", to=" << new_data_file_name << ", errno=" << Errno::no(); - return Status::OLAPInternalError(OLAP_ERR_OS_ERROR); - } else { - VLOG_NOTICE << "link data file from " << old_data_file_name << " to " - << new_data_file_name << " successfully"; - } - success_links->push_back(new_data_file_name); - std::string new_index_file_name = - construct_index_file_path(_rowset_path_prefix, segment_id); - if (FileUtils::check_exist(new_index_file_name)) { - LOG(INFO) << "file already exist, remove it. file=" << new_index_file_name; - - RETURN_WITH_WARN_IF_ERROR(FileUtils::remove(new_index_file_name), - Status::OLAPInternalError(OLAP_ERR_CANNOT_CREATE_DIR), - "remove path failed. path=" + new_index_file_name); - } - std::string old_index_file_name = construct_old_index_file_path(snapshot_path, segment_id); - if (link(old_index_file_name.c_str(), new_index_file_name.c_str()) != 0) { - LOG(WARNING) << "fail to create hard link. from=" << old_index_file_name - << ", to=" << new_index_file_name << ", errno=" << Errno::no(); - return Status::OLAPInternalError(OLAP_ERR_OS_ERROR); - } else { - VLOG_NOTICE << "link index file from " << old_index_file_name << " to " - << new_index_file_name << " successfully"; - } - success_links->push_back(new_index_file_name); - } - return Status::OK(); -} - -Status SegmentGroup::convert_to_old_files(const std::string& snapshot_path, - std::vector* success_links) { - if (_empty) { - return Status::OK(); - } - for (int segment_id = 0; segment_id < _num_segments; segment_id++) { - std::string new_data_file_name = construct_data_file_path(_rowset_path_prefix, segment_id); - std::string old_data_file_name = construct_old_data_file_path(snapshot_path, segment_id); - if (!FileUtils::check_exist(old_data_file_name)) { - if (link(new_data_file_name.c_str(), old_data_file_name.c_str()) != 0) { - LOG(WARNING) << "fail to create hard link. from=" << new_data_file_name << ", " - << "to=" << old_data_file_name << ", " - << "errno=" << Errno::no(); - return Status::OLAPInternalError(OLAP_ERR_OS_ERROR); - } - success_links->push_back(old_data_file_name); - } - VLOG_NOTICE << "create hard link. from=" << new_data_file_name << ", " - << "to=" << old_data_file_name; - std::string new_index_file_name = - construct_index_file_path(_rowset_path_prefix, segment_id); - std::string old_index_file_name = construct_old_index_file_path(snapshot_path, segment_id); - if (!FileUtils::check_exist(old_index_file_name)) { - if (link(new_index_file_name.c_str(), old_index_file_name.c_str()) != 0) { - LOG(WARNING) << "fail to create hard link. from=" << new_index_file_name << ", " - << "to=" << old_index_file_name << ", " - << "errno=" << Errno::no(); - return Status::OLAPInternalError(OLAP_ERR_OS_ERROR); - } - success_links->push_back(old_index_file_name); - } - VLOG_NOTICE << "create hard link. from=" << new_index_file_name << ", " - << "to=" << old_index_file_name; - } - return Status::OK(); -} - -Status SegmentGroup::remove_old_files(std::vector* links_to_remove) { - for (int segment_id = 0; segment_id < _num_segments; segment_id++) { - std::string old_data_file_name = - construct_old_data_file_path(_rowset_path_prefix, segment_id); - if (FileUtils::check_exist(old_data_file_name)) { - RETURN_WITH_WARN_IF_ERROR(FileUtils::remove(old_data_file_name), - Status::OLAPInternalError(OLAP_ERR_CANNOT_CREATE_DIR), - "remove path failed. path" + old_data_file_name); - - links_to_remove->push_back(old_data_file_name); - } - std::string old_index_file_name = - construct_old_index_file_path(_rowset_path_prefix, segment_id); - if (FileUtils::check_exist(old_index_file_name)) { - RETURN_WITH_WARN_IF_ERROR(FileUtils::remove(old_index_file_name), - Status::OLAPInternalError(OLAP_ERR_CANNOT_CREATE_DIR), - "remove path failed. path" + old_index_file_name); - - links_to_remove->push_back(old_index_file_name); - } - // if segment group id == 0, it maybe convert from old files which do not have segment group id in file path - if (_segment_group_id == 0) { - old_data_file_name = _construct_err_sg_data_file_path(_rowset_path_prefix, segment_id); - if (FileUtils::check_exist(old_data_file_name)) { - RETURN_WITH_WARN_IF_ERROR(FileUtils::remove(old_data_file_name), - Status::OLAPInternalError(OLAP_ERR_CANNOT_CREATE_DIR), - "remove path failed. path" + old_data_file_name); - links_to_remove->push_back(old_data_file_name); - } - old_index_file_name = - _construct_err_sg_index_file_path(_rowset_path_prefix, segment_id); - if (FileUtils::check_exist(old_index_file_name)) { - RETURN_WITH_WARN_IF_ERROR(FileUtils::remove(old_index_file_name), - Status::OLAPInternalError(OLAP_ERR_CANNOT_CREATE_DIR), - "remove path failed. path" + old_index_file_name); - - links_to_remove->push_back(old_index_file_name); - } - } - } - std::string pending_delta_path = _rowset_path_prefix + PENDING_DELTA_PREFIX; - if (FileUtils::check_exist(pending_delta_path)) { - LOG(INFO) << "remove pending delta path:" << pending_delta_path; - RETURN_WITH_WARN_IF_ERROR(FileUtils::remove_all(pending_delta_path), - Status::OLAPInternalError(OLAP_ERR_CANNOT_CREATE_DIR), - "remove path failed. path" + pending_delta_path); - } - return Status::OK(); -} - -Status SegmentGroup::link_segments_to_path(const std::string& dest_path, - const RowsetId& rowset_id) { - if (dest_path.empty()) { - LOG(WARNING) << "dest path is empty, return error"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - for (int segment_id = 0; segment_id < _num_segments; segment_id++) { - std::string data_file_name = _construct_file_name(rowset_id, segment_id, ".dat"); - std::string new_data_file_path = dest_path + "/" + data_file_name; - if (!FileUtils::check_exist(new_data_file_path)) { - std::string origin_data_file_path = - construct_data_file_path(_rowset_path_prefix, segment_id); - if (link(origin_data_file_path.c_str(), new_data_file_path.c_str()) != 0) { - LOG(WARNING) << "fail to create hard link. from=" << origin_data_file_path - << ", to=" << new_data_file_path << ", error=" << strerror(errno); - return Status::OLAPInternalError(OLAP_ERR_OS_ERROR); - } - } - std::string index_file_name = _construct_file_name(rowset_id, segment_id, ".idx"); - std::string new_index_file_path = dest_path + "/" + index_file_name; - if (!FileUtils::check_exist(new_index_file_path)) { - std::string origin_idx_file_path = - construct_index_file_path(_rowset_path_prefix, segment_id); - if (link(origin_idx_file_path.c_str(), new_index_file_path.c_str()) != 0) { - LOG(WARNING) << "fail to create hard link. from=" << origin_idx_file_path - << ", to=" << new_index_file_path << ", error=" << strerror(errno); - return Status::OLAPInternalError(OLAP_ERR_OS_ERROR); - } - } - } - return Status::OK(); -} - -std::string SegmentGroup::construct_old_index_file_path(const std::string& path_prefix, - int32_t segment_id) const { - if (_is_pending) { - return _construct_old_pending_file_path(path_prefix, segment_id, ".idx"); - } else { - return _construct_old_file_path(path_prefix, segment_id, ".idx"); - } -} - -std::string SegmentGroup::construct_old_data_file_path(const std::string& path_prefix, - int32_t segment_id) const { - if (_is_pending) { - return _construct_old_pending_file_path(path_prefix, segment_id, ".dat"); - } else { - return _construct_old_file_path(path_prefix, segment_id, ".dat"); - } -} - -std::string SegmentGroup::_construct_err_sg_index_file_path(const std::string& path_prefix, - int32_t segment_id) const { - if (_is_pending) { - return _construct_old_pending_file_path(path_prefix, segment_id, ".idx"); - } else { - return _construct_err_sg_file_path(path_prefix, segment_id, ".idx"); - } -} - -std::string SegmentGroup::_construct_err_sg_data_file_path(const std::string& path_prefix, - int32_t segment_id) const { - if (_is_pending) { - return _construct_old_pending_file_path(path_prefix, segment_id, ".dat"); - } else { - return _construct_err_sg_file_path(path_prefix, segment_id, ".dat"); - } -} - -std::string SegmentGroup::_construct_old_pending_file_path(const std::string& path_prefix, - int32_t segment_id, - const std::string& suffix) const { - std::stringstream file_path; - file_path << path_prefix << "/" << PENDING_DELTA_PREFIX << "/" << _txn_id << "_" - << _segment_group_id << "_" << segment_id << suffix; - return file_path.str(); -} - -std::string SegmentGroup::_construct_old_file_path(const std::string& path_prefix, - int32_t segment_id, - const std::string& suffix) const { - char file_path[OLAP_MAX_PATH_LEN]; - if (_segment_group_id == -1) { - snprintf(file_path, sizeof(file_path), "%s/%ld_%ld_%ld_%d%s", path_prefix.c_str(), - _tablet_id, _version.first, _version.second, segment_id, suffix.c_str()); - } else { - snprintf(file_path, sizeof(file_path), "%s/%ld_%ld_%ld_%d_%d%s", path_prefix.c_str(), - _tablet_id, _version.first, _version.second, _segment_group_id, segment_id, - suffix.c_str()); - } - - return file_path; -} - -// construct file path for sg_id == -1 -std::string SegmentGroup::_construct_err_sg_file_path(const std::string& path_prefix, - int32_t segment_id, - const std::string& suffix) const { - char file_path[OLAP_MAX_PATH_LEN]; - snprintf(file_path, sizeof(file_path), "%s/%ld_%ld_%ld_%d%s", path_prefix.c_str(), _tablet_id, - _version.first, _version.second, segment_id, suffix.c_str()); - - return file_path; -} - -} // namespace doris diff --git a/be/src/olap/rowset/segment_group.h b/be/src/olap/rowset/segment_group.h deleted file mode 100644 index e2031b7dd7..0000000000 --- a/be/src/olap/rowset/segment_group.h +++ /dev/null @@ -1,312 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_SEGMENT_GROUP_H -#define DORIS_BE_SRC_OLAP_ROWSET_SEGMENT_GROUP_H - -#include -#include -#include -#include -#include -#include -#include - -#include "gen_cpp/column_data_file.pb.h" -#include "gen_cpp/olap_file.pb.h" -#include "olap/column_mapping.h" -#include "olap/field.h" -#include "olap/file_helper.h" -#include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/olap_index.h" -#include "olap/row_cursor.h" -#include "olap/utils.h" - -namespace doris { - -// Class for segments management -// For fast key lookup, we maintain a sparse index for every data file. The -// index is sparse because we only have one pointer per row block. Each -// index entry contains the short key for the first row of the -// corresponding row block -class SegmentGroup { - friend class MemIndex; - -public: - SegmentGroup(int64_t tablet_id, const RowsetId& rowset_id, const TabletSchema* tablet_schema, - const std::string& rowset_path_prefix, Version version, bool delete_flag, - int segment_group_id, int32_t num_segments); - - SegmentGroup(int64_t tablet_id, const RowsetId& rowset_id, const TabletSchema* tablet_schema, - const std::string& rowset_path_prefix, bool delete_flag, int32_t segment_group_id, - int32_t num_segments, bool is_pending, TPartitionId partition_id, - TTransactionId transaction_id); - - virtual ~SegmentGroup(); - - // Load the index into memory. - Status load(bool use_cache = true); - bool index_loaded(); - Status load_pb(const char* file, uint32_t seg_id); - - bool has_zone_maps() { return _zone_maps.size() != 0; } - - Status add_zone_maps_for_linked_schema_change( - const std::vector>& zone_map_fields, - const SchemaMapping& schema_mapping); - - Status add_zone_maps( - const std::vector>& zone_map_fields); - - Status add_zone_maps(std::vector>& zone_map_strings, - std::vector& null_vec); - - const std::vector>& get_zone_maps() { - return _zone_maps; - } - - // 检查index文件和data文件的有效性 - Status validate(); - - // this function should be called after load - bool check(); - - // Finds position of first row block contain the smallest key equal - // to or greater than 'key'. Returns true on success. - Status find_short_key(const RowCursor& key, RowCursor* helper_cursor, bool find_last, - RowBlockPosition* position) const; - - // Returns position of the first row block in the index. - Status find_first_row_block(RowBlockPosition* position) const; - - // Returns position of the last row block in the index. - Status find_last_row_block(RowBlockPosition* position) const; - - // Given the position of a row block, finds position of the next block. - // Sets eof to tru if there are no more blocks to go through, and - // returns false. Returns true on success. - Status find_next_row_block(RowBlockPosition* position, bool* eof) const; - - // Given two positions in an index, low and high, set output to be - // the midpoint between those two positions. Returns the distance - // between low and high as computed by ComputeDistance. - Status find_mid_point(const RowBlockPosition& low, const RowBlockPosition& high, - RowBlockPosition* output, uint32_t* dis) const; - - Status find_prev_point(const RowBlockPosition& current, RowBlockPosition* prev) const; - - Status get_row_block_entry(const RowBlockPosition& pos, EntrySlice* entry) const; - - // Given a starting row block position, advances the position by - // num_row_blocks, then stores back the new position through the - // pointer. Returns true on success, false on attempt to seek past - // the last block. - Status advance_row_block(int64_t num_row_blocks, RowBlockPosition* position) const; - - // Computes the distance between two positions, in row blocks. - uint32_t compute_distance(const RowBlockPosition& position1, - const RowBlockPosition& position2) const; - - // The following four functions are used for creating new index - // files. AddSegment() and FinalizeSegment() start and end a new - // segment respectively, while IndexRowBlock() and IndexShortKey() - // add a new index entry to the current segment. - Status add_segment(); - Status add_short_key(const RowCursor& short_key, const uint32_t data_offset); - Status add_row_block(const RowBlock& row_block, const uint32_t data_offset); - Status finalize_segment(uint32_t data_segment_size, int64_t num_rows); - - // reference count - void acquire(); - void release(); - bool is_in_use(); - int64_t ref_count(); - - // delete all files (*.idx; *.dat) - bool delete_all_files(); - - Version version() const { return _version; } - void set_version(Version version) { _version = version; } - - bool is_pending() const { return _is_pending; } - void set_pending_finished() { _is_pending = false; } - - TPartitionId partition_id() const { return _partition_id; } - TTransactionId transaction_id() const { return _txn_id; } - - bool delete_flag() const { return _delete_flag; } - - int32_t segment_group_id() const { return _segment_group_id; } - void set_segment_group_id(int32_t segment_group_id) { _segment_group_id = segment_group_id; } - - PUniqueId load_id() const { return _load_id; } - void set_load_id(const PUniqueId& load_id) { _load_id = load_id; } - - int32_t num_segments() const { return _num_segments; } - void set_num_segments(int32_t num_segments) { _num_segments = num_segments; } - - size_t index_size() const { return _index.index_size(); } - - size_t data_size() const { return _index.data_size(); } - - int64_t num_rows() const { return _index.num_rows(); } - - const size_t short_key_length() const { return _short_key_length; } - - const size_t new_short_key_length() const { return _new_short_key_length; } - - const std::vector& short_key_columns() const { return _short_key_columns; } - - bool empty() const { return _empty; } - - bool zero_num_rows() const { - // previous version may has non-sense file in disk. - // to be compatible, it should be handled. - return _index.zero_num_rows(); - } - - void set_empty(bool empty) { _empty = empty; } - - // return count of entries in MemIndex - uint64_t num_index_entries() const; - - Status get_row_block_position(const OLAPIndexOffset& pos, RowBlockPosition* rbp) const { - return _index.get_row_block_position(pos, rbp); - } - - const FileHeader* get_seg_pb(uint32_t seg_id) const { - return &(_seg_pb_map.at(seg_id)); - } - - bool get_null_supported(uint32_t seg_id) { return _index.get_null_supported(seg_id); } - - std::string construct_index_file_path(const std::string& snapshot_path, - int32_t segment_id) const; - std::string construct_index_file_path(int32_t segment_id) const; - std::string construct_data_file_path(const std::string& snapshot_path, - int32_t segment_id) const; - std::string construct_data_file_path(int32_t segment_id) const; - - // these two functions are for compatible, and will be deleted later - // so it is better not to use it. - std::string construct_old_index_file_path(const std::string& path_prefix, - int32_t segment_id) const; - std::string construct_old_data_file_path(const std::string& path_prefix, - int32_t segment_id) const; - - size_t current_num_rows_per_row_block() const; - - const TabletSchema& get_tablet_schema(); - - int get_num_key_columns(); - - // for agg, uniq and replace model get_num_zone_map_columns() = get_num_key_columns(), - // for duplicate mode get_num_zone_map_columns() == num_columns - int get_num_zone_map_columns(); - - int get_num_short_key_columns(); - - size_t get_num_rows_per_row_block(); - - std::string rowset_path_prefix(); - - int64_t get_tablet_id(); - - const RowsetId& rowset_id() { return _rowset_id; } - - Status convert_from_old_files(const std::string& snapshot_path, - std::vector* success_links); - - Status convert_to_old_files(const std::string& snapshot_path, - std::vector* success_links); - - Status remove_old_files(std::vector* links_to_remove); - - Status copy_files_to(const std::string& dir); - - Status link_segments_to_path(const std::string& dest_path, const RowsetId& rowset_id); - -private: - std::string _construct_file_name(int32_t segment_id, const std::string& suffix) const; - std::string _construct_file_name(const RowsetId& rowset_id, int32_t segment_id, - const std::string& suffix) const; - - std::string _construct_old_pending_file_path(const std::string& path_prefix, int32_t segment_id, - const std::string& suffix) const; - - std::string _construct_old_file_path(const std::string& path_prefix, int32_t segment_id, - const std::string& suffix) const; - - std::string _construct_err_sg_file_path(const std::string& path_prefix, int32_t segment_id, - const std::string& suffix) const; - - std::string _construct_err_sg_index_file_path(const std::string& path_prefix, - int32_t segment_id) const; - - std::string _construct_err_sg_data_file_path(const std::string& path_prefix, - int32_t segment_id) const; - -private: - int64_t _tablet_id; - RowsetId _rowset_id; - const TabletSchema* _schema; - std::string _rowset_path_prefix; // path of rowset - Version _version; // version of associated data file - bool _delete_flag; - int32_t _segment_group_id; // segment group id of segment group - PUniqueId _load_id; // load id for segment group - int32_t _num_segments; // number of segments in this segment group - bool _index_loaded; // whether the segment group has been read - std::atomic _ref_count; // reference count - MemIndex _index; - bool _is_pending; - TPartitionId _partition_id; - TTransactionId _txn_id; - - // short key对应的column information - std::vector _short_key_columns; - // short key对应的总长度 - size_t _short_key_length; - size_t _new_short_key_length; - // 当前写入的short_key的buf - char* _short_key_buf; - // 当前写入的segment的checksum - uint32_t _checksum; - // 当前写入时用作索引项的RowCursor - RowCursor _current_index_row; - - // 以下是写入流程时需要的一些中间状态 - // 当前写入文件的FileHandler - FileHandler _current_file_handler; - bool _new_segment_created; - // 当前写入的FileHeader - FileHeader _file_header; - - bool _empty; - - // Lock held while loading the index. - mutable std::mutex _index_load_lock; - size_t _current_num_rows_per_row_block; - - std::vector> _zone_maps; - std::unordered_map> _seg_pb_map; -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_SEGMENT_GROUP_H diff --git a/be/src/olap/rowset/segment_reader.cpp b/be/src/olap/rowset/segment_reader.cpp deleted file mode 100644 index 1883e4bd01..0000000000 --- a/be/src/olap/rowset/segment_reader.cpp +++ /dev/null @@ -1,885 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/segment_reader.h" - -#include - -#include - -#include "olap/file_stream.h" -#include "olap/in_stream.h" -#include "olap/olap_cond.h" -#include "olap/out_stream.h" -#include "olap/row_block.h" -#include "olap/rowset/segment_group.h" - -namespace doris { - -static const uint32_t MIN_FILTER_BLOCK_NUM = 10; - -SegmentReader::SegmentReader(const std::string file, SegmentGroup* segment_group, - uint32_t segment_id, const std::vector& used_columns, - const std::set& load_bf_columns, - const Conditions* conditions, const DeleteHandler* delete_handler, - const DelCondSatisfied delete_status, Cache* lru_cache, - RuntimeState* runtime_state, OlapReaderStatistics* stats) - : _file_name(file), - _segment_group(segment_group), - _segment_id(segment_id), - _used_columns(used_columns), - _load_bf_columns(load_bf_columns), - _conditions(conditions), - _delete_handler(delete_handler), - _delete_status(delete_status), - _eof(false), - _end_block(-1), - // Make sure that the first call to _move_to_next_row will execute seek_to_block - _block_count(0), - _num_rows_in_block(0), - _null_supported(false), - _mmap_buffer(nullptr), - _include_blocks(nullptr), - _is_using_mmap(false), - _is_data_loaded(false), - _buffer_size(0), - _mem_pool(new MemPool("SegmentReader:" + file)), - _shared_buffer(nullptr), - _lru_cache(lru_cache), - _runtime_state(runtime_state), - _stats(stats) {} - -SegmentReader::~SegmentReader() { - SAFE_DELETE(_shared_buffer); - SAFE_DELETE_ARRAY(_include_blocks); - - for (auto& index_it : _indices) { - SAFE_DELETE(index_it.second); - } - - for (auto& bf_it : _bloom_filters) { - SAFE_DELETE(bf_it.second); - } - - for (auto handle : _cache_handle) { - if (handle != nullptr) { - _lru_cache->release(handle); - } - } - - _lru_cache = nullptr; - _file_handler.close(); - - for (auto& it : _streams) { - delete it.second; - } - - for (auto reader : _column_readers) { - delete reader; - } - - if (_is_using_mmap) { - SAFE_DELETE(_mmap_buffer); - } -} - -Status SegmentReader::_check_file_version() { - if (_header_message().magic_string().compare("COLUMN DATA") != 0) { - LOG(WARNING) << "not valid column data file, [magic_string = " - << _header_message().magic_string() << "]"; - return Status::OLAPInternalError(OLAP_ERR_FILE_FORMAT_ERROR); - } - - if (_header_message().version() > CURRENT_COLUMN_DATA_VERSION) { - LOG(WARNING) << "this file may generated by olap/ngine of higher version. " - "reading it would cause some unexpected error, [found version = " - << _header_message().version() << "]"; - } - - return Status::OK(); -} - -Status SegmentReader::_load_segment_file() { - Status res = Status::OK(); - - res = _file_handler.open_with_cache(_file_name, O_RDONLY); - if (!res.ok()) { - LOG(WARNING) << "fail to open segment file. [file='" << _file_name << "']"; - return res; - } - - //VLOG_NOTICE << "seg file : " << _file_name; - // In file_header.unserialize(), it validates file length, signature, checksum of protobuf. - _file_header = _segment_group->get_seg_pb(_segment_id); - _null_supported = _segment_group->get_null_supported(_segment_id); - _header_length = _file_header->size(); - - res = _check_file_version(); - if (!res.ok()) { - LOG(WARNING) << "file header corrupted or generated by higher version olap/ngine."; - return res; - } - - // If mmap is needed, then map - if (_is_using_mmap) { - _mmap_buffer = StorageByteBuffer::mmap(&_file_handler, 0, PROT_READ, MAP_PRIVATE); - - if (nullptr == _mmap_buffer) { - LOG(WARNING) << "fail to call mmap, using default mode"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - } - - return Status::OK(); -} - -Status SegmentReader::_set_decompressor() { - switch (_header_message().compress_kind()) { - case COMPRESS_NONE: { - _decompressor = nullptr; - break; - } -#ifdef DORIS_WITH_LZO - case COMPRESS_LZO: { - _decompressor = lzo_decompress; - break; - } -#endif - case COMPRESS_LZ4: { - _decompressor = lz4_decompress; - break; - } - default: { - LOG(WARNING) << "unknown decompressor"; - return Status::OLAPInternalError(OLAP_ERR_PARSE_PROTOBUF_ERROR); - } - } - return Status::OK(); -} - -Status SegmentReader::_set_segment_info() { - _num_rows_in_block = _header_message().num_rows_per_block(); - if (_num_rows_in_block == 0) { - _num_rows_in_block = _segment_group->get_num_rows_per_row_block(); - } - - _set_column_map(); - Status res = _set_decompressor(); - if (!res.ok()) { - LOG(WARNING) << "fail to get decompressor."; - return res; - } - return Status::OK(); -} - -Status SegmentReader::init(bool is_using_cache) { - SCOPED_RAW_TIMER(&_stats->index_load_ns); - - Status res = Status::OK(); - res = _load_segment_file(); - if (!res.ok()) { - LOG(WARNING) << "fail to load segment file. "; - return res; - } - // File header - res = _set_segment_info(); - if (!res.ok()) { - LOG(WARNING) << "fail to set segment info. "; - return res; - } - - _shared_buffer = - StorageByteBuffer::create(_header_message().stream_buffer_size() + sizeof(StreamHead)); - if (_shared_buffer == nullptr) { - LOG(WARNING) << "fail to create shared buffer. [size=" << sizeof(StorageByteBuffer) << "]"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - res = _pick_columns(); - if (!res.ok()) { - LOG(WARNING) << "fail to pick columns"; - return res; - } - - res = _load_index(is_using_cache); - if (!res.ok()) { - LOG(WARNING) << "fail to load index stream"; - return res; - } - - return Status::OK(); -} - -Status SegmentReader::seek_to_block(uint32_t first_block, uint32_t last_block, bool without_filter, - uint32_t* next_block_id, bool* eof) { - Status res = Status::OK(); - - if (!_is_data_loaded) { - _reset_readers(); - res = _read_all_data_streams(&_buffer_size); - if (!res.ok()) { - LOG(WARNING) << "fail to read data stream"; - return res; - } - - Status res = _create_reader(&_buffer_size); - if (!res.ok()) { - LOG(WARNING) << "fail to create reader"; - return res; - } - - _is_data_loaded = true; - } - - // If seek to block position, all stat will reset to initial - _eof = false; - _end_block = last_block >= _block_count ? _block_count - 1 : last_block; - _without_filter = without_filter; - delete[] _include_blocks; - _include_blocks = nullptr; - if (!_without_filter) { - /* - * row batch may be not empty before next read, - * should be clear here, otherwise dirty records - * will be read. - */ - _remain_block = last_block - first_block + 1; - res = _pick_row_groups(first_block, last_block); - if (!res.ok()) { - LOG(WARNING) << "fail to pick row groups"; - return res; - } - } - _seek_to_block(first_block, without_filter); - *next_block_id = _next_block_id; - *eof = _eof; - - // Must seek block when starts a ScanKey. - // In Doris, one block has 1024 rows. - // 1. If the previous ScanKey scan rows multiple blocks, - // and also the final block has 1024 rows just right. - // 2. The current ScanKey scan rows with number less than one block. - // Under the two conditions, if not seek block, the position - // of prefix shortkey columns is wrong. - _need_to_seek_block = true; - - return Status::OK(); -} - -Status SegmentReader::get_block(VectorizedRowBatch* batch, uint32_t* next_block_id, bool* eof) { - if (_eof) { - *eof = true; - return Status::OK(); - } - - // lazy seek - _seek_to_block_directly(_next_block_id, batch->columns()); - - int64_t num_rows_load = batch->limit(); - if (OLAP_UNLIKELY(_current_block_id == _block_count - 1)) { - int64_t num_rows_left = - _header_message().number_of_rows() - _num_rows_in_block * _current_block_id; - num_rows_load = std::min(num_rows_load, num_rows_left); - } - - auto res = _load_to_vectorized_row_batch(batch, num_rows_load); - if (!res.ok()) { - LOG(WARNING) << "fail to load block to vectorized_row_batch. res:" << res; - return res; - } - - _seek_to_block(_next_block_id + 1, _without_filter); - - *next_block_id = _next_block_id; - *eof = _eof; - return Status::OK(); -} - -void SegmentReader::_set_column_map() { - _encodings_map.clear(); - _tablet_id_to_unique_id_map.clear(); - _unique_id_to_tablet_id_map.clear(); - _unique_id_to_segment_id_map.clear(); - - for (ColumnId table_column_id : _used_columns) { - ColumnId unique_column_id = tablet_schema().column(table_column_id).unique_id(); - _tablet_id_to_unique_id_map[table_column_id] = unique_column_id; - _unique_id_to_tablet_id_map[unique_column_id] = table_column_id; - } - - for (ColumnId table_column_id : _load_bf_columns) { - ColumnId unique_column_id = tablet_schema().column(table_column_id).unique_id(); - _tablet_id_to_unique_id_map[table_column_id] = unique_column_id; - _unique_id_to_tablet_id_map[unique_column_id] = table_column_id; - } - - size_t segment_column_size = _header_message().column_size(); - for (ColumnId segment_column_id = 0; segment_column_id < segment_column_size; - ++segment_column_id) { - // If you can find it, create a mapping table - ColumnId unique_column_id = _header_message().column(segment_column_id).unique_id(); - if (_unique_id_to_tablet_id_map.find(unique_column_id) != - _unique_id_to_tablet_id_map.end()) { - _unique_id_to_segment_id_map[unique_column_id] = segment_column_id; - // The encoding should be in the same order as the segment schema. - _encodings_map[unique_column_id] = _header_message().column_encoding(segment_column_id); - } - } -} - -Status SegmentReader::_pick_columns() { - for (uint32_t i : _used_columns) { - ColumnId unique_column_id = _tablet_id_to_unique_id_map[i]; - _include_columns.insert(unique_column_id); - } - - for (uint32_t i : _load_bf_columns) { - ColumnId unique_column_id = _tablet_id_to_unique_id_map[i]; - _include_bf_columns.insert(unique_column_id); - } - - return Status::OK(); -} - -Status SegmentReader::_pick_delete_row_groups(uint32_t first_block, uint32_t last_block) { - VLOG_TRACE << "pick for " << first_block << " to " << last_block << " for delete_condition"; - - if (_delete_handler->empty()) { - return Status::OK(); - } - - if (DEL_NOT_SATISFIED == _delete_status) { - VLOG_TRACE << "the segment not satisfy the delete_conditions"; - return Status::OK(); - } - - for (auto& delete_condition : _delete_handler->get_delete_conditions()) { - if (delete_condition.filter_version <= _segment_group->version().first) { - continue; - } - - for (int64_t j = first_block; j <= last_block; ++j) { - if (DEL_SATISFIED == _include_blocks[j]) { - //if state is DEL_SATISFIED, continue - continue; - } - - bool del_partial_satisfied = false; - bool del_not_satisfied = false; - for (auto& i : delete_condition.del_cond->columns()) { - ColumnId table_column_id = i.first; - ColumnId unique_column_id = _tablet_id_to_unique_id_map[table_column_id]; - if (0 == _unique_id_to_segment_id_map.count(unique_column_id)) { - continue; - } - StreamIndexReader* index_reader = _indices[unique_column_id]; - int del_ret = i.second->del_eval(index_reader->entry(j).column_statistic().pair()); - if (DEL_SATISFIED == del_ret) { - continue; - } else if (DEL_PARTIAL_SATISFIED == del_ret) { - del_partial_satisfied = true; - } else { - del_not_satisfied = true; - break; - } - } - - if (true == del_not_satisfied || 0 == delete_condition.del_cond->columns().size()) { - //if state is DEL_PARTIAL_SATISFIED last_time, cannot be set as DEL_NOT_SATISFIED - //it is special for delete condition - if (DEL_PARTIAL_SATISFIED == _include_blocks[j]) { - continue; - } else { - _include_blocks[j] = DEL_NOT_SATISFIED; - } - } else if (true == del_partial_satisfied) { - _include_blocks[j] = DEL_PARTIAL_SATISFIED; - VLOG_TRACE << "filter block partially: " << j; - } else { - _include_blocks[j] = DEL_SATISFIED; - --_remain_block; - VLOG_TRACE << "filter block: " << j; - if (j < _block_count - 1) { - _stats->rows_del_filtered += _num_rows_in_block; - } else { - _stats->rows_del_filtered += - _header_message().number_of_rows() - j * _num_rows_in_block; - } - } - } - } - - return Status::OK(); -} - -Status SegmentReader::_init_include_blocks(uint32_t first_block, uint32_t last_block) { - if (nullptr == _include_blocks) { - _include_blocks = new (std::nothrow) uint8_t[_block_count]; - if (nullptr == _include_blocks) { - LOG(WARNING) << "fail to malloc include block array"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - } - - memset(_include_blocks, 0, _block_count); - memset(_include_blocks + first_block, 1, _remain_block); - - return Status::OK(); -} - -Status SegmentReader::_pick_row_groups(uint32_t first_block, uint32_t last_block) { - VLOG_TRACE << "pick from " << first_block << " to " << last_block; - - if (first_block > last_block) { - LOG(WARNING) << "invalid block offset. [first_block=" << first_block - << " last_block=" << last_block << "]"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - Status res = _init_include_blocks(first_block, last_block); - if (!res.ok()) { - return res; - } - - _pick_delete_row_groups(first_block, last_block); - - if (nullptr == _conditions || _conditions->columns().size() == 0) { - return Status::OK(); - } - - OlapStopWatch timer; - timer.reset(); - - for (auto& i : _conditions->columns()) { - FieldAggregationMethod aggregation = _get_aggregation_by_index(i.first); - bool is_continue = (aggregation == OLAP_FIELD_AGGREGATION_NONE); - if (!is_continue) { - continue; - } - - ColumnId table_column_id = i.first; - ColumnId unique_column_id = _tablet_id_to_unique_id_map[table_column_id]; - if (0 == _unique_id_to_segment_id_map.count(unique_column_id)) { - continue; - } - StreamIndexReader* index_reader = _indices[unique_column_id]; - for (int64_t j = first_block; j <= last_block; ++j) { - if (_include_blocks[j] == DEL_SATISFIED) { - continue; - } - - if (!i.second->eval(index_reader->entry(j).column_statistic().pair())) { - _include_blocks[j] = DEL_SATISFIED; - --_remain_block; - - if (j < _block_count - 1) { - _stats->rows_stats_filtered += _num_rows_in_block; - } else { - _stats->rows_stats_filtered += - _header_message().number_of_rows() - j * _num_rows_in_block; - } - } - } - } - - if (_remain_block < MIN_FILTER_BLOCK_NUM) { - VLOG_TRACE << "bloom filter is ignored for too few block remained. " - << "remain_block=" << _remain_block - << ", const_time=" << timer.get_elapse_time_us(); - return Status::OK(); - } - - for (uint32_t i : _load_bf_columns) { - FieldAggregationMethod aggregation = _get_aggregation_by_index(i); - bool is_continue = (aggregation == OLAP_FIELD_AGGREGATION_NONE); - if (!is_continue) { - continue; - } - - ColumnId table_column_id = i; - ColumnId unique_column_id = _tablet_id_to_unique_id_map[table_column_id]; - if (0 == _unique_id_to_segment_id_map.count(unique_column_id)) { - continue; - } - BloomFilterIndexReader* bf_reader = _bloom_filters[unique_column_id]; - for (int64_t j = first_block; j <= last_block; ++j) { - if (_include_blocks[j] == DEL_SATISFIED) { - continue; - } - - if (!_conditions->columns().at(i)->eval(bf_reader->entry(j))) { - _include_blocks[j] = DEL_SATISFIED; - --_remain_block; - if (j < _block_count - 1) { - _stats->rows_stats_filtered += _num_rows_in_block; - } else { - _stats->rows_stats_filtered += - _header_message().number_of_rows() - j * _num_rows_in_block; - } - } - } - } - - VLOG_TRACE << "pick row groups finished. remain_block=" << _remain_block - << ", const_time=" << timer.get_elapse_time_us(); - return Status::OK(); -} - -CacheKey SegmentReader::_construct_index_stream_key(char* buf, size_t len, - const std::string& file_name, - ColumnId unique_column_id, - StreamInfoMessage::Kind kind) { - char* current = buf; - size_t remain_len = len; - OLAP_CACHE_STRING_TO_BUF(current, file_name, remain_len); - OLAP_CACHE_NUMERIC_TO_BUF(current, unique_column_id, remain_len); - OLAP_CACHE_NUMERIC_TO_BUF(current, kind, remain_len); - - return CacheKey(buf, len - remain_len); -} - -void SegmentReader::_delete_cached_index_stream(const CacheKey& key, void* value) { - char* buffer = reinterpret_cast(value); - SAFE_DELETE_ARRAY(buffer); -} - -Status SegmentReader::_load_index(bool is_using_cache) { - Status res = Status::OK(); - - int32_t handle_num = _get_included_row_index_stream_num(); - _cache_handle.resize(handle_num, nullptr); - - ReadOnlyFileStream stream(&_file_handler, &_shared_buffer, _decompressor, - _header_message().stream_buffer_size(), _stats); - res = stream.init(); - if (!res.ok()) { - LOG(WARNING) << "fail to init stream. res = " << res; - return res; - } - - _indices.clear(); - _bloom_filters.clear(); - uint64_t stream_length = 0; - int32_t cache_handle_index = 0; - uint64_t stream_offset = _header_length; - int64_t expected_blocks = - static_cast(ceil(static_cast(_header_message().number_of_rows()) / - _header_message().num_rows_per_block())); - for (int64_t stream_index = 0; stream_index < _header_message().stream_info_size(); - ++stream_index, stream_offset += stream_length) { - // Find the required index, although some indexes do not need to be read - // Take, but in order to get the offset, it is still necessary to calculate it again - // Otherwise, the correct streamoffset cannot be obtained - const StreamInfoMessage& message = _header_message().stream_info(stream_index); - stream_length = message.length(); - ColumnId unique_column_id = message.column_unique_id(); - if (0 == _unique_id_to_segment_id_map.count(unique_column_id)) { - continue; - } - - if ((_is_column_included(unique_column_id) && - message.kind() == StreamInfoMessage::ROW_INDEX) || - (_is_bf_column_included(unique_column_id) && - message.kind() == StreamInfoMessage::BLOOM_FILTER)) { - } else { - continue; - } - - ColumnId table_column_id = _unique_id_to_tablet_id_map[unique_column_id]; - FieldType type = _get_field_type_by_index(table_column_id); - - char* stream_buffer = nullptr; - char key_buf[OLAP_LRU_CACHE_MAX_KEY_LENGTH]; - CacheKey key = - _construct_index_stream_key(key_buf, sizeof(key_buf), _file_handler.file_name(), - unique_column_id, message.kind()); - _cache_handle[cache_handle_index] = _lru_cache->lookup(key); - - if (nullptr != _cache_handle[cache_handle_index]) { - // 1. If you are in lru, take out the buffer and use it to initialize the index reader - is_using_cache = true; - stream_buffer = - reinterpret_cast(_lru_cache->value(_cache_handle[cache_handle_index])); - } else { - // 2. If it is not in lru, you need to create an index stream. - stream_buffer = new (std::nothrow) char[stream_length]; - if (nullptr == stream_buffer) { - LOG(WARNING) << "fail to malloc index stream. " - "[column_unique_id = " - << unique_column_id << ", offset = " << stream_offset << "]"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - size_t read_length = stream_length; - stream.reset(stream_offset, stream_length); - res = stream.read_all(stream_buffer, &read_length); - if (!res.ok()) { - LOG(WARNING) << "read index fail"; - return Status::OLAPInternalError(OLAP_ERR_FILE_FORMAT_ERROR); - } - - if (is_using_cache) { - // Put the read index into lru. - _cache_handle[cache_handle_index] = _lru_cache->insert( - key, stream_buffer, stream_length, &_delete_cached_index_stream); - if (nullptr == _cache_handle[cache_handle_index]) { - // It may be that malloc in cache insert failed, first return success - LOG(FATAL) << "fail to insert lru cache."; - } - } - } - cache_handle_index++; - - if (message.kind() == StreamInfoMessage::ROW_INDEX) { - StreamIndexReader* index_message = new (std::nothrow) StreamIndexReader; - if (index_message == nullptr) { - LOG(WARNING) << "fail to malloc memory. [size=" << sizeof(StreamIndexReader) << "]"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - res = index_message->init(stream_buffer, stream_length, type, is_using_cache, - _null_supported); - if (!res.ok()) { - LOG(WARNING) << "init index from cache fail"; - return res; - } - - _indices[unique_column_id] = index_message; - - // The number of entries for each index should be the same, that is, the number of blocks - _block_count = index_message->entry_count(); - } else { - BloomFilterIndexReader* bf_message = new (std::nothrow) BloomFilterIndexReader; - if (bf_message == nullptr) { - LOG(WARNING) << "fail to malloc memory. [size=" << sizeof(BloomFilterIndexReader) - << "]"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - res = bf_message->init(stream_buffer, stream_length, is_using_cache, - _header_message().bf_hash_function_num(), - _header_message().bf_bit_num()); - if (!res.ok()) { - LOG(WARNING) << "fail to init bloom filter reader. res = " << res; - return res; - } - - _bloom_filters[unique_column_id] = bf_message; - - // The number of entries for each index should be the same, that is, the number of blocks - _block_count = bf_message->entry_count(); - } - - if (_block_count != expected_blocks) { - LOG(WARNING) << "something wrong while reading index, expected=" << expected_blocks - << ", actual=" << _block_count; - LOG(WARNING) << "_header_message().number_of_rows()=" - << _header_message().number_of_rows() - << ", _header_message().num_rows_per_block()=" - << _header_message().num_rows_per_block() - << ", tablet_id=" << _segment_group->get_tablet_id() << ", version='" - << _segment_group->version().first << "-" - << _segment_group->version().second << "'"; - return Status::OLAPInternalError(OLAP_ERR_FILE_FORMAT_ERROR); - } - } - - VLOG_TRACE << "found index entry count: " << _block_count; - return Status::OK(); -} - -Status SegmentReader::_read_all_data_streams(size_t* buffer_size) { - int64_t stream_offset = _header_length; - uint64_t stream_length = 0; - - // Each stream is one piece - for (int64_t stream_index = 0; stream_index < _header_message().stream_info_size(); - ++stream_index, stream_offset += stream_length) { - const StreamInfoMessage& message = _header_message().stream_info(stream_index); - stream_length = message.length(); - ColumnId unique_column_id = message.column_unique_id(); - - if (_unique_id_to_segment_id_map.count(unique_column_id) == 0) { - continue; - } - - if (_include_columns.find(unique_column_id) == _include_columns.end() && - _include_bf_columns.find(unique_column_id) == _include_bf_columns.end()) { - continue; - } - - if (message.kind() == StreamInfoMessage::ROW_INDEX || - message.kind() == StreamInfoMessage::BLOOM_FILTER) { - continue; - } - - StreamName name(unique_column_id, message.kind()); - std::unique_ptr stream(new (std::nothrow) ReadOnlyFileStream( - &_file_handler, &_shared_buffer, stream_offset, stream_length, _decompressor, - _header_message().stream_buffer_size(), _stats)); - if (stream == nullptr) { - LOG(WARNING) << "fail to create stream"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - Status res = stream->init(); - if (!res.ok()) { - LOG(WARNING) << "fail to init stream"; - return res; - } - - *buffer_size += stream->get_buffer_size(); - _streams[name] = stream.release(); - } - - return Status::OK(); -} - -Status SegmentReader::_create_reader(size_t* buffer_size) { - _column_readers.resize(_segment_group->get_tablet_schema().num_columns(), nullptr); - _column_indices.resize(_segment_group->get_tablet_schema().num_columns(), nullptr); - for (auto table_column_id : _used_columns) { - ColumnId unique_column_id = _tablet_id_to_unique_id_map[table_column_id]; - // Currently, there will be no inconsistencies in the schema of the table and the segment. - std::unique_ptr reader(ColumnReader::create( - table_column_id, _segment_group->get_tablet_schema(), _unique_id_to_tablet_id_map, - _unique_id_to_segment_id_map, _encodings_map)); - if (reader == nullptr) { - LOG(WARNING) << "fail to create reader"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - auto res = reader->init(&_streams, _num_rows_in_block, _mem_pool.get(), _stats); - if (!res.ok()) { - LOG(WARNING) << "fail to init reader"; - return res; - } - - *buffer_size += reader->get_buffer_size(); - _column_readers[table_column_id] = reader.release(); - if (_indices.count(unique_column_id) != 0) { - _column_indices[table_column_id] = _indices[unique_column_id]; - } - } - - return Status::OK(); -} - -Status SegmentReader::_seek_to_block_directly(int64_t block_id, const std::vector& cids) { - if (!_need_to_seek_block && block_id == _current_block_id) { - // no need to execute seek - return Status::OK(); - } - SCOPED_RAW_TIMER(&_stats->block_seek_ns); - for (auto cid : cids) { - // If column is added through schema change, column index may not exist because of - // linked schema change. So we need to ignore this column's seek - if (_column_indices[cid] == nullptr) { - continue; - } - - Status res = Status::OK(); - PositionProvider position(&_column_indices[cid]->entry(block_id)); - if (!(res = _column_readers[cid]->seek(&position))) { - if (OLAP_ERR_COLUMN_STREAM_EOF == res.precise_code()) { - VLOG_TRACE << "Stream EOF. tablet_id=" << _segment_group->get_tablet_id() - << ", column_id=" << _column_readers[cid]->column_unique_id() - << ", block_id=" << block_id; - return Status::OLAPInternalError(OLAP_ERR_DATA_EOF); - } else { - LOG(WARNING) << "fail to seek to block. " - "[tablet_id=" - << _segment_group->get_tablet_id() - << " column_id=" << _column_readers[cid]->column_unique_id() - << " block_id=" << block_id << "]"; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_SEEK_ERROR); - } - } - } - _current_block_id = block_id; - _need_to_seek_block = false; - return Status::OK(); -} - -Status SegmentReader::_reset_readers() { - VLOG_TRACE << _streams.size() << " stream in total."; - - for (std::map::iterator it = _streams.begin(); - it != _streams.end(); ++it) { - delete it->second; - } - - _streams.clear(); - - for (std::vector::iterator it = _column_readers.begin(); - it != _column_readers.end(); ++it) { - if ((*it) == nullptr) { - continue; - } - delete (*it); - } - - _column_readers.clear(); - _eof = false; - return Status::OK(); -} - -void SegmentReader::_seek_to_block(int64_t block_id, bool without_filter) { - if (_include_blocks != nullptr && !without_filter) { - while (block_id <= _end_block && _include_blocks[block_id] == DEL_SATISFIED) { - block_id++; - } - } - if (block_id > _end_block) { - _eof = true; - } - _next_block_id = block_id; -} - -Status SegmentReader::_load_to_vectorized_row_batch(VectorizedRowBatch* batch, size_t size) { - SCOPED_RAW_TIMER(&_stats->block_load_ns); - MemPool* mem_pool = batch->mem_pool(); - for (auto cid : batch->columns()) { - auto reader = _column_readers[cid]; - auto res = reader->next_vector(batch->column(cid), size, mem_pool); - if (!res.ok()) { - LOG(WARNING) << "fail to read next, res=" << res - << ", column=" << reader->column_unique_id() << ", size=" << size; - return res; - } - } - batch->set_size(size); - if (_include_blocks != nullptr) { - batch->set_block_status(_include_blocks[_current_block_id]); - } else { - batch->set_block_status(DEL_PARTIAL_SATISFIED); - } - // If size is just _num_rows_in_block, after read, we point to next block start, - // so we increase _current_block_id - if (size == _num_rows_in_block) { - _current_block_id++; - } else { - _need_to_seek_block = true; - } - - _stats->blocks_load++; - _stats->raw_rows_read += size; - - return Status::OK(); -} - -} // namespace doris diff --git a/be/src/olap/rowset/segment_reader.h b/be/src/olap/rowset/segment_reader.h deleted file mode 100644 index 2dcbe5c2a6..0000000000 --- a/be/src/olap/rowset/segment_reader.h +++ /dev/null @@ -1,334 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_SEGMENT_READER_H -#define DORIS_BE_SRC_OLAP_ROWSET_SEGMENT_READER_H - -#include -#include - -#include -#include -#include - -#include "olap/bloom_filter_reader.h" -#include "olap/column_predicate.h" -#include "olap/compress.h" -#include "olap/delete_handler.h" -#include "olap/file_helper.h" -#include "olap/file_stream.h" -#include "olap/in_stream.h" -#include "olap/lru_cache.h" -#include "olap/olap_cond.h" -#include "olap/olap_define.h" -#include "olap/row_cursor.h" -#include "olap/rowset/column_reader.h" -#include "olap/rowset/segment_group.h" -#include "olap/stream_index_reader.h" -#include "runtime/mem_pool.h" -#include "runtime/runtime_state.h" - -namespace doris { - -// SegmentReader is used to read a Segment file -class SegmentReader { -public: - SegmentReader(const std::string file, SegmentGroup* segment_group, uint32_t segment_id, - const std::vector& used_columns, - const std::set& load_bf_columns, const Conditions* conditions, - const DeleteHandler* delete_handler, const DelCondSatisfied delete_status, - Cache* lru_cache, RuntimeState* runtime_state, OlapReaderStatistics* stats); - - ~SegmentReader(); - - // Initialize segmentreader: - // 1. Deserialize the pb header to obtain the necessary information; - // 2. Check the file version - // 3. Get the decompressor - // @return [description] - Status init(bool is_using_cache); - - // Must called before seek to block. - // TODO(zc) - Status prepare(const std::vector& columns); - - // Specify the first block and the last block to read, and initialize the column reader - // seek_to_block supports being called multiple times - // Inputs: - // first_block: the first block that needs to be read - // last_block: The last block that needs to be read, if last_block is greater than the largest block, - // read all blocks - // 1. Filter the statistics in segment_group according to conditions and determine the list of blocks that need to be read - // 2. Read blocks, construct InStream - // 3. Create and initialize Readers - // Outputs: - // next_block_id: - // block with next_block_id would read if get_block called again. - // this field is used to set batch's limit when client found logical end is reach - Status seek_to_block(uint32_t first_block, uint32_t last_block, bool without_filter, - uint32_t* next_block_id, bool* eof); - - // get vector batch from this segment. - // next_block_id: - // block with next_block_id would read if get_block called again. - // this field is used to set batch's limit when client found logical end is reach - // ATTN: If you change batch to contain more columns, you must call seek_to_block again. - Status get_block(VectorizedRowBatch* batch, uint32_t* next_block_id, bool* eof); - - bool eof() const { return _eof; } - - // Returns the number of blocks in the current segment - uint32_t block_count() const { return _block_count; } - - // Returns the number of rows in each block in the current segment - uint32_t num_rows_in_block() { return _num_rows_in_block; } - - bool is_using_mmap() { return _is_using_mmap; } - - // Only allowed to be selected before initialization, and cannot be changed afterwards - // There is no need for dynamic switching at the moment - void set_is_using_mmap(bool is_using_mmap) { _is_using_mmap = is_using_mmap; } - -private: - typedef std::vector::iterator ColumnIdIterator; - - // Used to indicate a range of data to be read - struct DiskRange { - int64_t offset; - int64_t end; - DiskRange() : offset(0), end(0) {} - }; - - struct VectorizedPositionInfo { - uint32_t column_id; - uint32_t column_position; - uint32_t offset_position; - }; - - static CacheKey _construct_index_stream_key(char* buf, size_t len, const std::string& file_name, - ColumnId unique_column_id, - StreamInfoMessage::Kind kind); - - static void _delete_cached_index_stream(const CacheKey& key, void* value); - - // Determine whether the current column needs to be read - // When _include_columns is empty, return true directly - bool _is_column_included(ColumnId column_unique_id) { - return _include_columns.count(column_unique_id) != 0; - } - - bool _is_bf_column_included(ColumnId column_unique_id) { - return _include_bf_columns.count(column_unique_id) != 0; - } - - // Load files and necessary file information - Status _load_segment_file(); - - // Set the encoding map and use it when creating columns - void _set_column_map(); - - //Get the current file compression format from the header and generate a decompressor, which can be called by _decompressor - // @return Return OLAP_SUCCESS on behalf of the version check passed - Status _set_decompressor(); - - // Set segment related information, decompressor, column, encoding, etc. - Status _set_segment_info(); - - // Check the listed file version - // @return Return OLAP_SUCCESS on behalf of the version check passed - Status _check_file_version(); - - // Select the column to be read - Status _pick_columns(); - - // Select the range to be read according to the conditions, and use the conditions to mark the appropriate block between the first block and the last block - // NOTE. Note that the range is [first_block, last_block], closed interval - // @param first_block : Starting block number - // @param last_block : End block number - // @return - Status _pick_row_groups(uint32_t first_block, uint32_t last_block); - Status _pick_delete_row_groups(uint32_t first_block, uint32_t last_block); - - // Load the index, read the index of the required column into memory - Status _load_index(bool is_using_cache); - - // Read all the columns, the complete stream, (here just create the stream, because there is no mmap in the orc file, - // it means the actual data is read, but there is no actual read here, just circle the required range) - Status _read_all_data_streams(size_t* buffer_size); - - // Filter and read, (like _read_all_data_streams, there is no actual read data) - // Create reader - Status _create_reader(size_t* buffer_size); - - // we implement seek to block in two phase. first, we just only move _next_block_id - // to the position that we want goto; second, we seek the column streams to the - // position we going to read. - void _seek_to_block(int64_t block_id, bool without_filter); - - // seek to block id without check. only seek in cids's read stream. - // because some columns may not be read - Status _seek_to_block_directly(int64_t block_id, const std::vector& cids); - - // Jump to a row entry - Status _seek_to_row_entry(int64_t block_id); - - Status _reset_readers(); - - // Get the current table-level schema. - const TabletSchema& tablet_schema() { return _segment_group->get_tablet_schema(); } - - const ColumnDataHeaderMessage& _header_message() { return _file_header->message(); } - - Status _init_include_blocks(uint32_t first_block, uint32_t last_block); - - const int32_t _get_included_row_index_stream_num() { - int32_t included_row_index_stream_num = 0; - for (int32_t i = 0; i < _header_message().stream_info_size(); ++i) { - const StreamInfoMessage& message = _header_message().stream_info(i); - ColumnId unique_column_id = message.column_unique_id(); - if (0 == _unique_id_to_segment_id_map.count(unique_column_id)) { - continue; - } - - if ((_is_column_included(unique_column_id) && - message.kind() == StreamInfoMessage::ROW_INDEX) || - (_is_bf_column_included(unique_column_id) && - message.kind() == StreamInfoMessage::BLOOM_FILTER)) { - ++included_row_index_stream_num; - } - } - return included_row_index_stream_num; - } - - Status _load_to_vectorized_row_batch(VectorizedRowBatch* batch, size_t size); - - FieldAggregationMethod _get_aggregation_by_index(uint32_t index) { - const TabletSchema& tablet_schema = _segment_group->get_tablet_schema(); - if (index < tablet_schema.num_columns()) { - return tablet_schema.column(index).aggregation(); - } - - return OLAP_FIELD_AGGREGATION_UNKNOWN; - } - - FieldType _get_field_type_by_index(uint32_t index) { - const TabletSchema& tablet_schema = _segment_group->get_tablet_schema(); - if (index < tablet_schema.num_columns()) { - return tablet_schema.column(index).type(); - } - - return OLAP_FIELD_TYPE_NONE; - } - -private: - static const int32_t BYTE_STREAM_POSITIONS = 1; - static const int32_t RUN_LENGTH_BYTE_POSITIONS = BYTE_STREAM_POSITIONS + 1; - static const int32_t BITFIELD_POSITIONS = RUN_LENGTH_BYTE_POSITIONS + 1; - static const int32_t RUN_LENGTH_INT_POSITIONS = BYTE_STREAM_POSITIONS + 1; - // The meaning of this value is that 8 = the largest int type length, and an int reader can read up to 12 characters at a time (MAX SCOPE). - // Assuming no compression at all, then there will be so many bytes, 2 should be the control character? - // Then read at most so many, it must be enough to solve the next field. - static const int32_t WORST_UNCOMPRESSED_SLOP = 2 + 8 * 512; - static const uint32_t CURRENT_COLUMN_DATA_VERSION = 1; - - std::string _file_name; // File name - SegmentGroup* _segment_group; - uint32_t _segment_id; - // columns that can be used by client. when client seek to range's start or end, - // client may read more columns than normal read. - // For example: - // table1's schema is 'k1, k2, v1'. which k1, k2 is key column, v1 is value column. - // for query 'select sum(v1) from table1', client split all data to sub-range in logical, - // so, one sub-range need to seek to right position with k1 and k2; then only read v1. - // In this situation, _used_columns contains (k1, k2, v1) - std::vector _used_columns; - UniqueIdSet _load_bf_columns; - const Conditions* _conditions; // Column filter - doris::FileHandler _file_handler; // File handler - - const DeleteHandler* _delete_handler = nullptr; - DelCondSatisfied _delete_status; - - bool _eof; // EOF Sign - - // If this field is true, client must to call seek_to_block before - // calling get_block. - bool _need_to_seek_block = true; - - int64_t _end_block; // The end block read this time - int64_t _current_block_id = 0; // Block currently read - - // this is set by _seek_to_block, when get_block is called, first - // seek to this block_id, then read block. - int64_t _next_block_id = 0; - int64_t _block_count; // In each column, the number of index entries should be equal. - - uint64_t _num_rows_in_block; - bool _null_supported; - uint64_t _header_length; // Header(FixHeader+PB) size, need to offset when reading data - - std::vector _column_readers; // Actual data reader - std::vector _column_indices; // Save the index of the column - - UniqueIdSet _include_columns; // Used to determine whether the column is included - UniqueIdSet _include_bf_columns; - UniqueIdToColumnIdMap _tablet_id_to_unique_id_map; // The mapping from tablet id to unique id - UniqueIdToColumnIdMap _unique_id_to_tablet_id_map; // Mapping from unique id to tablet id - UniqueIdToColumnIdMap _unique_id_to_segment_id_map; // Mapping from unique id to segment id - - std::map _indices; - std::map _streams; // Need to read the stream - UniqueIdEncodingMap _encodings_map; // Save encoding - std::map _bloom_filters; - Decompressor _decompressor; // According to the compression format, set the decompressor - StorageByteBuffer* _mmap_buffer; - - /* - * _include_blocks is used for saving the state of block when encountering delete conditions, - * in this place the delete condition include the delete condition, the query filter condition - * and the bloom filter condition. - * DEL_SATISFIED is for block meet the delete condition, should be filtered. - * but it is not stored, it reflect in _include_blocks. - * DEL_NOT_SATISFIED is for block not meet the delete condition, should be held to read. - * DEL_PARTIAL_SATISFIED is for block can't be filtered by the delete condition in block level. - */ - uint8_t* _include_blocks; - uint32_t _remain_block; - bool _need_block_filter; // Used in combination with include blocks, if none of them are in, no longer read - bool _is_using_mmap; // When this flag is true, use mmap to read the file - bool _is_data_loaded; - size_t _buffer_size; - - std::vector _cache_handle; - const FileHeader* _file_header; - - std::unique_ptr _mem_pool; - - StorageByteBuffer* _shared_buffer; - Cache* _lru_cache; - RuntimeState* _runtime_state; // Used to count runtime information such as memory consumption - OlapReaderStatistics* _stats; - - // Set when seek_to_block is called, valid until next seek_to_block is called. - bool _without_filter = false; - - DISALLOW_COPY_AND_ASSIGN(SegmentReader); -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_SEGMENT_READER_H diff --git a/be/src/olap/rowset/segment_v2/bloom_filter.cpp b/be/src/olap/rowset/segment_v2/bloom_filter.cpp index 523f00260f..cbb5181ac5 100644 --- a/be/src/olap/rowset/segment_v2/bloom_filter.cpp +++ b/be/src/olap/rowset/segment_v2/bloom_filter.cpp @@ -21,7 +21,6 @@ #include "gen_cpp/segment_v2.pb.h" #include "gutil/strings/substitute.h" #include "olap/rowset/segment_v2/block_split_bloom_filter.h" -#include "olap/serialize.h" #include "olap/utils.h" namespace doris { @@ -36,6 +35,15 @@ Status BloomFilter::create(BloomFilterAlgorithmPB algorithm, std::unique_ptr::iterator it = _root_writers.begin(); it != _root_writers.end(); - ++it) { - SAFE_DELETE(*it); - } -} - -Status SegmentWriter::init(uint32_t write_mbytes_per_sec) { - Status res = Status::OK(); - // 创建factory - _stream_factory = new (std::nothrow) OutStreamFactory(_compress_kind, _stream_buffer_size); - - if (nullptr == _stream_factory) { - LOG(WARNING) << "fail to allocate out stream factory"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } - - // 创建writer - for (uint32_t i = 0; i < _segment_group->get_tablet_schema().num_columns(); i++) { - ColumnWriter* writer = ColumnWriter::create( - i, _segment_group->get_tablet_schema(), _stream_factory, - _segment_group->get_num_rows_per_row_block(), _bloom_filter_fpp); - - if (nullptr == writer) { - LOG(WARNING) << "fail to create writer"; - return Status::OLAPInternalError(OLAP_ERR_MALLOC_ERROR); - } else { - _root_writers.push_back(writer); - } - - res = writer->init(); - if (!res.ok()) { - LOG(WARNING) << "fail to initialize ColumnWriter. res = " << res; - return res; - } - } - - _write_mbytes_per_sec = write_mbytes_per_sec; - - return Status::OK(); -} - -Status SegmentWriter::write_batch(RowBlock* block, RowCursor* cursor, bool is_finalize) { - DCHECK(block->row_block_info().row_num == _segment_group->get_num_rows_per_row_block() || - is_finalize) - << "write block not empty, num_rows=" << block->row_block_info().row_num - << ", table_num_rows=" << _segment_group->get_num_rows_per_row_block(); - Status res = Status::OK(); - for (auto col_writer : _root_writers) { - res = col_writer->write_batch(block, cursor); - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write row. res = " << res; - return res; - } - res = col_writer->create_row_index_entry(); - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to create row index. res = " << res; - return res; - } - } - _row_count += block->row_block_info().row_num; - ++_block_count; - return res; -} - -uint64_t SegmentWriter::estimate_segment_size() { - uint64_t result = 0; - - for (std::map::const_iterator it = _stream_factory->streams().begin(); - it != _stream_factory->streams().end(); ++it) { - result += it->second->get_total_buffer_size(); - } - - for (std::vector::iterator it = _root_writers.begin(); it != _root_writers.end(); - ++it) { - result += (*it)->estimate_buffered_memory(); - } - - return result; -} - -Status SegmentWriter::_make_file_header(ColumnDataHeaderMessage* file_header) { - Status res = Status::OK(); - file_header->set_number_of_rows(_row_count); - file_header->set_compress_kind(_compress_kind); - file_header->set_stream_buffer_size(_stream_buffer_size); - // TODO. 之前没设置 - file_header->set_magic_string("COLUMN DATA"); - file_header->set_version(1); - file_header->set_num_rows_per_block(_segment_group->get_num_rows_per_row_block()); - - // check if has bloom filter columns - bool has_bf_column = false; - uint32_t bf_hash_function_num = 0; - uint32_t bf_bit_num = 0; - for (std::vector::iterator it = _root_writers.begin(); it != _root_writers.end(); - ++it) { - (*it)->get_bloom_filter_info(&has_bf_column, &bf_hash_function_num, &bf_bit_num); - if (has_bf_column) { - file_header->set_bf_hash_function_num(bf_hash_function_num); - file_header->set_bf_bit_num(bf_bit_num); - break; - } - } - - for (std::vector::iterator it = _root_writers.begin(); it != _root_writers.end(); - ++it) { - // ColumnWriter::finalize will set: - // * column_unique_id - // * column_type - // * column_encoding - // * zone_maps - res = (*it)->finalize(file_header); - - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to finalize row writer. res = " << res; - return res; - } - } - - uint64_t index_length = 0; - uint64_t data_length = 0; - - for (std::map::const_iterator it = _stream_factory->streams().begin(); - it != _stream_factory->streams().end(); ++it) { - OutStream* stream = it->second; - - // 如果这个流没有被终止，flush - if (!stream->is_suppressed()) { - if (!(res = stream->flush())) { - LOG(WARNING) << "fail to flush out stream. res = " << res; - return res; - } - } else { - //如果被suspend，目前也就是present流，不写入信息 - continue; - } - - StreamInfoMessage* stream_info = file_header->add_stream_info(); - stream_info->set_length(stream->get_stream_length()); - stream_info->set_column_unique_id(it->first.unique_column_id()); - stream_info->set_kind(it->first.kind()); - - if (it->first.kind() == StreamInfoMessage::ROW_INDEX || - it->first.kind() == StreamInfoMessage::BLOOM_FILTER) { - index_length += stream->get_stream_length(); - } else { - data_length += stream->get_stream_length(); - } - - VLOG_TRACE << "stream id=" << it->first.unique_column_id() << ", type=" << it->first.kind() - << ", length=" << stream->get_stream_length(); - } - - file_header->set_index_length(index_length); - file_header->set_data_length(data_length); - return res; -} - -// 之前所有的数据都缓存在内存里, 现在创建文件, 写入数据 -Status SegmentWriter::finalize(uint32_t* segment_file_size) { - Status res = Status::OK(); - FileHandler file_handle; - FileHeader file_header; - StorageEngine* engine = StorageEngine::instance(); - DataDir* data_dir = nullptr; - if (engine != nullptr) { - std::filesystem::path tablet_path = std::string_view(_segment_group->rowset_path_prefix()); - std::filesystem::path data_dir_path = - tablet_path.parent_path().parent_path().parent_path().parent_path(); - std::string data_dir_string = data_dir_path.string(); - data_dir = engine->get_store(data_dir_string); - if (LIKELY(data_dir != nullptr)) { - data_dir->add_pending_ids(ROWSET_ID_PREFIX + _segment_group->rowset_id().to_string()); - } else { - LOG(WARNING) << "data dir not found. [data_dir=" << data_dir_string << "]"; - return Status::OLAPInternalError(OLAP_ERR_CANNOT_CREATE_DIR); - } - } - if (!(res = file_handle.open_with_mode(_file_name, O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR))) { - LOG(WARNING) << "fail to open file. [file_name=" << _file_name << "]"; - return res; - } - - res = _make_file_header(file_header.mutable_message()); - if (!res.ok()) { - LOG(WARNING) << "fail to make file header. res = " << res; - return res; - } - - // check disk capacity - if (data_dir != nullptr && data_dir->reach_capacity_limit((int64_t)file_header.file_length())) { - return Status::OLAPInternalError(OLAP_ERR_DISK_REACH_CAPACITY_LIMIT); - } - - if (!(res = file_handle.open_with_mode(_file_name, O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR))) { - LOG(WARNING) << "fail to open file. [file_name=" << _file_name << "]"; - return res; - } - - res = file_header.prepare(&file_handle); - if (!res.ok()) { - LOG(WARNING) << "write file header error. [err=" << Errno::str() << "]"; - return res; - } - - // 跳过FileHeader - if (-1 == file_handle.seek(file_header.size(), SEEK_SET)) { - LOG(WARNING) << "lseek header file error. [err=" << Errno::str() << "]"; - return Status::OLAPInternalError(OLAP_ERR_IO_ERROR); - } - - uint32_t checksum = CRC32_INIT; - - // 写入数据 - for (std::map::const_iterator it = _stream_factory->streams().begin(); - it != _stream_factory->streams().end(); ++it) { - OutStream* stream = it->second; - - // 输出没有被掐掉的流 - if (!stream->is_suppressed()) { - checksum = stream->crc32(checksum); - VLOG_TRACE << "stream id=" << it->first.unique_column_id() - << ", type=" << it->first.kind(); - res = stream->write_to_file(&file_handle, _write_mbytes_per_sec); - if (!res.ok()) { - LOG(WARNING) << "fail to write stream to file. res = " << res; - return res; - } - } - } - - uint64_t file_length = file_handle.tell(); - file_header.set_file_length(file_length); - file_header.set_checksum(checksum); - *segment_file_size = file_length; - - // 写入更新之后的FileHeader - res = file_header.serialize(&file_handle); - if (!res.ok()) { - LOG(WARNING) << "write file header error. [err=" << Errno::str() << "]"; - return res; - } - - res = file_handle.close(); - if (!res.ok()) { - LOG(WARNING) << "fail to close file. [err=" << Errno::str() << "]"; - return res; - } - - return res; -} - -} // namespace doris diff --git a/be/src/olap/rowset/segment_writer.h b/be/src/olap/rowset/segment_writer.h deleted file mode 100644 index 53d6767005..0000000000 --- a/be/src/olap/rowset/segment_writer.h +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef DORIS_BE_SRC_OLAP_ROWSET_SEGMENT_WRITER_H -#define DORIS_BE_SRC_OLAP_ROWSET_SEGMENT_WRITER_H - -#include "olap/olap_define.h" -#include "olap/rowset/column_data_writer.h" - -namespace doris { - -class ColumnWriter; -class OutStreamFactory; -class ColumnDataHeaderMessage; - -class SegmentWriter { -public: - explicit SegmentWriter(const std::string& file_name, SegmentGroup* segment_group, - uint32_t stream_buffer_size, CompressKind compress_kind, - double bloom_filter_fpp); - ~SegmentWriter(); - Status init(uint32_t write_mbytes_per_sec); - Status write_batch(RowBlock* block, RowCursor* cursor, bool is_finalize); - // 通过对缓存的使用,预估最终segment的大小 - uint64_t estimate_segment_size(); - // 生成文件并写入缓存的数据 - Status finalize(uint32_t* segment_file_size); - -private: - // Helper: 生成最终的PB文件头 - Status _make_file_header(ColumnDataHeaderMessage* file_header); - -private: - std::string _file_name; - SegmentGroup* _segment_group; - uint32_t _stream_buffer_size; // 输出缓冲区大小 - CompressKind _compress_kind; - double _bloom_filter_fpp; - std::vector _root_writers; - OutStreamFactory* _stream_factory; - uint64_t _row_count; // 已经写入的行总数 - uint64_t _block_count; // 已经写入的block个数 - - // write limit - uint32_t _write_mbytes_per_sec; - - DISALLOW_COPY_AND_ASSIGN(SegmentWriter); -}; - -} // namespace doris - -#endif // DORIS_BE_SRC_OLAP_ROWSET_SEGMENT_WRITER_H \ No newline at end of file diff --git a/be/src/olap/serialize.cpp b/be/src/olap/serialize.cpp deleted file mode 100644 index fa436dd54b..0000000000 --- a/be/src/olap/serialize.cpp +++ /dev/null @@ -1,334 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/serialize.h" - -#include "olap/file_stream.h" -#include "olap/out_stream.h" - -namespace doris { -namespace ser { - -Status write_var_unsigned(OutStream* stream, int64_t value) { - Status res = Status::OK(); - - while (res.ok()) { - if ((value & ~0x7f) == 0) { - return stream->write((char)value); - } else { - res = stream->write((char)(0x80 | (value & 0x7f))); - value = ((uint64_t)value) >> 7; - } - } - - return res; -} - -Status read_var_unsigned(ReadOnlyFileStream* stream, int64_t* value) { - Status res; - int64_t result = 0; - uint32_t offset = 0; - char byte = 0; - - do { - res = stream->read(&byte); - - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to read stream. res = " << res; - return Status::OLAPInternalError(OLAP_ERR_COLUMN_DATA_READ_VAR_INT); - } - - result |= ((int64_t)(0x7f & byte)) << offset; - offset += 7; - } while (byte & 0x80); - - *value = result; - return Status::OK(); -} - -uint32_t find_closet_num_bits(int64_t value) { - // counting leading zero, builtin function, this will generate BSR(Bit Scan Reverse) - // instruction for X86 - if (value == 0) { - return 1; - } - auto count = 64 - __builtin_clzll(value); - return get_closet_fixed_bits(count); -} - -Status bytes_to_long_be(ReadOnlyFileStream* stream, int32_t n, int64_t* value) { - Status res = Status::OK(); - int64_t out = 0; - - // TODO(lijiao) : 为什么不直接用switch-case(会更快) - while (n > 0) { - --n; - // store it in a long and then shift else integer overflow will occur - // TODO(yiguolei): why not init it before - uint8_t byte = 0; - res = stream->read((char*)&byte); - - if (OLAP_UNLIKELY(!res.ok())) { - return res; - } - - // 小端？ - out |= ((uint64_t)byte << (n * 8)); - } - - *value = out; - return Status::OK(); -} - -uint32_t encode_bit_width(uint32_t n) { - static uint8_t bits_map[65] = { - ONE, // 0 - ONE, - TWO, - THREE, - FOUR, - FIVE, - SIX, - SEVEN, - EIGHT, // 1 - 8 - NINE, - TEN, - ELEVEN, - TWELVE, - THIRTEEN, - FOURTEEN, - FIFTEEN, - SIXTEEN, // 9 - 16 - // 17 - 24 - SEVENTEEN, - EIGHTEEN, - NINETEEN, - TWENTY, - TWENTYONE, - TWENTYTWO, - TWENTYTHREE, - TWENTYFOUR, - // 25 - 32 - TWENTYSIX, - TWENTYSIX, - TWENTYEIGHT, - TWENTYEIGHT, - THIRTY, - THIRTY, - THIRTYTWO, - THIRTYTWO, - // 33 - 40 - FORTY, - FORTY, - FORTY, - FORTY, - FORTY, - FORTY, - FORTY, - FORTY, - // 41 - 48 - FORTYEIGHT, - FORTYEIGHT, - FORTYEIGHT, - FORTYEIGHT, - FORTYEIGHT, - FORTYEIGHT, - FORTYEIGHT, - FORTYEIGHT, - // 49 - 56 - FIFTYSIX, - FIFTYSIX, - FIFTYSIX, - FIFTYSIX, - FIFTYSIX, - FIFTYSIX, - FIFTYSIX, - FIFTYSIX, - // 57 - 64 - SIXTYFOUR, - SIXTYFOUR, - SIXTYFOUR, - SIXTYFOUR, - SIXTYFOUR, - SIXTYFOUR, - SIXTYFOUR, - SIXTYFOUR, - }; - return bits_map[n]; -} - -uint32_t decode_bit_width(uint32_t n) { - static uint8_t bits_map[SIXTYFOUR + 1] = { - 1, 2, 3, 4, 5, 6, 7, 8, // ONE - EIGHT - 9, 10, 11, 12, 13, 14, 15, 16, // NINE - SIXTEEN - 17, 18, 19, 20, 21, 22, 23, 24, // SEVENTEEN - TWENTYFOUR - 26, // TWENTYSIX - 28, // TWENTYEIGHT - 30, // THIRTY - 32, // THIRTYTWO - 40, // FORTY - 48, // FORTYEIGHT - 56, // FIFTYSIX - 64 // SIXTYFOUR - }; - return bits_map[n]; -} - -uint32_t percentile_bits(int64_t* data, uint16_t count, double p) { - // histogram that store the encoded bit requirement for each values. - // maximum number of bits that can encoded is 32 (refer FixedBitSizes) - uint16_t hist[65]; - memset(hist, 0, sizeof(hist)); - // compute the histogram - for (uint32_t i = 0; i < count; i++) { - hist[used_bits(data[i])]++; - } - int32_t per_len = (int32_t)(count * (1.0 - p)); - // return the bits required by pth percentile length - for (int32_t i = 64; i >= 0; i--) { - per_len -= hist[i]; - if (per_len < 0) { - return get_closet_fixed_bits(i); - } - } - return 0; -} - -Status write_ints(OutStream* output, int64_t* data, uint32_t count, uint32_t bit_width) { - Status res = Status::OK(); - uint32_t bits_left = 8; - char current = 0; - - for (uint32_t i = 0; i < count; i++) { - uint64_t value = (uint64_t)data[i]; - uint32_t bits_to_write = bit_width; - - while (bits_to_write > bits_left) { - // add the bits to the bottom of the current word - current |= value >> (bits_to_write - bits_left); - // subtract out the bits we just added - bits_to_write -= bits_left; - // zero out the bits above bits_to_write - value &= (1UL << bits_to_write) - 1; - res = output->write(current); - - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write byte to stream.res = " << res; - return res; - } - - current = 0; - bits_left = 8; - } - - bits_left -= bits_to_write; - current |= value << bits_left; - - if (bits_left == 0) { - res = output->write(current); - - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write byte to stream.res = " << res; - return res; - } - - current = 0; - bits_left = 8; - } - } - - if (bits_left != 8) { - res = output->write(current); - - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write byte to stream.res = " << res; - return res; - } - } - - return Status::OK(); -} - -Status read_ints(ReadOnlyFileStream* input, int64_t* data, uint32_t count, uint32_t bit_width) { - Status res = Status::OK(); - uint32_t bits_left = 0; - char current = '\0'; - - uint32_t read_bytes = (count * bit_width - 1) / 8 + 1; - uint32_t remaining_bytes = 0; - char* buf = nullptr; - input->get_buf(&buf, &remaining_bytes); - if (read_bytes <= remaining_bytes) { - uint32_t pos = 0; - input->get_position(&pos); - for (uint32_t i = 0; i < count; i++) { - int64_t result = 0; - uint32_t bits_left_to_read = bit_width; - - while (bits_left_to_read > bits_left) { - result <<= bits_left; - result |= current & ((1 << bits_left) - 1); - bits_left_to_read -= bits_left; - current = buf[pos++]; - bits_left = 8; - } - - // handle the left over bits - if (bits_left_to_read > 0) { - result <<= bits_left_to_read; - bits_left -= bits_left_to_read; - result |= (current >> bits_left) & ((1 << bits_left_to_read) - 1); - } - - data[i] = result; - } - input->set_position(pos); - } else { - for (uint32_t i = 0; i < count; i++) { - int64_t result = 0; - uint32_t bits_left_to_read = bit_width; - - while (bits_left_to_read > bits_left) { - result <<= bits_left; - result |= current & ((1 << bits_left) - 1); - bits_left_to_read -= bits_left; - res = input->read(¤t); - - if (OLAP_UNLIKELY(!res.ok())) { - LOG(WARNING) << "fail to write byte to stream.res = " << res; - return res; - } - - bits_left = 8; - } - - // handle the left over bits - if (bits_left_to_read > 0) { - result <<= bits_left_to_read; - bits_left -= bits_left_to_read; - result |= (current >> bits_left) & ((1 << bits_left_to_read) - 1); - } - - data[i] = result; - } - } - - return res; -} - -} // namespace ser -} // namespace doris diff --git a/be/src/olap/serialize.h b/be/src/olap/serialize.h deleted file mode 100644 index 28c424bd1f..0000000000 --- a/be/src/olap/serialize.h +++ /dev/null @@ -1,184 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "olap/byte_buffer.h" -#include "olap/olap_define.h" - -namespace doris { - -class OutStream; -class ReadOnlyFileStream; - -namespace ser { - -// ZigZag transformation: put the sign bit to the lowest bit, and flip the other bits when it is negative -inline int64_t zig_zag_encode(int64_t value) { - return (value << 1) ^ (value >> 63); -} - -// ZigZag decoding -inline int64_t zig_zag_decode(int64_t value) { - return (((uint64_t)value) >> 1) ^ -(value & 1); -} - -// Variable-length encoding writes unsigned data, and variable-length encoding uses the highest bit to indicate whether to terminate: -//-1 there will be data behind -//-0 this is the last byte of the data -// The so-called unsigned data means that the data is not easy to appear. The sign bit is 1, and the subsequent consecutive 0; or from the sign bit -// 1 Continuous occurrence. This situation is prone to occur when the signature data represents a negative number. under these circumstances, -// Variable length coding cannot effectively reduce the code length, for this, please use write_var_signed. -Status write_var_unsigned(OutStream* stream, int64_t value); - -// Write signed data with variable length encoding, in order to avoid the problem of continuous 1s in the high bits of negative numbers, the data is ZigZag transformed -inline Status write_var_signed(OutStream* stream, int64_t value) { - return write_var_unsigned(stream, zig_zag_encode(value)); -} - -// Read in write_var_unsigned encoded data -Status read_var_unsigned(ReadOnlyFileStream* stream, int64_t* value); - -// Read in write_var_signed encoded data -inline Status read_var_signed(ReadOnlyFileStream* stream, int64_t* value) { - Status res = read_var_unsigned(stream, value); - - if (res.ok()) { - *value = zig_zag_decode(*value); - } - - return res; -} - -// The bit_width in RunLengthIntegerWriter is all 5bit encoding, -// so it supports up to 2^5=32 bit lengths. However, it needs to represent at most 1~64 bits, -// a total of 64 bit lengths, so in 64 bit lengths Take 32 types. -// The remaining 32 bit lengths that are not in these 32 types are aligned up to the nearest bit length. -// FixedBitSize gives 32 bit lengths -enum FixedBitSize { - ONE = 0, - TWO, - THREE, - FOUR, - FIVE, - SIX, - SEVEN, - EIGHT, - NINE, - TEN, - ELEVEN, - TWELVE, - THIRTEEN, - FOURTEEN, - FIFTEEN, - SIXTEEN, - SEVENTEEN, - EIGHTEEN, - NINETEEN, - TWENTY, - TWENTYONE, - TWENTYTWO, - TWENTYTHREE, - TWENTYFOUR, - TWENTYSIX, - TWENTYEIGHT, - THIRTY, - THIRTYTWO, - FORTY, - FORTYEIGHT, - FIFTYSIX, - SIXTYFOUR -}; - -inline uint32_t used_bits(uint64_t value) { - // counting leading zero, builtin function, this will generate BSR(Bit Scan Reverse) - // instruction for X86 - if (value == 0) { - return 0; - } - return 64 - __builtin_clzll(value); -} - -inline void compute_hists(int64_t* data, uint16_t count, uint16_t hists[65]) { - memset(hists, 0, sizeof(uint16_t) * 65); - // compute the histogram - for (uint32_t i = 0; i < count; i++) { - hists[used_bits(data[i])]++; - } -} - -// Returns the FixedBiteSize greater than or equal to n and closest to n -inline uint32_t get_closet_fixed_bits(uint32_t n) { - static uint8_t bits_map[65] = { - 1, // 0 - 1, 2, 3, 4, 5, 6, 7, 8, // 1 - 8 - 9, 10, 11, 12, 13, 14, 15, 16, // 9 - 16 - 17, 18, 19, 20, 21, 22, 23, 24, // 17 - 24 - 26, 26, 28, 28, 30, 30, 32, 32, // 25 - 32 - 40, 40, 40, 40, 40, 40, 40, 40, // 33 - 40 - 48, 48, 48, 48, 48, 48, 48, 48, // 41 - 48 - 56, 56, 56, 56, 56, 56, 56, 56, // 49 - 56 - 64, 64, 64, 64, 64, 64, 64, 64, // 57 - 64 - }; - return bits_map[n]; -} - -inline uint32_t percentile_bits_with_hist(uint16_t hists[65], uint16_t count, double p) { - int32_t per_len = (int32_t)(count * (1.0 - p)); - // return the bits required by pth percentile length - for (int32_t i = 64; i >= 0; i--) { - per_len -= hists[i]; - if (per_len < 0) { - return get_closet_fixed_bits(i); - } - } - return 0; -} - -// First calculate the bit length of value (the highest bit of 1), and then use get_closet_fixed_bits -// Return the closest FixedBiteSize -uint32_t find_closet_num_bits(int64_t value); - -// Read n bytes in big endian order and convert to long -Status bytes_to_long_be(ReadOnlyFileStream* stream, int32_t n, int64_t* value); - -// Encode the bit length as one of 32 fixed-length bits, and the return value is between 0 and 31 -uint32_t encode_bit_width(uint32_t n); - -// Decode the result of encode_bit_width encoding -uint32_t decode_bit_width(uint32_t n); - -// Sort the data in data according to the bit length, and return the maximum bit length under a given ratio p. -// For example: p == 1.0, which means the maximum bit length of all data -// p == 0.9, which means the maximum bit length of 90% of the data with the shortest bit position -// p == 0.5, which means the maximum bit length of the 50% data with the shortest bit position -uint32_t percentile_bits(int64_t* data, uint16_t count, double p); - -// Output a set of integers to output in a compact manner -Status write_ints(OutStream* output, int64_t* data, uint32_t count, uint32_t bit_width); - -// Read the data output by write_ints -Status read_ints(ReadOnlyFileStream* input, int64_t* data, uint32_t count, uint32_t bit_width); - -// Do not want to use Guava LongMath.checkedSubtract() here as it will throw -// ArithmeticException in case of overflow -inline bool is_safe_subtract(int64_t left, int64_t right) { - return ((left ^ right) >= 0) | ((left ^ (left - right)) >= 0); -} - -} // namespace ser -} // namespace doris diff --git a/be/src/olap/short_key_index.h b/be/src/olap/short_key_index.h index 86cf67f9f2..347c609433 100644 --- a/be/src/olap/short_key_index.h +++ b/be/src/olap/short_key_index.h @@ -118,8 +118,7 @@ void encode_key(std::string* buf, const RowType& row, size_t num_keys) { // ... // builder.add_item(keyN); // builder.finalize(segment_size, num_rows, &slices); -// NOTE: This is used for BetaRowset and is not compatible with AlphaRowset's -// short key index format. +// NOTE: This is used for BetaRowset // TODO(zc): // 1. If this can leverage binary page to save key and offset data // 2. Extending this to save in a BTree like struct, which can index full key diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h index aab134297f..93c68e1cf1 100644 --- a/be/src/olap/snapshot_manager.h +++ b/be/src/olap/snapshot_manager.h @@ -33,7 +33,6 @@ #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/push_handler.h" -#include "olap/rowset/column_data.h" #include "olap/tablet.h" #include "olap/tablet_meta_manager.h" #include "util/doris_metrics.h" diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index fd6aba53cb..2fefea2990 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -46,9 +46,7 @@ #include "olap/memtable_flush_executor.h" #include "olap/push_handler.h" #include "olap/reader.h" -#include "olap/rowset/alpha_rowset.h" #include "olap/rowset/alpha_rowset_meta.h" -#include "olap/rowset/column_data_writer.h" #include "olap/rowset/rowset_meta_manager.h" #include "olap/rowset/unique_rowset_id_generator.h" #include "olap/schema_change.h" @@ -163,9 +161,6 @@ StorageEngine::~StorageEngine() { if (_cumu_compaction_thread_pool) { _cumu_compaction_thread_pool->shutdown(); } - if (_convert_rowset_thread_pool) { - _convert_rowset_thread_pool->shutdown(); - } if (_tablet_meta_checkpoint_thread_pool) { _tablet_meta_checkpoint_thread_pool->shutdown(); } @@ -582,9 +577,6 @@ void StorageEngine::stop() { } THREAD_JOIN(_compaction_tasks_producer_thread); - if (_alpha_rowset_scan_thread) { - THREAD_JOIN(_alpha_rowset_scan_thread); - } THREAD_JOIN(_unused_rowset_monitor_thread); THREAD_JOIN(_garbage_sweeper_thread); THREAD_JOIN(_disk_stat_monitor_thread); diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index e54a8d5f91..df7f24022e 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -255,8 +255,6 @@ private: void _compaction_tasks_producer_callback(); - void _alpha_rowset_scan_thread_callback(); - std::vector _generate_compaction_tasks(CompactionType compaction_type, std::vector& data_dirs, bool check_score); @@ -387,9 +385,6 @@ private: std::unique_ptr _base_compaction_thread_pool; std::unique_ptr _cumu_compaction_thread_pool; - scoped_refptr _alpha_rowset_scan_thread; - std::unique_ptr _convert_rowset_thread_pool; - std::unique_ptr _tablet_meta_checkpoint_thread_pool; CompactionPermitLimiter _permit_limiter; diff --git a/be/src/olap/stream_index_reader.cpp b/be/src/olap/stream_index_reader.cpp deleted file mode 100644 index 97a0446568..0000000000 --- a/be/src/olap/stream_index_reader.cpp +++ /dev/null @@ -1,145 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/stream_index_reader.h" - -namespace doris { - -PositionEntryReader::PositionEntryReader() - : _positions(nullptr), _positions_count(0), _statistics_offset(0) {} - -Status PositionEntryReader::init(StreamIndexHeader* header, FieldType type, bool null_supported) { - if (nullptr == header) { - return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); - } - - set_positions_count(header->position_format); - - if (!_statistics.init(type, null_supported)) { - return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); - } - - return Status::OK(); -} - -void PositionEntryReader::attach(char* buffer) { - _positions = reinterpret_cast(buffer); - _statistics.attach(buffer + _statistics_offset); -} - -int64_t PositionEntryReader::positions(size_t index) const { - if (index < _positions_count) { - return static_cast(_positions[index]); - } - - return -1; -} - -size_t PositionEntryReader::entry_size() const { - return sizeof(uint32_t) * _positions_count + _statistics.size(); -} - -const ColumnStatistics& PositionEntryReader::column_statistic() const { - return _statistics; -} - -void PositionEntryReader::set_positions_count(size_t count) { - _positions_count = count; - _statistics_offset = count * sizeof(uint32_t); -} - -int32_t PositionEntryReader::positions_count() const { - return _positions_count; -} - -StreamIndexReader::StreamIndexReader() - : _buffer(nullptr), - _buffer_size(0), - _start_offset(0), - _step_size(0), - _is_using_cache(false), - _entry() {} - -///////////////////////////////////////////////////////////////////////////// - -StreamIndexReader::~StreamIndexReader() { - if (!_is_using_cache) { - SAFE_DELETE_ARRAY(_buffer); - } -} - -Status StreamIndexReader::init(char* buffer, size_t buffer_size, FieldType type, - bool is_using_cache, bool null_supported) { - if (nullptr == buffer) { - LOG(WARNING) << "buffer given is invalid."; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - _buffer = buffer; - _buffer_size = buffer_size; - _is_using_cache = is_using_cache; - _null_supported = null_supported; - Status res = _parse_header(type); - - if (!res.ok()) { - LOG(WARNING) << "fail to parse header"; - return res; - } - - return Status::OK(); -} - -const PositionEntryReader& StreamIndexReader::entry(uint64_t entry_id) { - _entry.attach(_buffer + _start_offset + _step_size * entry_id); - return _entry; -} - -size_t StreamIndexReader::entry_count() { - return _entry_count; -} - -Status StreamIndexReader::_parse_header(FieldType type) { - if (_buffer_size < sizeof(StreamIndexHeader)) { - LOG(WARNING) << "invalid length"; - return Status::OLAPInternalError(OLAP_ERR_OUT_OF_BOUND); - } - - StreamIndexHeader* header = reinterpret_cast(_buffer); - Status res = Status::OK(); - - res = _entry.init(header, type, _null_supported); - - if (!res.ok()) { - LOG(WARNING) << "fail to init statistic reader"; - return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED); - } - - _start_offset = sizeof(StreamIndexHeader); - _step_size = _entry.entry_size(); - _entry_count = header->block_count; - - if (_entry_count * _step_size + _start_offset > _buffer_size) { - LOG(WARNING) << "invalid header length, entry_count=" << _entry_count - << ", step_size=" << _step_size << ", start_offset=" << _start_offset - << ", buffer_size=" << _buffer_size; - return Status::OLAPInternalError(OLAP_ERR_FILE_FORMAT_ERROR); - } - - return res; -} - -} // namespace doris diff --git a/be/src/olap/stream_index_reader.h b/be/src/olap/stream_index_reader.h deleted file mode 100644 index 4162ada5ae..0000000000 --- a/be/src/olap/stream_index_reader.h +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "olap/olap_define.h" -#include "olap/stream_index_common.h" - -namespace doris { - -class PositionEntryReader { -public: - PositionEntryReader(); - ~PositionEntryReader() {} - // 使用前需要初始化，需要header来计算每一组position/stat的偏移量 - Status init(StreamIndexHeader* header, FieldType type, bool null_supported); - // attach到一块内存上读取position和stat - void attach(char* buffer); - // 返回指定下标的position - int64_t positions(size_t index) const; - // 一个entry的内存大小 - size_t entry_size() const; - // 统计信息 - const ColumnStatistics& column_statistic() const; - // 设置position的个数，并据此计算出偏移 - void set_positions_count(size_t count); - // 一共有几个position - int32_t positions_count() const; - bool all_null() const { - return _statistics.minimum()->is_null() && _statistics.maximum()->is_null(); - } - -private: - const uint32_t* _positions; - size_t _positions_count; - ColumnStatistics _statistics; - size_t _statistics_offset; -}; - -class PositionProvider { -public: - PositionProvider() : _entry(nullptr), _index(0) {} - explicit PositionProvider(const PositionEntryReader* entry) : _entry(entry), _index(0) {} - ~PositionProvider() {} - int64_t get_next() { return _entry->positions(_index++); } - bool all_null() const { return _entry->all_null(); } - -private: - const PositionEntryReader* _entry; - int _index; -}; - -class StreamIndexReader { -public: - StreamIndexReader(); - ~StreamIndexReader(); - - Status init(char* buffer, size_t buffer_size, FieldType type, bool is_using_cache, - bool null_supported); - const PositionEntryReader& entry(uint64_t entry_id); - size_t entry_count(); - -protected: - Status _parse_header(FieldType type); - -private: - char* _buffer; - size_t _buffer_size; - size_t _start_offset; - size_t _step_size; - size_t _entry_count; - bool _is_using_cache; - bool _null_supported; - PositionEntryReader _entry; -}; - -} // namespace doris diff --git a/be/src/olap/stream_index_writer.cpp b/be/src/olap/stream_index_writer.cpp deleted file mode 100644 index 810836d609..0000000000 --- a/be/src/olap/stream_index_writer.cpp +++ /dev/null @@ -1,178 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/stream_index_writer.h" - -#include - -namespace doris { - -PositionEntryWriter::PositionEntryWriter() : _positions_count(0), _statistics_size(0) { - memset(_statistics_buffer, 0, sizeof(_statistics_buffer)); -} - -int64_t PositionEntryWriter::positions(size_t index) const { - if (index < _positions_count) { - return static_cast(_positions[index]); - } - - return -1; -} - -Status PositionEntryWriter::set_statistic(ColumnStatistics* statistic) { - _statistics_size = statistic->size(); - return statistic->write_to_buffer(_statistics_buffer, MAX_STATISTIC_LENGTH); -} - -bool PositionEntryWriter::has_statistic() const { - return _statistics_size != 0; -} - -int32_t PositionEntryWriter::positions_count() const { - return _positions_count; -} - -int32_t PositionEntryWriter::output_size() const { - return _positions_count * sizeof(uint32_t) + _statistics_size; -} - -Status PositionEntryWriter::add_position(uint32_t position) { - if (_positions_count < MAX_POSITION_SIZE) { - _positions[_positions_count] = position; - _positions_count++; - return Status::OK(); - } - - return Status::OLAPInternalError(OLAP_ERR_BUFFER_OVERFLOW); -} - -void PositionEntryWriter::reset_write_offset() { - _positions_count = 0; -} - -Status PositionEntryWriter::remove_written_position(uint32_t from, size_t count) { - if (from + count > _positions_count) { - return Status::OLAPInternalError(OLAP_ERR_OUT_OF_BOUND); - } - - for (size_t i = from; i < _positions_count - count; ++i) { - _positions[i] = _positions[i + count]; - } - - _positions_count -= count; - return Status::OK(); -} - -void PositionEntryWriter::write_to_buffer(char* out_buffer) { - // 实际上这样比较快 - //for(uint32_t i = 0; i < _positions_count; ++i) { - // buffer[i] = _positions[i]; - //} - size_t position_output_size = _positions_count * sizeof(uint32_t); - memcpy(out_buffer, _positions, position_output_size); - memcpy(out_buffer + position_output_size, _statistics_buffer, _statistics_size); -} - -///////////////////////////////////////////////////////////////////////////// - -StreamIndexWriter::StreamIndexWriter(FieldType field_type) : _field_type(field_type) {} - -StreamIndexWriter::~StreamIndexWriter() {} - -Status StreamIndexWriter::add_index_entry(const PositionEntryWriter& entry) { - try { - _index_to_write.push_back(entry); - } catch (...) { - LOG(WARNING) << "add entry to index vector fail"; - return Status::OLAPInternalError(OLAP_ERR_STL_ERROR); - } - - return Status::OK(); -} - -PositionEntryWriter* StreamIndexWriter::mutable_entry(uint32_t index) { - if (index < _index_to_write.size()) { - return &_index_to_write[index]; - } - - return nullptr; -} - -size_t StreamIndexWriter::entry_size() { - return _index_to_write.size(); -} - -Status StreamIndexWriter::reset() { - try { - _index_to_write.clear(); - } catch (...) { - LOG(WARNING) << "add entry to index vector fail"; - return Status::OLAPInternalError(OLAP_ERR_STL_ERROR); - } - - return Status::OK(); -} - -size_t StreamIndexWriter::output_size() { - if (_index_to_write.size() == 0) { - return sizeof(_header); - } else { - return _index_to_write.size() * _index_to_write[0].output_size() + sizeof(_header); - } -} - -Status StreamIndexWriter::write_to_buffer(char* buffer, size_t buffer_size) { - if (nullptr == buffer) { - LOG(WARNING) << "given buffer is null"; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - if (output_size() > buffer_size) { - LOG(WARNING) << "need more buffer, size=" << output_size() << ", given=" << buffer_size; - return Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR); - } - - // write header - int32_t entry_size = 0; - - if (_index_to_write.size() != 0) { - // entry size 包含了position和统计信息的长度 - // string列的统计信息可能是0,因此实际上写不进去 - entry_size = _index_to_write[0].output_size(); - _header.position_format = _index_to_write[0].positions_count(); - - if (_index_to_write[0].has_statistic()) { - _header.statistic_format = _field_type; - } - } - - _header.block_count = _index_to_write.size(); - VLOG_TRACE << "header info. pos: " << _header.position_format - << ", stat:" << _header.statistic_format << ", entry_size:" << entry_size; - memcpy(buffer, reinterpret_cast(&_header), sizeof(_header)); - // set offset, write data - char* write_offset = buffer + sizeof(_header); - - for (size_t i = 0; i < _index_to_write.size(); ++i) { - _index_to_write[i].write_to_buffer(write_offset); - write_offset += entry_size; - } - - return Status::OK(); -} - -} // namespace doris diff --git a/be/src/olap/stream_index_writer.h b/be/src/olap/stream_index_writer.h deleted file mode 100644 index fc41adfbf4..0000000000 --- a/be/src/olap/stream_index_writer.h +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "olap/file_stream.h" -#include "olap/stream_index_common.h" - -namespace doris { - -// 目前写入和读取分离了，主要是因为写入用了一个定长的buffer -// 不过实际上可以合并 -class PositionEntryWriter { -public: - PositionEntryWriter(); - // 获取指定位置的position信息 - int64_t positions(size_t index) const; - // 获取position的个数 - int32_t positions_count() const; - // 设置统计信息 - Status set_statistic(ColumnStatistics* statistic); - // 判断当前entry是否写入了统计信息 - bool has_statistic() const; - // 输出到buffer的大小，用来预先分配内存用 - int32_t output_size() const; - // 增加一个position, 目前给出的上限是16 - // ，目前不会有超出这个数量的位置信息 - Status add_position(uint32_t position); - // 实际上是重置position，便于重写时写入正确的位置 - void reset_write_offset(); - // 移除从from起，count个已经写入的位置，用于去除present流 - Status remove_written_position(uint32_t from, size_t count); - // 将内容输出到outbuffer中。 - void write_to_buffer(char* out_buffer); - -private: - uint32_t _positions[MAX_POSITION_SIZE]; - size_t _positions_count; - char _statistics_buffer[MAX_STATISTIC_LENGTH]; - size_t _statistics_size; -}; - -class StreamIndexWriter { -public: - StreamIndexWriter(FieldType field_type = OLAP_FIELD_TYPE_NONE); - ~StreamIndexWriter(); - - Status add_index_entry(const PositionEntryWriter& entry); - PositionEntryWriter* mutable_entry(uint32_t index); - size_t entry_size(); - Status reset(); - - size_t output_size(); - Status write_to_buffer(char* buffer, size_t buffer_size); - -protected: - std::vector _index_to_write; - StreamIndexHeader _header; - FieldType _field_type; -}; - -} // namespace doris diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index da29ce7253..ac92ff63d7 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1086,15 +1086,6 @@ void Tablet::pick_candidate_rowsets_to_cumulative_compaction( candidate_rowsets); } -void Tablet::find_alpha_rowsets(std::vector* rowsets) const { - std::shared_lock rdlock(_meta_lock); - for (auto& it : _rs_version_map) { - if (it.second->rowset_meta()->rowset_type() == RowsetTypePB::ALPHA_ROWSET) { - rowsets->push_back(it.second); - } - } -} - void Tablet::pick_candidate_rowsets_to_base_compaction(vector* candidate_rowsets) { std::shared_lock rdlock(_meta_lock); for (auto& it : _rs_version_map) { diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index e0f124988d..25563a3487 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -268,8 +268,6 @@ public: return _tablet_meta->all_beta(); } - void find_alpha_rowsets(std::vector* rowsets) const; - Status create_rowset_writer(const Version& version, const RowsetStatePB& rowset_state, const SegmentsOverlapPB& overlap, std::unique_ptr* rowset_writer); diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index e002b07cde..98954a7b1d 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -39,7 +39,6 @@ #include "olap/olap_common.h" #include "olap/push_handler.h" #include "olap/reader.h" -#include "olap/rowset/column_data_writer.h" #include "olap/rowset/rowset_id_generator.h" #include "olap/schema_change.h" #include "olap/tablet.h" @@ -633,20 +632,6 @@ void TabletManager::get_tablet_stat(TTabletStatResult* result) { result->__set_tablet_stat_list(*local_cache); } -void TabletManager::find_tablet_have_alpha_rowset(std::vector& tablets) { - for (const auto& tablets_shard : _tablets_shards) { - std::shared_lock rdlock(tablets_shard.lock); - for (const auto& tablet_map : tablets_shard.tablet_map) { - const TabletSharedPtr& tablet_ptr = tablet_map.second; - if (!tablet_ptr->all_beta() && - tablet_ptr->can_do_compaction(tablet_ptr->data_dir()->path_hash(), - BASE_COMPACTION)) { - tablets.push_back(tablet_ptr); - } - } - } -} - TabletSharedPtr TabletManager::find_best_tablet_to_compaction( CompactionType compaction_type, DataDir* data_dir, const std::unordered_set& tablet_submitted_compaction, uint32_t* score, diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h index 1e2af26d99..8052361d03 100644 --- a/be/src/olap/tablet_manager.h +++ b/be/src/olap/tablet_manager.h @@ -140,8 +140,6 @@ public: void get_all_tablets_storage_format(TCheckStorageFormatResult* result); - void find_tablet_have_alpha_rowset(std::vector& tablets); - private: // Add a tablet pointer to StorageEngine // If force, drop the existing tablet add this new one diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp index bad96a794f..0cf09c0876 100644 --- a/be/src/olap/txn_manager.cpp +++ b/be/src/olap/txn_manager.cpp @@ -38,7 +38,6 @@ #include "olap/lru_cache.h" #include "olap/push_handler.h" #include "olap/reader.h" -#include "olap/rowset/column_data_writer.h" #include "olap/rowset/rowset_meta_manager.h" #include "olap/schema_change.h" #include "olap/storage_engine.h" diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt index 2cc56e137b..0d27c398a7 100644 --- a/be/src/runtime/CMakeLists.txt +++ b/be/src/runtime/CMakeLists.txt @@ -55,7 +55,6 @@ set(RUNTIME_FILES collection_value.cpp tuple.cpp tuple_row.cpp - vectorized_row_batch.cpp dpp_writer.cpp qsorter.cpp fragment_mgr.cpp diff --git a/be/src/runtime/vectorized_row_batch.cpp b/be/src/runtime/vectorized_row_batch.cpp deleted file mode 100644 index 7e1d6449f1..0000000000 --- a/be/src/runtime/vectorized_row_batch.cpp +++ /dev/null @@ -1,140 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "runtime/vectorized_row_batch.h" - -#include "common/logging.h" -#include "olap/row_block.h" - -namespace doris { - -VectorizedRowBatch::VectorizedRowBatch(const TabletSchema* schema, - const std::vector& cols, int capacity) - : _schema(schema), _cols(cols), _capacity(capacity), _limit(capacity) { - _selected_in_use = false; - _size = 0; - - _mem_pool.reset(new MemPool()); - - _selected = reinterpret_cast(new char[sizeof(uint16_t) * _capacity]); - - _col_vectors.resize(schema->num_columns(), nullptr); - for (ColumnId column_id : cols) { - _col_vectors[column_id] = new ColumnVector(); - } -} - -void VectorizedRowBatch::dump_to_row_block(RowBlock* row_block) { - if (_selected_in_use) { - for (auto column_id : _cols) { - ColumnVector* col_vec = _col_vectors[column_id]; - - bool no_nulls = col_vec->no_nulls(); - // pointer of this field's vector - char* vec_field_ptr = (char*)col_vec->col_data(); - // pointer of this field in row block - char* row_field_ptr = - row_block->_mem_buf + row_block->_field_offset_in_memory[column_id]; - const TabletColumn& column = _schema->column(column_id); - size_t field_size = 0; - if (column.is_length_variable_type()) { - field_size = sizeof(Slice); - } else { - field_size = column.length(); - } - if (no_nulls) { - for (int row = 0; row < _size; ++row) { - char* vec_field = vec_field_ptr + _selected[row] * field_size; - // Set not null - *row_field_ptr = 0; - memory_copy(row_field_ptr + 1, vec_field, field_size); - - // point to next row - row_field_ptr += row_block->_mem_row_bytes; - } - } else { - bool* is_null = col_vec->is_null(); - for (int row = 0; row < _size; ++row) { - if (is_null[_selected[row]]) { - *row_field_ptr = 1; - } else { - char* vec_field = vec_field_ptr + _selected[row] * field_size; - // Set not null - *row_field_ptr = 0; - memory_copy(row_field_ptr + 1, vec_field, field_size); - } - row_field_ptr += row_block->_mem_row_bytes; - } - } - } - } else { - for (auto column_id : _cols) { - ColumnVector* col_vec = _col_vectors[column_id]; - - bool no_nulls = col_vec->no_nulls(); - - char* vec_field_ptr = (char*)col_vec->col_data(); - char* row_field_ptr = - row_block->_mem_buf + row_block->_field_offset_in_memory[column_id]; - - const TabletColumn& column = _schema->column(column_id); - size_t field_size = 0; - if (column.is_length_variable_type()) { - field_size = sizeof(Slice); - } else { - field_size = column.length(); - } - - if (no_nulls) { - for (int row = 0; row < _size; ++row) { - char* vec_field = vec_field_ptr; - // Set not null - *row_field_ptr = 0; - memory_copy(row_field_ptr + 1, vec_field, field_size); - row_field_ptr += row_block->_mem_row_bytes; - vec_field_ptr += field_size; - } - } else { - bool* is_null = col_vec->is_null(); - for (int row = 0; row < _size; ++row) { - if (is_null[row]) { - *row_field_ptr = 1; - } else { - char* vec_field = vec_field_ptr; - // Set not null - *row_field_ptr = 0; - memory_copy(row_field_ptr + 1, vec_field, field_size); - } - row_field_ptr += row_block->_mem_row_bytes; - vec_field_ptr += field_size; - } - } - } - } - - row_block->_pos = 0; - row_block->_limit = _size; - row_block->_info.row_num = _size; - row_block->_block_status = _block_status; - - // exchange two memory pool to reduce chunk allocate in MemPool, - row_block->mem_pool()->exchange_data(_mem_pool.get()); - // Clear to reuse already allocated chunk - _mem_pool->clear(); -} - -} // namespace doris diff --git a/be/src/runtime/vectorized_row_batch.h b/be/src/runtime/vectorized_row_batch.h deleted file mode 100644 index 85c5c8e7ce..0000000000 --- a/be/src/runtime/vectorized_row_batch.h +++ /dev/null @@ -1,125 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "olap/field.h" -#include "olap/row_cursor.h" -#include "runtime/descriptors.h" -#include "runtime/mem_pool.h" -#include "runtime/row_batch.h" -#include "runtime/row_batch_interface.hpp" -#include "runtime/tuple.h" -#include "runtime/tuple_row.h" -#include "util/mem_util.hpp" - -namespace doris { - -class VectorizedRowBatch; -class RowBlock; - -class ColumnVector { -public: - ColumnVector() {} - ~ColumnVector() {} - - bool* is_null() const { return _is_null; } - - void set_is_null(bool* is_null) { _is_null = is_null; } - - bool no_nulls() const { return _no_nulls; } - - void set_no_nulls(bool no_nulls) { _no_nulls = no_nulls; } - - void* col_data() { return _col_data; } - void set_col_data(void* data) { _col_data = data; } - -private: - void* _col_data = nullptr; - bool _no_nulls = false; - bool* _is_null = nullptr; -}; - -class VectorizedRowBatch { -public: - VectorizedRowBatch(const TabletSchema* schema, const std::vector& cols, int capacity); - - ~VectorizedRowBatch() { - for (auto vec : _col_vectors) { - delete vec; - } - delete[] _selected; - } - - MemPool* mem_pool() { return _mem_pool.get(); } - - ColumnVector* column(int column_index) { return _col_vectors[column_index]; } - - const std::vector& columns() const { return _cols; } - - uint16_t capacity() { return _capacity; } - - uint16_t size() { return _size; } - - void set_size(int size) { - DCHECK_LE(size, _capacity); - _size = size; - } - - int num_rows() { return _size; } - - bool selected_in_use() const { return _selected_in_use; } - - void set_selected_in_use(bool selected_in_use) { _selected_in_use = selected_in_use; } - - uint16_t* selected() const { return _selected; } - - void clear() { - _size = 0; - _selected_in_use = false; - _limit = _capacity; - _mem_pool->clear(); - } - - uint16_t limit() const { return _limit; } - void set_limit(uint16_t limit) { _limit = limit; } - void set_block_status(uint8_t status) { _block_status = status; } - uint8_t block_status() const { return _block_status; } - - // Dump this vector batch to RowBlock; - void dump_to_row_block(RowBlock* row_block); - -private: - const TabletSchema* _schema; - const std::vector& _cols; - const uint16_t _capacity; - uint16_t _size = 0; - uint16_t* _selected = nullptr; - std::vector _col_vectors; - - bool _selected_in_use = false; - uint8_t _block_status; - - std::unique_ptr _mem_pool; - uint16_t _limit; -}; - -} // namespace doris diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index 6ee7bccaed..9e2e7cb2f5 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -132,32 +132,24 @@ set(HTTP_TEST_FILES ) set(OLAP_TEST_FILES olap/engine_storage_migration_task_test.cpp - olap/schema_change_test.cpp olap/timestamped_version_tracker_test.cpp olap/tablet_schema_helper.cpp olap/delta_writer_test.cpp olap/delete_handler_test.cpp olap/row_block_test.cpp olap/row_block_v2_test.cpp - olap/bit_field_test.cpp olap/byte_buffer_test.cpp - olap/run_length_byte_test.cpp - olap/run_length_integer_test.cpp - olap/stream_index_test.cpp olap/lru_cache_test.cpp olap/bloom_filter_test.cpp olap/bloom_filter_column_predicate_test.cpp - olap/bloom_filter_index_test.cpp olap/comparison_predicate_test.cpp olap/in_list_predicate_test.cpp olap/null_predicate_test.cpp olap/file_helper_test.cpp olap/file_utils_test.cpp - olap/column_reader_test.cpp olap/cumulative_compaction_policy_test.cpp olap/row_cursor_test.cpp olap/skiplist_test.cpp - olap/serialize_test.cpp olap/olap_meta_test.cpp olap/decimal12_test.cpp olap/column_vector_test.cpp @@ -183,12 +175,13 @@ set(OLAP_TEST_FILES olap/tablet_meta_manager_test.cpp olap/tablet_mgr_test.cpp olap/tablet_test.cpp - olap/rowset/rowset_meta_manager_test.cpp + # TODO yiguolei: it is using alpha rowset to test, should use beta rowset + #olap/rowset/rowset_meta_manager_test.cpp olap/rowset/rowset_meta_test.cpp - olap/rowset/alpha_rowset_test.cpp olap/rowset/beta_rowset_test.cpp olap/rowset/unique_rowset_id_generator_test.cpp - olap/txn_manager_test.cpp + # TODO yiguolei: it is using alpha rowset to test, should use beta rowset + #olap/txn_manager_test.cpp olap/generic_iterators_test.cpp olap/key_coder_test.cpp olap/short_key_index_test.cpp @@ -247,7 +240,7 @@ set(RUNTIME_TEST_FILES runtime/memory/system_allocator_test.cpp runtime/cache/partition_cache_test.cpp runtime/collection_value_test.cpp - runtime/array_test.cpp + #runtime/array_test.cpp ) set(TESTUTIL_TEST_FILES testutil/test_util.cpp diff --git a/be/test/exprs/binary_predicate_test.cpp b/be/test/exprs/binary_predicate_test.cpp index 88d1566cd3..a53d3754fc 100644 --- a/be/test/exprs/binary_predicate_test.cpp +++ b/be/test/exprs/binary_predicate_test.cpp @@ -26,7 +26,6 @@ #include "gen_cpp/Exprs_types.h" #include "runtime/row_batch.h" #include "runtime/runtime_state.h" -#include "runtime/vectorized_row_batch.h" #include "util/debug_util.h" namespace doris { @@ -155,91 +154,6 @@ TEST_F(BinaryOpTest, PrepareTest) { EXPECT_TRUE(expr->prepare(runtime_state(), *row_desc()).ok()); } -TEST_F(BinaryOpTest, NormalTest) { - Expr* expr = create_expr(); - EXPECT_TRUE(expr != nullptr); - EXPECT_TRUE(expr->prepare(runtime_state(), *row_desc()).ok()); - int capacity = 256; - VectorizedRowBatch* vec_row_batch = - object_pool()->add(new VectorizedRowBatch(_schema, capacity)); - MemPool* mem_pool = vec_row_batch->mem_pool(); - int32_t* vec_data = reinterpret_cast(mem_pool->allocate(sizeof(int32_t) * capacity)); - vec_row_batch->column(0)->set_col_data(vec_data); - - for (int i = 0; i < capacity; ++i) { - vec_data[i] = i; - } - - vec_row_batch->set_size(capacity); - expr->evaluate(vec_row_batch); - EXPECT_EQ(vec_row_batch->size(), 10); - - Tuple tuple; - int vv = 0; - - while (vec_row_batch->get_next_tuple(&tuple, - *runtime_state()->desc_tbl().get_tuple_descriptor(0))) { - EXPECT_EQ(vv++, *reinterpret_cast(tuple.get_slot(4))); - } -} - -TEST_F(BinaryOpTest, SimplePerformanceTest) { - EXPECT_EQ(1, _row_desc->tuple_descriptors().size()); - for (int capacity = 128; capacity <= 1024 * 128; capacity *= 2) { - Expr* expr = create_expr(); - EXPECT_TRUE(expr != nullptr); - EXPECT_TRUE(expr->prepare(runtime_state(), *row_desc()).ok()); - int size = 1024 * 1024 / capacity; - VectorizedRowBatch* vec_row_batches[size]; - srand(time(nullptr)); - - for (int i = 0; i < size; ++i) { - vec_row_batches[i] = object_pool()->add(new VectorizedRowBatch(_schema, capacity)); - MemPool* mem_pool = vec_row_batches[i]->mem_pool(); - int32_t* vec_data = - reinterpret_cast(mem_pool->allocate(sizeof(int32_t) * capacity)); - vec_row_batches[i]->column(0)->set_col_data(vec_data); - - for (int i = 0; i < capacity; ++i) { - vec_data[i] = rand() % 20; - } - - vec_row_batches[i]->set_size(capacity); - } - - RowBatch* row_batches[size]; - - for (int i = 0; i < size; ++i) { - row_batches[i] = object_pool()->add(new RowBatch(*row_desc(), capacity)); - vec_row_batches[i]->to_row_batch(row_batches[i], - *runtime_state()->desc_tbl().get_tuple_descriptor(0)); - } - - MonotonicStopWatch stopwatch; - stopwatch.start(); - - for (int i = 0; i < size; ++i) { - expr->evaluate(vec_row_batches[i]); - } - - uint64_t vec_time = stopwatch.elapsed_time(); - VLOG_CRITICAL << PrettyPrinter::print(vec_time, TCounterType::TIME_NS); - - stopwatch.start(); - - for (int i = 0; i < size; ++i) { - for (int j = 0; j < capacity; ++j) { - ExecNode::eval_conjuncts(&expr, 1, row_batches[i]->get_row(j)); - } - } - - uint64_t row_time = stopwatch.elapsed_time(); - VLOG_CRITICAL << PrettyPrinter::print(row_time, TCounterType::TIME_NS); - - VLOG_CRITICAL << "capacity: " << capacity << " multiple: " << row_time / vec_time; - } -} - } // namespace doris /* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ diff --git a/be/test/exprs/in_op_test.cpp b/be/test/exprs/in_op_test.cpp index 9e0413d7b8..c1de71f4e9 100644 --- a/be/test/exprs/in_op_test.cpp +++ b/be/test/exprs/in_op_test.cpp @@ -25,7 +25,6 @@ #include "gen_cpp/Exprs_types.h" #include "runtime/row_batch.h" #include "runtime/runtime_state.h" -#include "runtime/vectorized_row_batch.h" #include "util/debug_util.h" namespace doris { @@ -146,89 +145,6 @@ TEST_F(InOpTest, PrepareTest) { EXPECT_TRUE(expr->prepare(_runtime_state, *_row_desc).ok()); } -TEST_F(InOpTest, NormalTest) { - Expr* expr = create_expr(); - EXPECT_TRUE(expr != nullptr); - EXPECT_TRUE(expr->prepare(_runtime_state, *_row_desc).ok()); - int capacity = 256; - VectorizedRowBatch* vec_row_batch = _object_pool->add( - new VectorizedRowBatch(*_runtime_state->desc_tbl().get_tuple_descriptor(0), capacity)); - MemPool* mem_pool = vec_row_batch->mem_pool(); - int32_t* vec_data = reinterpret_cast(mem_pool->allocate(sizeof(int32_t) * capacity)); - vec_row_batch->column(0)->set_col_data(vec_data); - - for (int i = 0; i < capacity; ++i) { - vec_data[i] = i; - } - - vec_row_batch->set_size(capacity); - expr->evaluate(vec_row_batch); - EXPECT_EQ(vec_row_batch->size(), 128); - - Tuple tuple; - int vv = 0; - - while (vec_row_batch->get_next_tuple(&tuple)) { - EXPECT_EQ(vv++, *reinterpret_cast(tuple.get_slot(4))); - } -} - -TEST_F(InOpTest, SimplePerformanceTest) { - for (int capacity = 128; capacity <= 1024 * 128; capacity *= 2) { - Expr* expr = create_expr(); - EXPECT_TRUE(expr != nullptr); - EXPECT_TRUE(expr->prepare(_runtime_state, *_row_desc).ok()); - int size = 1024 * 1024 / capacity; - VectorizedRowBatch* vec_row_batches[size]; - srand(time(nullptr)); - - for (int i = 0; i < size; ++i) { - vec_row_batches[i] = _object_pool->add(new VectorizedRowBatch( - *_runtime_state->desc_tbl().get_tuple_descriptor(0), capacity)); - MemPool* mem_pool = vec_row_batches[i]->mem_pool(); - int32_t* vec_data = - reinterpret_cast(mem_pool->allocate(sizeof(int32_t) * capacity)); - vec_row_batches[i]->column(0)->set_col_data(vec_data); - - for (int i = 0; i < capacity; ++i) { - vec_data[i] = rand() % 256; - } - - vec_row_batches[i]->set_size(capacity); - } - - RowBatch* row_batches[size]; - - for (int i = 0; i < size; ++i) { - row_batches[i] = _object_pool->add(new RowBatch(*_row_desc, capacity)); - vec_row_batches[i]->to_row_batch(row_batches[i]); - } - - MonotonicStopWatch stopwatch; - stopwatch.start(); - - for (int i = 0; i < size; ++i) { - expr->evaluate(vec_row_batches[i]); - } - - uint64_t vec_time = stopwatch.elapsed_time(); - VLOG_CRITICAL << PrettyPrinter::print(vec_time, TCounterType::TIME_NS); - - stopwatch.start(); - - for (int i = 0; i < size; ++i) { - for (int j = 0; j < capacity; ++j) { - ExecNode::eval_conjuncts(&expr, 1, row_batches[i]->get_row(j)); - } - } - - uint64_t row_time = stopwatch.elapsed_time(); - VLOG_CRITICAL << PrettyPrinter::print(row_time, TCounterType::TIME_NS); - - VLOG_CRITICAL << "capacity: " << capacity << " multiple: " << row_time / vec_time; - } -} - } // namespace doris /* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ diff --git a/be/test/olap/bit_field_test.cpp b/be/test/olap/bit_field_test.cpp deleted file mode 100644 index b3ce669981..0000000000 --- a/be/test/olap/bit_field_test.cpp +++ /dev/null @@ -1,180 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include "olap/byte_buffer.h" -#include "olap/in_stream.h" -#include "olap/out_stream.h" -#include "olap/rowset/bit_field_reader.h" -#include "olap/rowset/bit_field_writer.h" -#include "util/logging.h" - -namespace doris { - -class TestBitField : public testing::Test { -public: - TestBitField() {} - - virtual ~TestBitField() {} - - void SetUp() { - EXPECT_EQ(system("mkdir -p ./ut_dir/"), 0); - EXPECT_EQ(system("rm ./ut_dir/tmp_file"), 0); - _out_stream = new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, nullptr); - EXPECT_TRUE(_out_stream != nullptr); - _writer = new (std::nothrow) BitFieldWriter(_out_stream); - EXPECT_TRUE(_writer != nullptr); - _writer->init(); - } - - void TearDown() { - SAFE_DELETE(_reader); - SAFE_DELETE(_out_stream); - SAFE_DELETE(_writer); - SAFE_DELETE(_shared_buffer); - SAFE_DELETE(_stream); - } - - void CreateReader() { - EXPECT_EQ(Status::OK(), - _helper.open_with_mode(_file_path.c_str(), O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR)); - _out_stream->write_to_file(&_helper, 0); - _helper.close(); - - EXPECT_EQ(Status::OK(), - _helper.open_with_mode(_file_path.c_str(), O_RDONLY, S_IRUSR | S_IWUSR)); - - _shared_buffer = StorageByteBuffer::create(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE + - sizeof(StreamHead)); - EXPECT_TRUE(_shared_buffer != nullptr); - - _stream = new (std::nothrow) - ReadOnlyFileStream(&_helper, &_shared_buffer, 0, _helper.length(), nullptr, - OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, &_stats); - EXPECT_EQ(Status::OK(), _stream->init()); - - _reader = new (std::nothrow) BitFieldReader(_stream); - EXPECT_TRUE(_reader != nullptr); - _reader->init(); - } - - BitFieldReader* _reader; - OutStream* _out_stream; - BitFieldWriter* _writer; - FileHandler _helper; - StorageByteBuffer* _shared_buffer; - ReadOnlyFileStream* _stream; - OlapReaderStatistics _stats; - - std::string _file_path = "./ut_dir/tmp_file"; -}; - -TEST_F(TestBitField, ReadWriteOneBit) { - // write data - EXPECT_EQ(Status::OK(), _writer->write(true)); - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - char value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 1); -} - -TEST_F(TestBitField, ReadWriteMultiBits) { - // write data - for (int32_t i = 0; i < 100; i++) { - if (0 == i % 2) { - EXPECT_EQ(Status::OK(), _writer->write(true)); - } else { - EXPECT_EQ(Status::OK(), _writer->write(false)); - } - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - char value = 0; - for (int32_t i = 0; i < 100; i++) { - EXPECT_EQ(Status::OK(), _reader->next(&value)); - if (0 == i % 2) { - EXPECT_EQ(value, 1); - } else { - EXPECT_EQ(value, 0); - } - } -} - -TEST_F(TestBitField, Seek) { - // write data - for (int32_t i = 0; i < 100; i++) { - if (0 == i % 2) { - EXPECT_EQ(Status::OK(), _writer->write(true)); - } else { - EXPECT_EQ(Status::OK(), _writer->write(false)); - } - } - PositionEntryWriter index_entry; - _writer->get_position(&index_entry); - - EXPECT_EQ(Status::OK(), _writer->write(true)); - - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - char value = 0; - PositionEntryReader entry; - entry._positions = index_entry._positions; - entry._positions_count = index_entry._positions_count; - entry._statistics.init(OLAP_FIELD_TYPE_TINYINT, false); - - PositionProvider position(&entry); - _reader->seek(&position); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 1); -} - -TEST_F(TestBitField, Skip) { - // write data - for (int32_t i = 0; i < 100; i++) { - if (0 == i % 2) { - EXPECT_EQ(Status::OK(), _writer->write(true)); - } else { - EXPECT_EQ(Status::OK(), _writer->write(false)); - } - } - - EXPECT_EQ(Status::OK(), _writer->write(true)); - - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - char value = 0; - _reader->skip(100); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 1); -} - -} // namespace doris diff --git a/be/test/olap/block_column_predicate_test.cpp b/be/test/olap/block_column_predicate_test.cpp index c2b3a6cdce..a027196c0c 100644 --- a/be/test/olap/block_column_predicate_test.cpp +++ b/be/test/olap/block_column_predicate_test.cpp @@ -27,7 +27,6 @@ #include "olap/wrapper_field.h" #include "runtime/mem_pool.h" #include "runtime/string_value.hpp" -#include "runtime/vectorized_row_batch.h" #include "util/logging.h" #include "vec/columns/predicate_column.h" diff --git a/be/test/olap/bloom_filter_column_predicate_test.cpp b/be/test/olap/bloom_filter_column_predicate_test.cpp index ffc100684b..c42ab8d6cd 100644 --- a/be/test/olap/bloom_filter_column_predicate_test.cpp +++ b/be/test/olap/bloom_filter_column_predicate_test.cpp @@ -26,7 +26,6 @@ #include "olap/row_block2.h" #include "runtime/mem_pool.h" #include "runtime/string_value.hpp" -#include "runtime/vectorized_row_batch.h" #include "util/logging.h" #include "vec/columns/column_nullable.h" #include "vec/columns/predicate_column.h" @@ -38,16 +37,12 @@ namespace doris { class TestBloomFilterColumnPredicate : public testing::Test { public: - TestBloomFilterColumnPredicate() : _vectorized_batch(nullptr), _row_block(nullptr) { + TestBloomFilterColumnPredicate() : _row_block(nullptr) { _mem_tracker.reset(new MemTracker(-1)); _mem_pool.reset(new MemPool(_mem_tracker.get())); } - ~TestBloomFilterColumnPredicate() { - if (_vectorized_batch != nullptr) { - delete _vectorized_batch; - } - } + ~TestBloomFilterColumnPredicate() {} void SetTabletSchema(std::string name, const std::string& type, const std::string& aggregation, uint32_t length, bool is_allow_null, bool is_key, @@ -69,12 +64,6 @@ public: tablet_schema->init_from_pb(tablet_schema_pb); } - void InitVectorizedBatch(const TabletSchema* tablet_schema, const std::vector& ids, - int size) { - _vectorized_batch = new VectorizedRowBatch(tablet_schema, ids, size); - _vectorized_batch->set_size(size); - } - void init_row_block(const TabletSchema* tablet_schema, int size) { Schema schema(*tablet_schema); _row_block.reset(new RowBlockV2(schema, size)); @@ -82,7 +71,6 @@ public: std::shared_ptr _mem_tracker; std::unique_ptr _mem_pool; - VectorizedRowBatch* _vectorized_batch; std::unique_ptr _row_block; }; @@ -107,22 +95,7 @@ TEST_F(TestBloomFilterColumnPredicate, FLOAT_COLUMN) { bloom_filter->insert(reinterpret_cast(&value)); ColumnPredicate* pred = BloomFilterColumnPredicateFactory::create_column_predicate( 0, bloom_filter, OLAP_FIELD_TYPE_FLOAT); - - // for VectorizedBatch no null - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); auto* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(float))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - *(col_data + i) = i + 0.1f; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 10); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_FLOAT_EQ(*(col_data + sel[0]), 0.1); - EXPECT_FLOAT_EQ(*(col_data + sel[1]), 1.1); - EXPECT_FLOAT_EQ(*(col_data + sel[2]), 2.1); // for ColumnBlock no null init_row_block(&tablet_schema, size); @@ -139,27 +112,6 @@ TEST_F(TestBloomFilterColumnPredicate, FLOAT_COLUMN) { EXPECT_FLOAT_EQ(*(float*)col_block.cell(_row_block->selection_vector()[1]).cell_ptr(), 5.1); EXPECT_FLOAT_EQ(*(float*)col_block.cell(_row_block->selection_vector()[2]).cell_ptr(), 6.1); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - *(col_data + i) = i + 0.1; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 10); - sel = _vectorized_batch->selected(); - EXPECT_FLOAT_EQ(*(col_data + sel[0]), 0.1); - EXPECT_FLOAT_EQ(*(col_data + sel[1]), 1.1); - EXPECT_FLOAT_EQ(*(col_data + sel[2]), 2.1); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { diff --git a/be/test/olap/bloom_filter_index_test.cpp b/be/test/olap/bloom_filter_index_test.cpp deleted file mode 100644 index 86882676ab..0000000000 --- a/be/test/olap/bloom_filter_index_test.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include - -#include "olap/bloom_filter_reader.h" -#include "olap/bloom_filter_writer.h" -#include "util/logging.h" - -using std::string; - -namespace doris { - -class TestBloomFilterIndex : public testing::Test { -public: - virtual ~TestBloomFilterIndex() {} - - virtual void SetUp() {} - virtual void TearDown() {} -}; - -// Test the normal read and write process -TEST_F(TestBloomFilterIndex, normal_read_and_write) { - string bytes; - BloomFilterIndexReader reader; - BloomFilterIndexWriter writer; - - BloomFilter* bf_0 = new (std::nothrow) BloomFilter(); - bf_0->init(1024); - bytes = "hello"; - bf_0->add_bytes(nullptr, 0); - bf_0->add_bytes(bytes.c_str(), bytes.size()); - writer.add_bloom_filter(bf_0); - - BloomFilter* bf_1 = new (std::nothrow) BloomFilter(); - bf_1->init(1024); - bytes = "doris"; - bf_1->add_bytes(bytes.c_str(), bytes.size()); - writer.add_bloom_filter(bf_1); - - uint64_t expect_size = sizeof(BloomFilterIndexHeader) + bf_0->bit_num() * 2 / 8; - EXPECT_EQ(expect_size, writer.estimate_buffered_memory()); - - char buffer[expect_size]; - memset(buffer, 0, expect_size); - EXPECT_EQ(Status::OK(), writer.write_to_buffer(buffer, expect_size)); - - EXPECT_EQ(Status::OK(), - reader.init(buffer, expect_size, true, bf_0->hash_function_num(), bf_0->bit_num())); - EXPECT_EQ(2, reader.entry_count()); - - bytes = "hello"; - const BloomFilter& bf__0 = reader.entry(0); - EXPECT_TRUE(bf__0.test_bytes(nullptr, 0)); - EXPECT_TRUE(bf__0.test_bytes(bytes.c_str(), bytes.size())); - - bytes = "doris"; - const BloomFilter& bf__1 = reader.entry(1); - EXPECT_TRUE(bf__1.test_bytes(bytes.c_str(), bytes.size())); -} - -// Test abnormal write case -TEST_F(TestBloomFilterIndex, abnormal_write) { - char buffer[24]; - BloomFilterIndexWriter writer; - EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), - writer.write_to_buffer(nullptr)); - EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), - writer.write_to_buffer(nullptr, 0)); - EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), - writer.write_to_buffer(buffer, 0)); - EXPECT_EQ(sizeof(BloomFilterIndexHeader), writer.estimate_buffered_memory()); -} - -// Test abnormal read case -TEST_F(TestBloomFilterIndex, abnormal_read) { - uint32_t bit_num = 64; - uint32_t buffer_size = 24; - uint32_t hash_function_num = 3; - char buffer[buffer_size]; - BloomFilterIndexHeader* header = reinterpret_cast(buffer); - BloomFilterIndexReader reader; - - header->block_count = 1; - EXPECT_EQ(Status::OK(), reader.init(buffer, buffer_size, true, hash_function_num, bit_num)); - - header->block_count = 3; - EXPECT_EQ(Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), - reader.init(buffer, buffer_size, true, hash_function_num, bit_num)); -} - -} // namespace doris diff --git a/be/test/olap/column_reader_test.cpp b/be/test/olap/column_reader_test.cpp deleted file mode 100644 index c3c1001b3e..0000000000 --- a/be/test/olap/column_reader_test.cpp +++ /dev/null @@ -1,2939 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/column_reader.h" - -#include - -#include "olap/byte_buffer.h" -#include "olap/field.h" -#include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/row_block.h" -#include "olap/row_cursor.h" -#include "olap/rowset/column_writer.h" -#include "olap/stream_name.h" -#include "runtime/mem_pool.h" -#include "runtime/string_value.hpp" -#include "runtime/vectorized_row_batch.h" -#include "util/logging.h" - -using std::string; - -namespace doris { - -class TestColumn : public testing::Test { -public: - TestColumn() : _column_writer(nullptr), _column_reader(nullptr), _stream_factory(nullptr) { - _offsets.clear(); - _mem_tracker.reset(new MemTracker(-1)); - _mem_pool.reset(new MemPool(_mem_tracker.get())); - } - - virtual ~TestColumn() { - SAFE_DELETE(_column_writer); - SAFE_DELETE(_column_reader); - SAFE_DELETE(_stream_factory); - } - - virtual void SetUp() { - _offsets.push_back(0); - - _stream_factory = new (std::nothrow) - OutStreamFactory(COMPRESS_LZ4, OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); - EXPECT_TRUE(_stream_factory != nullptr); - config::column_dictionary_key_ratio_threshold = 30; - config::column_dictionary_key_size_threshold = 1000; - } - - virtual void TearDown() { - SAFE_DELETE(_column_writer); - SAFE_DELETE(_column_reader); - SAFE_DELETE(_stream_factory); - SAFE_DELETE(_shared_buffer); - - _offsets.clear(); - for (auto in_stream : _map_in_streams) { - delete in_stream.second; - } - _map_in_streams.clear(); - _present_buffers.clear(); - _data_buffers.clear(); - _second_buffers.clear(); - _dictionary_buffers.clear(); - _length_buffers.clear(); - } - - void create_column_writer(const TabletSchema& tablet_schema) { - _column_writer = ColumnWriter::create(0, tablet_schema, _stream_factory, 1024, - BLOOM_FILTER_DEFAULT_FPP); - - EXPECT_TRUE(_column_writer != nullptr); - EXPECT_EQ(_column_writer->init(), Status::OK()); - } - - void create_column_reader(const TabletSchema& tablet_schema) { - UniqueIdEncodingMap encodings; - encodings[0] = ColumnEncodingMessage(); - encodings[0].set_kind(ColumnEncodingMessage::DIRECT); - encodings[0].set_dictionary_size(1); - create_column_reader(tablet_schema, encodings); - } - - void create_column_reader(const TabletSchema& tablet_schema, UniqueIdEncodingMap& encodings) { - UniqueIdToColumnIdMap included; - included[0] = 0; - UniqueIdToColumnIdMap segment_included; - segment_included[0] = 0; - - _column_reader = - ColumnReader::create(0, tablet_schema, included, segment_included, encodings); - - EXPECT_TRUE(_column_reader != nullptr); - - EXPECT_EQ(system("mkdir -p ./ut_dir"), 0); - EXPECT_EQ(system("rm ./ut_dir/tmp_file"), 0); - - EXPECT_EQ(Status::OK(), - helper.open_with_mode("./ut_dir/tmp_file", O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR)); - std::vector off; - std::vector length; - std::vector buffer_size; - std::vector name; - - std::map::const_iterator it = _stream_factory->streams().begin(); - for (; it != _stream_factory->streams().end(); ++it) { - StreamName stream_name = it->first; - OutStream* out_stream = it->second; - std::vector* buffers = nullptr; - - if (out_stream->is_suppressed()) { - continue; - } - - if (stream_name.kind() == StreamInfoMessage::ROW_INDEX) { - continue; - } else if (stream_name.kind() == StreamInfoMessage::PRESENT) { - buffers = &_present_buffers; - } else if (stream_name.kind() == StreamInfoMessage::DATA) { - buffers = &_data_buffers; - } else if (stream_name.kind() == StreamInfoMessage::SECONDARY) { - buffers = &_second_buffers; - } else if (stream_name.kind() == StreamInfoMessage::DICTIONARY_DATA) { - buffers = &_dictionary_buffers; - } else if (stream_name.kind() == StreamInfoMessage::LENGTH) { - buffers = &_length_buffers; - } else { - EXPECT_TRUE(false); - } - - EXPECT_TRUE(buffers != nullptr); - off.push_back(helper.tell()); - out_stream->write_to_file(&helper, 0); - length.push_back(out_stream->get_stream_length()); - buffer_size.push_back(out_stream->get_total_buffer_size()); - name.push_back(stream_name); - } - helper.close(); - - EXPECT_EQ(Status::OK(), - helper.open_with_mode("./ut_dir/tmp_file", O_RDONLY, S_IRUSR | S_IWUSR)); - - _shared_buffer = StorageByteBuffer::create(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE + - sizeof(StreamHead)); - EXPECT_TRUE(_shared_buffer != nullptr); - - for (int i = 0; i < off.size(); ++i) { - ReadOnlyFileStream* in_stream = new (std::nothrow) - ReadOnlyFileStream(&helper, &_shared_buffer, off[i], length[i], lz4_decompress, - buffer_size[i], &_stats); - EXPECT_EQ(Status::OK(), in_stream->init()); - - _map_in_streams[name[i]] = in_stream; - } - - EXPECT_EQ(_column_reader->init(&_map_in_streams, 1024, _mem_pool.get(), &_stats), - Status::OK()); - } - - void set_tablet_schema_with_one_column(std::string name, std::string type, - std::string aggregation, uint32_t length, - bool is_allow_null, bool is_key, - TabletSchema* tablet_schema) { - TabletSchemaPB tablet_schema_pb; - ColumnPB* column = tablet_schema_pb.add_column(); - column->set_unique_id(0); - column->set_name(name); - column->set_name(name); - column->set_type(type); - column->set_is_key(is_key); - column->set_is_nullable(is_allow_null); - column->set_length(length); - column->set_aggregation(aggregation); - column->set_precision(1000); - column->set_frac(1000); - column->set_is_bf_column(false); - tablet_schema->init_from_pb(tablet_schema_pb); - } - - void create_and_save_last_position() { - EXPECT_EQ(_column_writer->create_row_index_entry(), Status::OK()); - } - - ColumnWriter* _column_writer; - ColumnReader* _column_reader; - std::shared_ptr _mem_tracker; - std::unique_ptr _mem_pool; - std::unique_ptr _col_vector; - - OutStreamFactory* _stream_factory; - - std::vector _offsets; - std::vector _present_buffers; - std::vector _data_buffers; - std::vector _second_buffers; - std::vector _dictionary_buffers; - std::vector _length_buffers; - - StorageByteBuffer* _shared_buffer; - std::map _map_in_streams; - FileHandler helper; - OlapReaderStatistics _stats; -}; - -TEST_F(TestColumn, VectorizedTinyColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("TinyColumn", "TINYINT", "REPLACE", 1, false, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - char value = 1; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(0, write_row); - - value = 3; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(1, write_row); - block.finalize(2); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - - value = *reinterpret_cast(data); - EXPECT_EQ(value, 1); - - data++; - value = *reinterpret_cast(data); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, SeekTinyColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("TinyColumn", "TINYINT", "REPLACE", 1, false, true, - &tablet_schema); - - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - char value = 1; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(0, write_row); - - value = 2; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(1, write_row); - block.finalize(2); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - value = 3; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - PositionEntryReader entry1; - entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; - entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; - entry1._statistics.init(OLAP_FIELD_TYPE_TINYINT, false); - - PositionEntryReader entry2; - entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; - entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; - entry2._statistics.init(OLAP_FIELD_TYPE_TINYINT, false); - - PositionProvider position0(&entry1); - PositionProvider position1(&entry2); - - EXPECT_EQ(_column_reader->seek(&position0), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 1); - data++; - value = *reinterpret_cast(data); - EXPECT_EQ(value, 2); - - EXPECT_EQ(_column_reader->seek(&position1), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, SkipTinyColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("TinyColumn", "TINYINT", "REPLACE", 1, false, true, - &tablet_schema); - - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - char value = 1; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(0, write_row); - - value = 2; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(1, write_row); - - value = 3; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(2, write_row); - block.finalize(3); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - EXPECT_EQ(_column_reader->skip(2), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, VectorizedTinyColumnWithPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("TinyColumn", "TINYINT", "REPLACE", 1, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - - write_row.set_not_null(0); - char value = 3; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(1, write_row); - block.finalize(2); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - - char* data = reinterpret_cast(_col_vector->col_data()); - EXPECT_EQ(is_null[1], false); - value = *reinterpret_cast(data + 1); -} - -TEST_F(TestColumn, TinyColumnIndex) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("TinyColumn", "TINYINT", "REPLACE", 1, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - char value = 1; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(0, write_row); - - value = 3; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(1, write_row); - block.finalize(2); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 1); - - value = *reinterpret_cast(data + 1); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, SeekTinyColumnWithPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("TinyColumn", "TINYINT", "REPLACE", 1, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - char value = 1; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(0, write_row); - - value = 2; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(1, write_row); - block.finalize(2); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - value = 3; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - PositionEntryReader entry1; - entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; - entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; - entry1._statistics.init(OLAP_FIELD_TYPE_TINYINT, false); - - PositionEntryReader entry2; - entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; - entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; - entry2._statistics.init(OLAP_FIELD_TYPE_TINYINT, false); - - PositionProvider position1(&entry1); - PositionProvider position2(&entry2); - - EXPECT_EQ(_column_reader->seek(&position1), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 1); - value = *reinterpret_cast(data + 1); - EXPECT_EQ(value, 2); - - EXPECT_EQ(_column_reader->seek(&position2), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, SkipTinyColumnWithPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("TinyColumn", "TINYINT", "REPLACE", 1, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - char value = 1; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(0, write_row); - - value = 2; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(1, write_row); - - value = 3; - write_row.set_field_content(0, &value, _mem_pool.get()); - block.set_row(2, write_row); - block.finalize(3); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - EXPECT_EQ(_column_reader->skip(2), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, VectorizedShortColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("ShortColumn", "SMALLINT", "REPLACE", 2, false, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - int16_t value = 1; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - - value = 3; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(1, write_row); - block.finalize(2); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 1); - - value = *reinterpret_cast(data + sizeof(int16_t)); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, SeekShortColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("ShortColumn", "SMALLINT", "REPLACE", 2, false, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - int16_t value = 1; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - - value = 2; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(1, write_row); - block.finalize(2); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - value = 3; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - PositionEntryReader entry1; - entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; - entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; - entry1._statistics.init(OLAP_FIELD_TYPE_SMALLINT, false); - - PositionEntryReader entry2; - entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; - entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; - entry2._statistics.init(OLAP_FIELD_TYPE_SMALLINT, false); - - PositionProvider position0(&entry1); - PositionProvider position1(&entry2); - - EXPECT_EQ(_column_reader->seek(&position0), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 1); - - value = *reinterpret_cast(data + sizeof(int16_t)); - EXPECT_EQ(value, 2); - - EXPECT_EQ(_column_reader->seek(&position1), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, SkipShortColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("ShortColumn", "SMALLINT", "REPLACE", 2, false, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - int16_t value = 1; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - - value = 2; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(1, write_row); - - value = 3; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(2, write_row); - block.finalize(3); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - EXPECT_EQ(_column_reader->skip(2), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, SeekShortColumnWithPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("ShortColumn", "SMALLINT", "REPLACE", 2, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - int16_t value = 1; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - - value = 2; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(1, write_row); - block.finalize(2); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - value = 3; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - PositionEntryReader entry1; - entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; - entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; - entry1._statistics.init(OLAP_FIELD_TYPE_SMALLINT, false); - - PositionEntryReader entry2; - entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; - entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; - entry2._statistics.init(OLAP_FIELD_TYPE_SMALLINT, false); - - PositionProvider position0(&entry1); - PositionProvider position1(&entry2); - - EXPECT_EQ(_column_reader->seek(&position0), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 1); - - EXPECT_EQ(_column_reader->seek(&position1), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, VectorizedShortColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("ShortColumn", "SMALLINT", "REPLACE", 2, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - - int16_t value = 3; - write_row.set_not_null(0); - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(1, write_row); - block.finalize(2); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - - char* data = reinterpret_cast(_col_vector->col_data()); - EXPECT_EQ(is_null[1], false); - - value = *reinterpret_cast(data + sizeof(int16_t)); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, SkipShortColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("ShortColumn", "SMALLINT", "REPLACE", 2, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - int16_t value = 1; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - - value = 2; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(1, write_row); - - value = 3; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(2, write_row); - block.finalize(3); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - EXPECT_EQ(_column_reader->skip(2), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, VectorizedIntColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("IntColumn", "INT", "REPLACE", 4, false, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 1024; - block.init(block_info); - - int32_t value = 1; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - - value = 3; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(1, write_row); - block.finalize(2); - - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 1); - - value = *reinterpret_cast(data + sizeof(int)); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, VectorizedIntColumnMassWithoutPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("IntColumn", "INT", "REPLACE", 4, false, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - for (int32_t i = 0; i < 10000; i++) { - write_row.set_field_content(0, reinterpret_cast(&i), _mem_pool.get()); - block.set_row(i, write_row); - } - block.finalize(10000); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - - char* data = nullptr; - for (int32_t i = 0; i < 10000; ++i) { - if (i % 1000 == 0) { - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1000, _mem_pool.get()), - Status::OK()); - data = reinterpret_cast(_col_vector->col_data()); - } - - int32_t value = 0; - value = *reinterpret_cast(data); - EXPECT_EQ(value, i); - data += sizeof(int32_t); - } -} - -TEST_F(TestColumn, VectorizedIntColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("IntColumn", "INT", "REPLACE", 4, true, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - int32_t value = -1; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - write_row.set_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - - bool* is_null = _col_vector->is_null(); - char* data = reinterpret_cast(_col_vector->col_data()); - EXPECT_EQ(is_null[0], false); - value = *reinterpret_cast(data); - EXPECT_EQ(value, -1); - - EXPECT_EQ(is_null[1], true); -} - -TEST_F(TestColumn, VectorizedLongColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("LongColumnWithoutPresent", "BIGINT", "REPLACE", 8, false, - true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - int64_t value = 1; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - value = 3; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_EQ(value, 1); - - value = *reinterpret_cast(data + sizeof(int64_t)); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, VectorizedLongColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("LongColumnWithPresent", "BIGINT", "REPLACE", 8, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - int64_t value = 3; - write_row.set_not_null(0); - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - - char* data = reinterpret_cast(_col_vector->col_data()); - EXPECT_EQ(is_null[1], false); - - value = *reinterpret_cast(data + sizeof(int64_t)); - EXPECT_EQ(value, 3); -} - -TEST_F(TestColumn, VectorizedFloatColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("FloatColumnWithoutPresent", "FLOAT", "REPLACE", 4, false, - true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - float value = 1.234; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - value = 3.234; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_FLOAT_EQ(value, 1.234); - - data += sizeof(float); - value = *reinterpret_cast(data); - EXPECT_FLOAT_EQ(value, 3.234); -} - -TEST_F(TestColumn, VectorizedFloatColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - float value = 3.234; - write_row.set_not_null(0); - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - - EXPECT_EQ(is_null[1], false); - - char* data = reinterpret_cast(_col_vector->col_data()) + sizeof(float); - value = *reinterpret_cast(data); - EXPECT_FLOAT_EQ(value, 3.234); -} - -TEST_F(TestColumn, SeekFloatColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - float value = 1.234; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - value = 3.234; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - PositionEntryReader entry1; - entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; - entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; - entry1._statistics.init(OLAP_FIELD_TYPE_FLOAT, false); - - PositionEntryReader entry2; - entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; - entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; - entry2._statistics.init(OLAP_FIELD_TYPE_FLOAT, false); - - PositionProvider position0(&entry1); - PositionProvider position1(&entry2); - - EXPECT_EQ(_column_reader->seek(&position0), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_FLOAT_EQ(value, 1.234); - - value = *reinterpret_cast(data + sizeof(float)); - EXPECT_FLOAT_EQ(value, 3.234); -} - -TEST_F(TestColumn, SkipFloatColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("FloatColumnWithPresent", "FLOAT", "REPLACE", 4, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - float value = 1.234; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - value = 3.234; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - EXPECT_EQ(_column_reader->skip(1), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_FLOAT_EQ(value, 3.234); -} - -TEST_F(TestColumn, VectorizedDoubleColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DoubleColumnWithoutPresent", "DOUBLE", "REPLACE", 8, false, - true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - double value = 1.23456789; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - value = 3.23456789; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - value = *reinterpret_cast(data); - EXPECT_DOUBLE_EQ(value, 1.23456789); - - data += sizeof(double); - value = *reinterpret_cast(data); - EXPECT_DOUBLE_EQ(value, 3.23456789); -} - -TEST_F(TestColumn, VectorizedDoubleColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DoubleColumnWithPresent", "DOUBLE", "REPLACE", 8, true, true, - &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - double value = 3.23456789; - write_row.set_not_null(0); - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - - char* data = reinterpret_cast(_col_vector->col_data()); - EXPECT_EQ(is_null[1], false); - - data += sizeof(double); - value = *reinterpret_cast(data); - EXPECT_DOUBLE_EQ(value, 3.23456789); -} - -TEST_F(TestColumn, VectorizedDatetimeColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DatetimeColumnWithoutPresent", "DATETIME", "REPLACE", 8, - false, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("2000-10-10 10:10:10"); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), "0&2000-10-10 10:10:10", - strlen("0&2000-10-10 10:10:10")) == 0); -} - -TEST_F(TestColumn, VectorizedDatetimeColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DatetimeColumnWithoutPresent", "DATETIME", "REPLACE", 8, - true, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - std::vector val_string_array; - val_string_array.push_back("2000-10-10 10:10:10"); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - write_row.set_not_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - - char* data = reinterpret_cast(_col_vector->col_data()); - EXPECT_EQ(is_null[1], false); - - data += sizeof(uint64_t); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), "0&2000-10-10 10:10:10", - strlen("0&2000-10-10 10:10:10")) == 0); - - EXPECT_NE(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); -} -TEST_F(TestColumn, VectorizedDatetimeColumnZero) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DatetimeColumnWithoutPresent", "DATETIME", "REPLACE", 8, - true, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - std::vector val_string_array; - val_string_array.push_back("1000-01-01 00:00:00"); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - write_row.set_not_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - - char* data = reinterpret_cast(_col_vector->col_data()); - EXPECT_EQ(is_null[1], false); - - data += sizeof(uint64_t); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), "0&1000-01-01 00:00:00", - strlen("0&1000-01-01 00:00:00")) == 0); - - EXPECT_NE(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); -} - -TEST_F(TestColumn, VectorizedDateColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DateColumnWithoutoutPresent", "DATE", "REPLACE", 3, false, - true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("2000-10-10"); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), "0&2000-10-10", strlen("0&2000-10-10")) == 0); -} - -TEST_F(TestColumn, VectorizedDateColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DateColumnWithoutoutPresent", "DATE", "REPLACE", 3, true, - true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - std::vector val_string_array; - val_string_array.push_back("2000-10-10"); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - for (uint32_t i = 0; i < 100; ++i) { - write_row.set_not_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - } - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 101, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - - char* data = reinterpret_cast(_col_vector->col_data()); - for (uint32_t i = 0; i < 100; ++i) { - data += sizeof(uint24_t); - EXPECT_EQ(is_null[i + 1], false); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), "0&2000-10-10", strlen("0&2000-10-10")) == - 0); - } -} - -TEST_F(TestColumn, VectorizedDecimalColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DecimalColumnWithoutoutPresent", "DECIMAL", "REPLACE", 12, - false, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("1234.5678"); - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("5678.1234"); - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), "0&1234.5678", strlen("0&1234.5678")) == 0); - - data += sizeof(decimal12_t); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), "0&5678.1234", strlen("0&5678.1234")) == 0); -} - -TEST_F(TestColumn, VectorizedDecimalColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DecimalColumnWithoutoutPresent", "DECIMAL", "REPLACE", 12, - true, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - write_row.set_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("5678.1234"); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - write_row.set_not_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - - char* data = reinterpret_cast(_col_vector->col_data()); - data += sizeof(decimal12_t); - EXPECT_EQ(is_null[1], false); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), "0&5678.1234", strlen("0&5678.1234")) == 0); -} - -TEST_F(TestColumn, SkipDecimalColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DecimalColumnWithPresent", "DECIMAL", "REPLACE", 12, true, - true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("1234.5678"); - write_row.from_tuple(val_string_array); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("5678.1234"); - write_row.from_tuple(val_string_array); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - char read_value[20]; - memset(read_value, 0, 20); - EXPECT_EQ(_column_reader->skip(1), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), "0&5678.1234", strlen("0&5678.1234")) == 0); -} - -TEST_F(TestColumn, SeekDecimalColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DecimalColumnWithPresent", "DECIMAL", "REPLACE", 12, true, - true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("1234.5678"); - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - val_string_array.clear(); - val_string_array.push_back("5678.1234"); - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - create_and_save_last_position(); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - PositionEntryReader entry1; - entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; - entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; - entry1._statistics.init(OLAP_FIELD_TYPE_FLOAT, false); - - PositionEntryReader entry2; - entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; - entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; - entry2._statistics.init(OLAP_FIELD_TYPE_FLOAT, false); - - PositionProvider position0(&entry1); - PositionProvider position1(&entry2); - - char read_value[20]; - memset(read_value, 0, 20); - EXPECT_EQ(_column_reader->seek(&position0), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), "0&1234.5678", strlen("0&1234.5678")) == 0); - - memset(read_value, 0, 20); - EXPECT_EQ(_column_reader->seek(&position1), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - data = reinterpret_cast(_col_vector->col_data()); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), "0&5678.1234", strlen("0&5678.1234")) == 0); -} - -TEST_F(TestColumn, VectorizedLargeIntColumnWithoutPresent) { - // init tablet schema - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("LargeIntColumnWithoutoutPresent", "LARGEINT", "SUM", 16, - false, true, &tablet_schema); - // test data - string value1 = "100000000000000000000000000000000000000"; - string value2 = "-170141183460469231731687303715884105728"; - - // write data - - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back(value1); - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back(value2); - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.set_field_content(0, data, _mem_pool.get()); - value1 = "0&" + value1; - value2 = "0&" + value2; - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), value1.c_str(), value1.size()) == 0); - - read_row.set_field_content(0, data + sizeof(int128_t), _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), value2.c_str(), value2.size()) == 0); -} - -TEST_F(TestColumn, VectorizedLargeIntColumnWithPresent) { - // init tablet schema - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("LargeIntColumnWithoutoutPresent", "LARGEINT", "SUM", 16, - true, true, &tablet_schema); - - // test data - string value1 = "100000000000000000000000000000000000000"; - string value2 = "-170141183460469231731687303715884105728"; - - // write data - - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - std::vector val_string_array; - val_string_array.push_back(value1); - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - write_row.set_not_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back(value2); - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - write_row.set_not_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 3, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - EXPECT_EQ(is_null[1], false); - EXPECT_EQ(is_null[2], false); - - char* data = reinterpret_cast(_col_vector->col_data()); - value1 = "0&" + value1; - value2 = "0&" + value2; - - data += sizeof(int128_t); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), value1.c_str(), value1.size()) == 0); - - data += sizeof(int128_t); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), value2.c_str(), value2.size()) == 0); -} - -TEST_F(TestColumn, SkipLargeIntColumnWithPresent) { - // init tablet schema - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("LargeIntColumnWithPresent", "LARGEINT", "SUM", 16, true, - true, &tablet_schema); - // test data - string value1 = "100000000000000000000000000000000000000"; - string value2 = "-170141183460469231731687303715884105728"; - - // write data - - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back(value1); - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back(value2); - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(tablet_schema); - - value2 = "0&" + value2; - EXPECT_EQ(_column_reader->skip(1), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.set_field_content(0, data, _mem_pool.get()); - EXPECT_TRUE(strncmp(read_row.to_string().c_str(), value2.c_str(), value2.size()) == 0); -} - -TEST_F(TestColumn, VectorizedDirectVarcharColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DirectVarcharColumnWithoutoutPresent", "VARCHAR", "REPLACE", - 10, false, true, &tablet_schema); - - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("YWJjZGU="); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - for (uint32_t i = 0; i < 2; i++) { - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - } - val_string_array.clear(); - val_string_array.push_back("ZWRjYmE="); //"edcba" base_64_encode is "ZWRjYmE=" - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - for (uint32_t i = 0; i < 2; i++) { - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - } - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - _col_vector.reset(new ColumnVector()); - - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 5, _mem_pool.get()), Status::OK()); - Slice* value = reinterpret_cast(_col_vector->col_data()); - EXPECT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); - for (uint32_t i = 0; i < 2; i++) { - value++; - EXPECT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); - } - for (uint32_t i = 0; i < 2; i++) { - value++; - EXPECT_TRUE(strncmp(value->data, "ZWRjYmE=", value->size) == 0); - } - EXPECT_NE(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); -} - -TEST_F(TestColumn, VectorizedDirectVarcharColumnWithPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("DirectVarcharColumnWithoutoutPresent", "VARCHAR", "REPLACE", - 10, true, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - std::vector val_string_array; - val_string_array.push_back("YWJjZGU="); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - write_row.set_not_null(0); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - EXPECT_EQ(is_null[1], false); - - Slice* value = reinterpret_cast(_col_vector->col_data()); - value++; - EXPECT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); -} - -TEST_F(TestColumn, SkipDirectVarcharColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DirectVarcharColumnWithPresent", "VARCHAR", "REPLACE", 10, - true, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("YWJjZGU="); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("YWFhYWE="); //"aaaaa" base_64_encode is "YWJjZGU=" - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - char read_value[20]; - memset(read_value, 0, 20); - EXPECT_EQ(_column_reader->skip(1), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - Slice* value = reinterpret_cast(_col_vector->col_data()); - EXPECT_TRUE(strncmp(value->data, "YWFhYWE=", value->size) == 0); -} - -TEST_F(TestColumn, SeekDirectVarcharColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DirectVarcharColumnWithPresent", "VARCHAR", "REPLACE", 10, - false, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("YWJjZGU="); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - _column_writer->create_row_index_entry(); - - val_string_array.clear(); - val_string_array.push_back("YWFhYWE="); //"aaaaa" base_64_encode is "YWJjZGU=" - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - _column_writer->create_row_index_entry(); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - PositionEntryReader entry1; - entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; - entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; - entry1._statistics.init(OLAP_FIELD_TYPE_VARCHAR, false); - - PositionEntryReader entry2; - entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; - entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; - entry2._statistics.init(OLAP_FIELD_TYPE_VARCHAR, false); - - PositionProvider position0(&entry1); - PositionProvider position1(&entry2); - - EXPECT_EQ(_column_reader->seek(&position0), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - Slice* value = reinterpret_cast(_col_vector->col_data()); - EXPECT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); - - EXPECT_EQ(_column_reader->seek(&position1), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - value = reinterpret_cast(_col_vector->col_data()); - EXPECT_TRUE(strncmp(value->data, "YWFhYWE=", value->size) == 0); -} - -TEST_F(TestColumn, SeekDirectVarcharColumnWithPresent) { - // write data - TabletSchema tablet_schema; - - set_tablet_schema_with_one_column("DirectVarcharColumnWithPresent", "VARCHAR", "REPLACE", 10, - true, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("YWJjZGU="); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - _column_writer->create_row_index_entry(); - - val_string_array.clear(); - val_string_array.push_back("YWFhYWE="); //"aaaaa" base_64_encode is "YWJjZGU=" - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - _column_writer->create_row_index_entry(); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - PositionEntryReader entry1; - entry1._positions = _column_writer->index()->mutable_entry(0)->_positions; - entry1._positions_count = _column_writer->index()->mutable_entry(0)->_positions_count; - entry1._statistics.init(OLAP_FIELD_TYPE_VARCHAR, false); - - PositionEntryReader entry2; - entry2._positions = _column_writer->index()->mutable_entry(1)->_positions; - entry2._positions_count = _column_writer->index()->mutable_entry(1)->_positions_count; - entry2._statistics.init(OLAP_FIELD_TYPE_VARCHAR, false); - - PositionProvider position0(&entry1); - PositionProvider position1(&entry2); - - EXPECT_EQ(_column_reader->seek(&position0), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - Slice* value = reinterpret_cast(_col_vector->col_data()); - EXPECT_TRUE(strncmp(value->data, "YWJjZGU=", value->size) == 0); - - EXPECT_EQ(_column_reader->seek(&position1), Status::OK()); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - value = reinterpret_cast(_col_vector->col_data()); - EXPECT_TRUE(strncmp(value->data, "YWFhYWE=", value->size) == 0); -} - -TEST_F(TestColumn, VectorizedCharColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("CharColumnWithoutoutPresent", "CHAR", "REPLACE", - strlen("abcde"), false, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("abcde"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - for (uint32_t i = 0; i < 2; i++) { - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - } - val_string_array.clear(); - val_string_array.push_back("edcba"); //"edcba" base_64_encode is "ZWRjYmE=" - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - for (uint32_t i = 0; i < 2; i++) { - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - } - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 5, _mem_pool.get()), Status::OK()); - Slice* value = reinterpret_cast(_col_vector->col_data()); - - EXPECT_TRUE(strncmp(value->data, "abcde", value->size) == 0); - for (uint32_t i = 0; i < 2; i++) { - value++; - EXPECT_TRUE(strncmp(value->data, "abcde", value->size) == 0); - } - for (uint32_t i = 0; i < 2; i++) { - value++; - EXPECT_TRUE(strncmp(value->data, "edcba", value->size) == 0); - } - EXPECT_NE(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); -} - -TEST_F(TestColumn, VectorizedCharColumnWithPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("CharColumnWithoutoutPresent", "CHAR", "REPLACE", - strlen("abcde"), true, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - - std::vector val_string_array; - val_string_array.push_back("abcde"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - write_row.set_not_null(0); - block.set_row(1, write_row); - block.finalize(2); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - EXPECT_EQ(is_null[1], false); - - Slice* value = reinterpret_cast(_col_vector->col_data()); - value++; - EXPECT_TRUE(strncmp(value->data, "abcde", value->size) == 0); -} - -TEST_F(TestColumn, VectorizedCharColumnWithoutoutPresent2) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("CharColumnWithoutoutPresent", "CHAR", "REPLACE", 20, false, - true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("abcde"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("aaaaa"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("bbbbb"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple3(val_string_array); - write_row.from_tuple(tuple3); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("ccccc"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple4(val_string_array); - write_row.from_tuple(tuple4); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("ddddd"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple5(val_string_array); - write_row.from_tuple(tuple5); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 5, _mem_pool.get()), Status::OK()); - Slice* value = reinterpret_cast(_col_vector->col_data()); - - EXPECT_TRUE(strncmp(value->data, "abcde", value->size) == 0); - - value++; - EXPECT_TRUE(strncmp(value->data, "aaaaa", value->size) == 0); - - value++; - EXPECT_TRUE(strncmp(value->data, "bbbbb", value->size) == 0); - - value++; - EXPECT_TRUE(strncmp(value->data, "ccccc", value->size) == 0); - - value++; - EXPECT_TRUE(strncmp(value->data, "ddddd", value->size) == 0); -} - -TEST_F(TestColumn, VectorizedStringColumnWithoutPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("StringColumnWithoutoutPresent", "STRING", "REPLACE", 0, - false, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("abcde"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - for (uint32_t i = 0; i < 2; i++) { - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - } - val_string_array.clear(); - val_string_array.push_back("edcba"); //"edcba" base_64_encode is "ZWRjYmE=" - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - for (uint32_t i = 0; i < 2; i++) { - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - } - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 5, _mem_pool.get()), Status::OK()); - Slice* value = reinterpret_cast(_col_vector->col_data()); - - EXPECT_TRUE(strncmp(value->data, "abcde", value->size) == 0); - for (uint32_t i = 0; i < 2; i++) { - value++; - EXPECT_TRUE(strncmp(value->data, "abcde", value->size) == 0); - } - for (uint32_t i = 0; i < 2; i++) { - value++; - EXPECT_TRUE(strncmp(value->data, "edcba", value->size) == 0); - } - EXPECT_NE(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); -} - -TEST_F(TestColumn, VectorizedStringColumnWithPresent) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("StringColumnWithoutoutPresent", "STRING", "REPLACE", 0, true, - true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - write_row.set_null(0); - block.set_row(0, write_row); - - std::vector val_string_array; - val_string_array.push_back("abcde"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - write_row.set_not_null(0); - block.set_row(1, write_row); - block.finalize(2); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - bool* is_null = _col_vector->is_null(); - EXPECT_EQ(is_null[0], true); - EXPECT_EQ(is_null[1], false); - - Slice* value = reinterpret_cast(_col_vector->col_data()); - value++; - EXPECT_TRUE(strncmp(value->data, "abcde", value->size) == 0); -} - -TEST_F(TestColumn, VectorizedStringColumnWithoutoutPresent2) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("StringColumnWithoutoutPresent", "STRING", "REPLACE", 0, - false, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back("abcde"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple1(val_string_array); - write_row.from_tuple(tuple1); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("aaaaa"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("bbbbb"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple3(val_string_array); - write_row.from_tuple(tuple3); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("ccccc"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple4(val_string_array); - write_row.from_tuple(tuple4); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("ddddd"); //"abcde" base_64_encode is "YWJjZGU=" - OlapTuple tuple5(val_string_array); - write_row.from_tuple(tuple5); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 5, _mem_pool.get()), Status::OK()); - Slice* value = reinterpret_cast(_col_vector->col_data()); - - EXPECT_TRUE(strncmp(value->data, "abcde", value->size) == 0); - - value++; - EXPECT_TRUE(strncmp(value->data, "aaaaa", value->size) == 0); - - value++; - EXPECT_TRUE(strncmp(value->data, "bbbbb", value->size) == 0); - - value++; - EXPECT_TRUE(strncmp(value->data, "ccccc", value->size) == 0); - - value++; - EXPECT_TRUE(strncmp(value->data, "ddddd", value->size) == 0); -} - -TEST_F(TestColumn, VectorizedDirectVarcharColumnWith65533) { - // write data - TabletSchema tablet_schema; - set_tablet_schema_with_one_column("DirectVarcharColumnWithoutoutPresent", "VARCHAR", "REPLACE", - 65535, false, true, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.push_back(std::string(65533, 'a')); - OlapTuple tuple1(val_string_array); - EXPECT_EQ(Status::OK(), write_row.from_tuple(tuple1)); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - val_string_array.clear(); - val_string_array.push_back("edcba"); //"edcba" base_64_encode is "ZWRjYmE=" - OlapTuple tuple2(val_string_array); - write_row.from_tuple(tuple2); - for (uint32_t i = 0; i < 2; i++) { - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - } - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(tablet_schema); - read_row.allocate_memory_for_string_type(tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 3, _mem_pool.get()), Status::OK()); - Slice* value = reinterpret_cast(_col_vector->col_data()); - - for (uint32_t i = 0; i < 65533; i++) { - EXPECT_TRUE(strncmp(value->data + i, "a", 1) == 0); - } - - for (uint32_t i = 0; i < 2; i++) { - value++; - EXPECT_TRUE(strncmp(value->data, "edcba", value->size) == 0); - } -} - -} // namespace doris diff --git a/be/test/olap/comparison_predicate_test.cpp b/be/test/olap/comparison_predicate_test.cpp index 19fff88c60..23538dd6fc 100644 --- a/be/test/olap/comparison_predicate_test.cpp +++ b/be/test/olap/comparison_predicate_test.cpp @@ -27,7 +27,6 @@ #include "olap/wrapper_field.h" #include "runtime/mem_pool.h" #include "runtime/string_value.hpp" -#include "runtime/vectorized_row_batch.h" #include "util/logging.h" namespace doris { @@ -89,15 +88,11 @@ static std::string to_datetime_string(uint64_t& datetime_value) { #define TEST_PREDICATE_DEFINITION(CLASS_NAME) \ class CLASS_NAME : public testing::Test { \ public: \ - CLASS_NAME() : _vectorized_batch(nullptr) { \ + CLASS_NAME() { \ _mem_tracker.reset(new MemTracker(-1)); \ _mem_pool.reset(new MemPool(_mem_tracker.get())); \ } \ - ~CLASS_NAME() { \ - if (_vectorized_batch != nullptr) { \ - delete _vectorized_batch; \ - } \ - } \ + ~CLASS_NAME() {} \ void SetTabletSchema(std::string name, const std::string& type, \ const std::string& aggregation, uint32_t length, bool is_allow_null, \ bool is_key, TabletSchema* tablet_schema) { \ @@ -115,11 +110,6 @@ static std::string to_datetime_string(uint64_t& datetime_value) { column->set_frac(1000); \ column->set_is_bf_column(false); \ tablet_schema->init_from_pb(tablet_schema_pb); \ - } \ - void InitVectorizedBatch(const TabletSchema* tablet_schema, \ - const std::vector& ids, int size) { \ - _vectorized_batch = new VectorizedRowBatch(tablet_schema, ids, size); \ - _vectorized_batch->set_size(size); \ } \ \ void init_row_block(const TabletSchema* tablet_schema, int size) { \ @@ -128,67 +118,12 @@ static std::string to_datetime_string(uint64_t& datetime_value) { } \ std::shared_ptr _mem_tracker; \ std::unique_ptr _mem_pool; \ - VectorizedRowBatch* _vectorized_batch; \ std::unique_ptr _row_block; \ }; TEST_PREDICATE_DEFINITION(TestEqualPredicate) TEST_PREDICATE_DEFINITION(TestLessPredicate) -#define TEST_EQUAL_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ - TEST_F(TestEqualPredicate, TYPE_NAME##_COLUMN) { \ - TabletSchema tablet_schema; \ - SetTabletSchema(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, "REPLACE", 1, false, true, \ - &tablet_schema); \ - int size = 10; \ - std::vector return_columns; \ - for (int i = 0; i < tablet_schema.num_columns(); ++i) { \ - return_columns.push_back(i); \ - } \ - InitVectorizedBatch(&tablet_schema, return_columns, size); \ - ColumnVector* col_vector = _vectorized_batch->column(0); \ - \ - /* for no nulls */ \ - col_vector->set_no_nulls(true); \ - TYPE* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(TYPE))); \ - col_vector->set_col_data(col_data); \ - for (int i = 0; i < size; ++i) { \ - *(col_data + i) = i; \ - } \ - TYPE value = 5; \ - ColumnPredicate* pred = new EqualPredicate(0, value); \ - pred->evaluate(_vectorized_batch); \ - EXPECT_EQ(_vectorized_batch->size(), 1); \ - uint16_t* sel = _vectorized_batch->selected(); \ - EXPECT_EQ(*(col_data + sel[0]), 5); \ - \ - /* for has nulls */ \ - col_vector->set_no_nulls(false); \ - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); \ - memset(is_null, 0, size); \ - col_vector->set_is_null(is_null); \ - for (int i = 0; i < size; ++i) { \ - if (i % 2 == 0) { \ - is_null[i] = true; \ - } else { \ - *(col_data + i) = i; \ - } \ - } \ - _vectorized_batch->set_size(size); \ - _vectorized_batch->set_selected_in_use(false); \ - pred->evaluate(_vectorized_batch); \ - EXPECT_EQ(_vectorized_batch->size(), 1); \ - sel = _vectorized_batch->selected(); \ - EXPECT_EQ(*(col_data + sel[0]), 5); \ - delete pred; \ - } - -TEST_EQUAL_PREDICATE(int8_t, TINYINT, "TINYINT") -TEST_EQUAL_PREDICATE(int16_t, SMALLINT, "SMALLINT") -TEST_EQUAL_PREDICATE(int32_t, INT, "INT") -TEST_EQUAL_PREDICATE(int64_t, BIGINT, "BIGINT") -TEST_EQUAL_PREDICATE(int128_t, LARGEINT, "LARGEINT") - TEST_F(TestEqualPredicate, FLOAT_COLUMN) { TabletSchema tablet_schema; SetTabletSchema(std::string("FLOAT_COLUMN"), "FLOAT", "REPLACE", 1, true, true, &tablet_schema); @@ -200,20 +135,6 @@ TEST_F(TestEqualPredicate, FLOAT_COLUMN) { float value = 5.0; ColumnPredicate* pred = new EqualPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - float* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(float))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - *(col_data + i) = i; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_FLOAT_EQ(*(col_data + sel[0]), 5.0); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -227,25 +148,6 @@ TEST_F(TestEqualPredicate, FLOAT_COLUMN) { EXPECT_EQ(select_size, 1); EXPECT_FLOAT_EQ(*(float*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr(), 5.0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - *(col_data + i) = i; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_FLOAT_EQ(*(col_data + sel[0]), 5.0); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -277,20 +179,6 @@ TEST_F(TestEqualPredicate, DOUBLE_COLUMN) { double value = 5.0; ColumnPredicate* pred = new EqualPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - double* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(double))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - *(col_data + i) = i; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_DOUBLE_EQ(*(col_data + sel[0]), 5.0); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -304,25 +192,6 @@ TEST_F(TestEqualPredicate, DOUBLE_COLUMN) { EXPECT_EQ(select_size, 1); EXPECT_DOUBLE_EQ(*(double*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr(), 5.0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - *(col_data + i) = i; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_DOUBLE_EQ(*(col_data + sel[0]), 5.0); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -354,22 +223,6 @@ TEST_F(TestEqualPredicate, DECIMAL_COLUMN) { decimal12_t value = {5, 5}; ColumnPredicate* pred = new EqualPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - decimal12_t* col_data = - reinterpret_cast(_mem_pool->allocate(size * sizeof(decimal12_t))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - (*(col_data + i)).integer = i; - (*(col_data + i)).fraction = i; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(*(col_data + sel[0]), value); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -384,26 +237,6 @@ TEST_F(TestEqualPredicate, DECIMAL_COLUMN) { EXPECT_EQ(select_size, 1); EXPECT_EQ(*(decimal12_t*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr(), value); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - (*(col_data + i)).integer = i; - (*(col_data + i)).fraction = i; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(*(col_data + sel[0]), value); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -455,34 +288,12 @@ TEST_F(TestEqualPredicate, STRING_COLUMN) { ColumnPredicate* pred = new EqualPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - StringValue* col_data = - reinterpret_cast(_mem_pool->allocate(size * sizeof(StringValue))); - col_vector->set_col_data(col_data); - char* string_buffer = reinterpret_cast(_mem_pool->allocate(55)); - for (int i = 0; i < size; ++i) { - for (int j = 0; j <= i; ++j) { - string_buffer[j] = 'a' + i; - } - (*(col_data + i)).len = i + 1; - (*(col_data + i)).ptr = string_buffer; - string_buffer += i + 1; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(sel[0], 3); - EXPECT_EQ(*(col_data + sel[0]), value); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); auto select_size = _row_block->selected_size(); ColumnBlockView col_block_view(&col_block); - string_buffer = reinterpret_cast(_mem_pool->allocate(60)); + char* string_buffer = reinterpret_cast(_mem_pool->allocate(60)); memset(string_buffer, 0, 60); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { col_block_view.set_null_bits(1, false); @@ -497,31 +308,6 @@ TEST_F(TestEqualPredicate, STRING_COLUMN) { EXPECT_EQ(select_size, 1); EXPECT_EQ(*(StringValue*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr(), value); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - string_buffer = reinterpret_cast(_mem_pool->allocate(55)); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - for (int j = 0; j <= i; ++j) { - string_buffer[j] = 'a' + i; - } - (*(col_data + i)).len = i + 1; - (*(col_data + i)).ptr = string_buffer; - } - string_buffer += i + 1; - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(*(col_data + sel[0]), value); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); string_buffer = reinterpret_cast(_mem_pool->allocate(55)); @@ -559,13 +345,6 @@ TEST_F(TestEqualPredicate, DATE_COLUMN) { uint24_t value = datetime::to_date_timestamp("2017-09-10"); ColumnPredicate* pred = new EqualPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - uint24_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint24_t))); - col_vector->set_col_data(col_data); - std::vector date_array; date_array.push_back("2017-09-07"); date_array.push_back("2017-09-08"); @@ -573,16 +352,6 @@ TEST_F(TestEqualPredicate, DATE_COLUMN) { date_array.push_back("2017-09-10"); date_array.push_back("2017-09-11"); date_array.push_back("2017-09-12"); - for (int i = 0; i < size; ++i) { - uint24_t timestamp = datetime::to_date_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(sel[0], 3); - EXPECT_EQ(*(col_data + sel[0]), value); - EXPECT_EQ(datetime::to_date_string(*(col_data + sel[0])), "2017-09-10"); // for ColumnBlock no nulls init_row_block(&tablet_schema, size); @@ -600,27 +369,6 @@ TEST_F(TestEqualPredicate, DATE_COLUMN) { *(uint24_t*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr()), "2017-09-10"); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - uint24_t timestamp = datetime::to_date_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(*(col_data + sel[0]), value); - EXPECT_EQ(datetime::to_date_string(*(col_data + sel[0])), "2017-09-10"); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -655,13 +403,6 @@ TEST_F(TestEqualPredicate, DATETIME_COLUMN) { uint64_t value = datetime::to_datetime_timestamp("2017-09-10 01:00:00"); ColumnPredicate* pred = new EqualPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - uint64_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint64_t))); - col_vector->set_col_data(col_data); - std::vector date_array; date_array.push_back("2017-09-07 00:00:00"); date_array.push_back("2017-09-08 00:01:00"); @@ -669,16 +410,6 @@ TEST_F(TestEqualPredicate, DATETIME_COLUMN) { date_array.push_back("2017-09-10 01:00:00"); date_array.push_back("2017-09-11 01:01:00"); date_array.push_back("2017-09-12 01:01:01"); - for (int i = 0; i < size; ++i) { - uint64_t timestamp = datetime::to_datetime_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(sel[0], 3); - EXPECT_EQ(*(col_data + sel[0]), value); - EXPECT_EQ(datetime::to_datetime_string(*(col_data + sel[0])), "2017-09-10 01:00:00"); // for ColumnBlock no nulls init_row_block(&tablet_schema, size); @@ -696,27 +427,6 @@ TEST_F(TestEqualPredicate, DATETIME_COLUMN) { *(uint64_t*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr()), "2017-09-10 01:00:00"); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - uint64_t timestamp = datetime::to_datetime_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(*(col_data + sel[0]), value); - EXPECT_EQ(datetime::to_datetime_string(*(col_data + sel[0])), "2017-09-10 01:00:00"); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -739,68 +449,6 @@ TEST_F(TestEqualPredicate, DATETIME_COLUMN) { delete pred; } -#define TEST_LESS_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ - TEST_F(TestLessPredicate, TYPE_NAME##_COLUMN) { \ - TabletSchema tablet_schema; \ - SetTabletSchema(std::string("TYPE_NAME_COLUMN"), FIELD_TYPE, "REPLACE", 1, false, true, \ - &tablet_schema); \ - int size = 10; \ - std::vector return_columns; \ - for (int i = 0; i < tablet_schema.num_columns(); ++i) { \ - return_columns.push_back(i); \ - } \ - InitVectorizedBatch(&tablet_schema, return_columns, size); \ - ColumnVector* col_vector = _vectorized_batch->column(0); \ - \ - /* for no nulls */ \ - col_vector->set_no_nulls(true); \ - TYPE* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(TYPE))); \ - col_vector->set_col_data(col_data); \ - for (int i = 0; i < size; ++i) { \ - *(col_data + i) = i; \ - } \ - TYPE value = 5; \ - ColumnPredicate* pred = new LessPredicate(0, value); \ - pred->evaluate(_vectorized_batch); \ - EXPECT_EQ(_vectorized_batch->size(), 5); \ - uint16_t* sel = _vectorized_batch->selected(); \ - TYPE sum = 0; \ - for (int i = 0; i < _vectorized_batch->size(); ++i) { \ - sum += *(col_data + sel[i]); \ - } \ - EXPECT_EQ(sum, 10); \ - \ - /* for has nulls */ \ - col_vector->set_no_nulls(false); \ - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); \ - memset(is_null, 0, size); \ - col_vector->set_is_null(is_null); \ - for (int i = 0; i < size; ++i) { \ - if (i % 2 == 0) { \ - is_null[i] = true; \ - } else { \ - *(col_data + i) = i; \ - } \ - } \ - _vectorized_batch->set_size(size); \ - _vectorized_batch->set_selected_in_use(false); \ - pred->evaluate(_vectorized_batch); \ - EXPECT_EQ(_vectorized_batch->size(), 2); \ - sel = _vectorized_batch->selected(); \ - sum = 0; \ - for (int i = 0; i < _vectorized_batch->size(); ++i) { \ - sum += *(col_data + sel[i]); \ - } \ - EXPECT_EQ(sum, 4); \ - delete pred; \ - } - -TEST_LESS_PREDICATE(int8_t, TINYINT, "TINYINT") -TEST_LESS_PREDICATE(int16_t, SMALLINT, "SMALLINT") -TEST_LESS_PREDICATE(int32_t, INT, "INT") -TEST_LESS_PREDICATE(int64_t, BIGINT, "BIGINT") -TEST_LESS_PREDICATE(int128_t, LARGEINT, "LARGEINT") - TEST_F(TestLessPredicate, FLOAT_COLUMN) { TabletSchema tablet_schema; SetTabletSchema(std::string("FLOAT_COLUMN"), "FLOAT", "REPLACE", 1, true, true, &tablet_schema); @@ -812,24 +460,6 @@ TEST_F(TestLessPredicate, FLOAT_COLUMN) { float value = 5.0; ColumnPredicate* pred = new LessPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - float* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(float))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - *(col_data + i) = i; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 5); - uint16_t* sel = _vectorized_batch->selected(); - float sum = 0; - for (int i = 0; i < _vectorized_batch->size(); ++i) { - sum += *(col_data + sel[i]); - } - EXPECT_FLOAT_EQ(sum, 10.0); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -841,35 +471,12 @@ TEST_F(TestLessPredicate, FLOAT_COLUMN) { } pred->evaluate(&col_block, _row_block->selection_vector(), &select_size); EXPECT_EQ(select_size, 5); - sum = 0; + float sum = 0; for (int i = 0; i < 5; ++i) { sum += *(float*)col_block.cell(_row_block->selection_vector()[i]).cell_ptr(); } EXPECT_FLOAT_EQ(sum, 10.0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - *(col_data + i) = i; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 2); - sel = _vectorized_batch->selected(); - sum = 0; - for (int i = 0; i < _vectorized_batch->size(); ++i) { - sum += *(col_data + sel[i]); - } - EXPECT_FLOAT_EQ(sum, 4.0); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -905,24 +512,6 @@ TEST_F(TestLessPredicate, DOUBLE_COLUMN) { double value = 5.0; ColumnPredicate* pred = new LessPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - double* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(double))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - *(col_data + i) = i; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 5); - uint16_t* sel = _vectorized_batch->selected(); - double sum = 0; - for (int i = 0; i < _vectorized_batch->size(); ++i) { - sum += *(col_data + sel[i]); - } - EXPECT_DOUBLE_EQ(sum, 10.0); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -934,35 +523,12 @@ TEST_F(TestLessPredicate, DOUBLE_COLUMN) { } pred->evaluate(&col_block, _row_block->selection_vector(), &select_size); EXPECT_EQ(select_size, 5); - sum = 0; + double sum = 0; for (int i = 0; i < 5; ++i) { sum += *(double*)col_block.cell(_row_block->selection_vector()[i]).cell_ptr(); } EXPECT_DOUBLE_EQ(sum, 10.0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - *(col_data + i) = i; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 2); - sel = _vectorized_batch->selected(); - sum = 0; - for (int i = 0; i < _vectorized_batch->size(); ++i) { - sum += *(col_data + sel[i]); - } - EXPECT_DOUBLE_EQ(sum, 4.0); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -998,27 +564,6 @@ TEST_F(TestLessPredicate, DECIMAL_COLUMN) { decimal12_t value = {5, 5}; ColumnPredicate* pred = new LessPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - decimal12_t* col_data = - reinterpret_cast(_mem_pool->allocate(size * sizeof(decimal12_t))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - (*(col_data + i)).integer = i; - (*(col_data + i)).fraction = i; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 5); - uint16_t* sel = _vectorized_batch->selected(); - decimal12_t sum = {0, 0}; - for (int i = 0; i < _vectorized_batch->size(); ++i) { - sum += *(col_data + sel[i]); - } - EXPECT_EQ(sum.integer, 10); - EXPECT_EQ(sum.fraction, 10); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -1031,38 +576,6 @@ TEST_F(TestLessPredicate, DECIMAL_COLUMN) { } pred->evaluate(&col_block, _row_block->selection_vector(), &select_size); EXPECT_EQ(select_size, 5); - sum.integer = 0; - sum.fraction = 0; - for (int i = 0; i < _vectorized_batch->size(); ++i) { - sum += *(col_data + sel[i]); - } - EXPECT_EQ(sum.integer, 10); - EXPECT_EQ(sum.fraction, 10); - - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - (*(col_data + i)).integer = i; - (*(col_data + i)).fraction = i; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 2); - sum.integer = 0; - sum.fraction = 0; - for (int i = 0; i < _vectorized_batch->size(); ++i) { - sum += *(col_data + sel[i]); - } - EXPECT_EQ(sum.integer, 4); - EXPECT_EQ(sum.fraction, 4); // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); @@ -1079,13 +592,6 @@ TEST_F(TestLessPredicate, DECIMAL_COLUMN) { select_size = _row_block->selected_size(); pred->evaluate(&col_block, _row_block->selection_vector(), &select_size); EXPECT_EQ(select_size, 2); - sum.integer = 0; - sum.fraction = 0; - for (int i = 0; i < _vectorized_batch->size(); ++i) { - sum += *(col_data + sel[i]); - } - EXPECT_EQ(sum.integer, 4); - EXPECT_EQ(sum.fraction, 4); delete pred; } @@ -1106,33 +612,12 @@ TEST_F(TestLessPredicate, STRING_COLUMN) { value.ptr = const_cast(value_buffer); ColumnPredicate* pred = new LessPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - StringValue* col_data = - reinterpret_cast(_mem_pool->allocate(size * sizeof(StringValue))); - col_vector->set_col_data(col_data); - char* string_buffer = reinterpret_cast(_mem_pool->allocate(55)); - for (int i = 0; i < size; ++i) { - for (int j = 0; j <= i; ++j) { - string_buffer[j] = 'a' + i; - } - (*(col_data + i)).len = i + 1; - (*(col_data + i)).ptr = string_buffer; - string_buffer += i + 1; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 3); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_TRUE(strncmp((*(col_data + sel[0])).ptr, "a", 1) == 0); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); auto select_size = _row_block->selected_size(); ColumnBlockView col_block_view(&col_block); - string_buffer = reinterpret_cast(_mem_pool->allocate(60)); + char* string_buffer = reinterpret_cast(_mem_pool->allocate(60)); memset(string_buffer, 0, 60); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { col_block_view.set_null_bits(1, false); @@ -1150,31 +635,6 @@ TEST_F(TestLessPredicate, STRING_COLUMN) { .ptr, "a", 1) == 0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - string_buffer = reinterpret_cast(_mem_pool->allocate(55)); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - for (int j = 0; j <= i; ++j) { - string_buffer[j] = 'a' + i; - } - (*(col_data + i)).len = i + 1; - (*(col_data + i)).ptr = string_buffer; - } - string_buffer += i + 1; - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_TRUE(strncmp((*(col_data + sel[0])).ptr, "bb", 2) == 0); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); string_buffer = reinterpret_cast(_mem_pool->allocate(55)); @@ -1214,13 +674,6 @@ TEST_F(TestLessPredicate, DATE_COLUMN) { uint24_t value = datetime::to_date_timestamp("2017-09-10"); ColumnPredicate* pred = new LessPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - uint24_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint24_t))); - col_vector->set_col_data(col_data); - std::vector date_array; date_array.push_back("2017-09-07"); date_array.push_back("2017-09-08"); @@ -1228,14 +681,6 @@ TEST_F(TestLessPredicate, DATE_COLUMN) { date_array.push_back("2017-09-10"); date_array.push_back("2017-09-11"); date_array.push_back("2017-09-12"); - for (int i = 0; i < size; ++i) { - uint24_t timestamp = datetime::to_date_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 3); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(datetime::to_date_string(*(col_data + sel[0])), "2017-09-07"); // for ColumnBlock no nulls init_row_block(&tablet_schema, size); @@ -1253,26 +698,6 @@ TEST_F(TestLessPredicate, DATE_COLUMN) { *(uint24_t*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr()), "2017-09-07"); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - uint24_t timestamp = datetime::to_date_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(datetime::to_date_string(*(col_data + sel[0])), "2017-09-08"); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -1309,13 +734,6 @@ TEST_F(TestLessPredicate, DATETIME_COLUMN) { uint64_t value = datetime::to_datetime_timestamp("2017-09-10 01:00:00"); ColumnPredicate* pred = new LessPredicate(0, value); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - uint64_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint64_t))); - col_vector->set_col_data(col_data); - std::vector date_array; date_array.push_back("2017-09-07 00:00:00"); date_array.push_back("2017-09-08 00:01:00"); @@ -1323,14 +741,6 @@ TEST_F(TestLessPredicate, DATETIME_COLUMN) { date_array.push_back("2017-09-10 01:00:00"); date_array.push_back("2017-09-11 01:01:00"); date_array.push_back("2017-09-12 01:01:01"); - for (int i = 0; i < size; ++i) { - uint64_t timestamp = datetime::to_datetime_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 3); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(datetime::to_datetime_string(*(col_data + sel[0])), "2017-09-07 00:00:00"); // for ColumnBlock no nulls init_row_block(&tablet_schema, size); @@ -1348,26 +758,6 @@ TEST_F(TestLessPredicate, DATETIME_COLUMN) { *(uint64_t*)col_block.cell(_row_block->selection_vector()[0]).cell_ptr()), "2017-09-07 00:00:00"); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - uint64_t timestamp = datetime::to_datetime_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(datetime::to_datetime_string(*(col_data + sel[0])), "2017-09-08 00:01:00"); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { diff --git a/be/test/olap/in_list_predicate_test.cpp b/be/test/olap/in_list_predicate_test.cpp index 00a2280614..29cb35d51c 100644 --- a/be/test/olap/in_list_predicate_test.cpp +++ b/be/test/olap/in_list_predicate_test.cpp @@ -26,7 +26,6 @@ #include "olap/row_block2.h" #include "runtime/mem_pool.h" #include "runtime/string_value.hpp" -#include "runtime/vectorized_row_batch.h" #include "util/logging.h" namespace doris { @@ -108,16 +107,12 @@ static std::string to_datetime_string(uint64_t& datetime_value) { class TestInListPredicate : public testing::Test { public: - TestInListPredicate() : _vectorized_batch(nullptr), _row_block(nullptr) { + TestInListPredicate() : _row_block(nullptr) { _mem_tracker.reset(new MemTracker(-1)); _mem_pool.reset(new MemPool(_mem_tracker.get())); } - ~TestInListPredicate() { - if (_vectorized_batch != nullptr) { - delete _vectorized_batch; - } - } + ~TestInListPredicate() {} void SetTabletSchema(std::string name, const std::string& type, const std::string& aggregation, uint32_t length, bool is_allow_null, bool is_key, @@ -139,12 +134,6 @@ public: tablet_schema->init_from_pb(tablet_schema_pb); } - void InitVectorizedBatch(const TabletSchema* tablet_schema, const std::vector& ids, - int size) { - _vectorized_batch = new VectorizedRowBatch(tablet_schema, ids, size); - _vectorized_batch->set_size(size); - } - void init_row_block(const TabletSchema* tablet_schema, int size) { _schema = std::make_unique(*tablet_schema); _row_block.reset(new RowBlockV2(*_schema, size)); @@ -152,71 +141,10 @@ public: std::shared_ptr _mem_tracker; std::unique_ptr _mem_pool; - VectorizedRowBatch* _vectorized_batch; std::unique_ptr _row_block; std::unique_ptr _schema; }; -#define TEST_IN_LIST_PREDICATE(TYPE, TYPE_NAME, FIELD_TYPE) \ - TEST_F(TestInListPredicate, TYPE_NAME##_COLUMN) { \ - TabletSchema tablet_schema; \ - SetTabletSchema(std::string("TYPE_NAME##_COLUMN"), FIELD_TYPE, "REPLACE", 1, false, true, \ - &tablet_schema); \ - int size = 10; \ - std::vector return_columns; \ - for (int i = 0; i < tablet_schema.num_columns(); ++i) { \ - return_columns.push_back(i); \ - } \ - InitVectorizedBatch(&tablet_schema, return_columns, size); \ - ColumnVector* col_vector = _vectorized_batch->column(0); \ - \ - /* for no nulls */ \ - col_vector->set_no_nulls(true); \ - TYPE* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(TYPE))); \ - col_vector->set_col_data(col_data); \ - for (int i = 0; i < size; ++i) { \ - *(col_data + i) = i; \ - } \ - \ - phmap::flat_hash_set values; \ - values.insert(4); \ - values.insert(5); \ - values.insert(6); \ - ColumnPredicate* pred = new InListPredicate(0, std::move(values)); \ - pred->evaluate(_vectorized_batch); \ - EXPECT_EQ(_vectorized_batch->size(), 3); \ - uint16_t* sel = _vectorized_batch->selected(); \ - EXPECT_EQ(*(col_data + sel[0]), 4); \ - EXPECT_EQ(*(col_data + sel[1]), 5); \ - EXPECT_EQ(*(col_data + sel[2]), 6); \ - \ - /* for has nulls */ \ - col_vector->set_no_nulls(false); \ - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); \ - memset(is_null, 0, size); \ - col_vector->set_is_null(is_null); \ - for (int i = 0; i < size; ++i) { \ - if (i % 2 == 0) { \ - is_null[i] = true; \ - } else { \ - *(col_data + i) = i; \ - } \ - } \ - _vectorized_batch->set_size(size); \ - _vectorized_batch->set_selected_in_use(false); \ - pred->evaluate(_vectorized_batch); \ - EXPECT_EQ(_vectorized_batch->size(), 1); \ - sel = _vectorized_batch->selected(); \ - EXPECT_EQ(*(col_data + sel[0]), 5); \ - delete pred; \ - } - -TEST_IN_LIST_PREDICATE(int8_t, TINYINT, "TINYINT") -TEST_IN_LIST_PREDICATE(int16_t, SMALLINT, "SMALLINT") -TEST_IN_LIST_PREDICATE(int32_t, INT, "INT") -TEST_IN_LIST_PREDICATE(int64_t, BIGINT, "BIGINT") -TEST_IN_LIST_PREDICATE(int128_t, LARGEINT, "LARGEINT") - #define TEST_IN_LIST_PREDICATE_V2(TYPE, TYPE_NAME, FIELD_TYPE) \ TEST_F(TestInListPredicate, TYPE_NAME##_COLUMN_V2) { \ TabletSchema tablet_schema; \ @@ -296,22 +224,6 @@ TEST_F(TestInListPredicate, FLOAT_COLUMN) { values.insert(6.1); ColumnPredicate* pred = new InListPredicate(0, std::move(values)); - // for VectorizedBatch no null - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - float* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(float))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - *(col_data + i) = i + 0.1; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 3); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_FLOAT_EQ(*(col_data + sel[0]), 4.1); - EXPECT_FLOAT_EQ(*(col_data + sel[1]), 5.1); - EXPECT_FLOAT_EQ(*(col_data + sel[2]), 6.1); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -327,25 +239,6 @@ TEST_F(TestInListPredicate, FLOAT_COLUMN) { EXPECT_FLOAT_EQ(*(float*)col_block.cell(_row_block->selection_vector()[1]).cell_ptr(), 5.1); EXPECT_FLOAT_EQ(*(float*)col_block.cell(_row_block->selection_vector()[2]).cell_ptr(), 6.1); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - *(col_data + i) = i + 0.1; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_FLOAT_EQ(*(col_data + sel[0]), 5.1); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -381,22 +274,6 @@ TEST_F(TestInListPredicate, DOUBLE_COLUMN) { ColumnPredicate* pred = new InListPredicate(0, std::move(values)); - // for VectorizedBatch no null - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - double* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(double))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - *(col_data + i) = i + 0.1; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 3); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_DOUBLE_EQ(*(col_data + sel[0]), 4.1); - EXPECT_DOUBLE_EQ(*(col_data + sel[1]), 5.1); - EXPECT_DOUBLE_EQ(*(col_data + sel[2]), 6.1); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -412,25 +289,6 @@ TEST_F(TestInListPredicate, DOUBLE_COLUMN) { EXPECT_DOUBLE_EQ(*(double*)col_block.cell(_row_block->selection_vector()[1]).cell_ptr(), 5.1); EXPECT_DOUBLE_EQ(*(double*)col_block.cell(_row_block->selection_vector()[2]).cell_ptr(), 6.1); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - *(col_data + i) = i + 0.1; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_DOUBLE_EQ(*(col_data + sel[0]), 5.1); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -470,24 +328,6 @@ TEST_F(TestInListPredicate, DECIMAL_COLUMN) { ColumnPredicate* pred = new InListPredicate(0, std::move(values)); - // for VectorizedBatch no null - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - decimal12_t* col_data = - reinterpret_cast(_mem_pool->allocate(size * sizeof(decimal12_t))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - (*(col_data + i)).integer = i; - (*(col_data + i)).fraction = i; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 3); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(*(col_data + sel[0]), value1); - EXPECT_EQ(*(col_data + sel[1]), value2); - EXPECT_EQ(*(col_data + sel[2]), value3); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -504,26 +344,6 @@ TEST_F(TestInListPredicate, DECIMAL_COLUMN) { EXPECT_EQ(*(decimal12_t*)col_block.cell(_row_block->selection_vector()[1]).cell_ptr(), value2); EXPECT_EQ(*(decimal12_t*)col_block.cell(_row_block->selection_vector()[2]).cell_ptr(), value3); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - (*(col_data + i)).integer = i; - (*(col_data + i)).fraction = i; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(*(col_data + sel[0]), value2); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -573,37 +393,12 @@ TEST_F(TestInListPredicate, CHAR_COLUMN) { ColumnPredicate* pred = new InListPredicate(0, std::move(values)); - // for VectorizedBatch no null - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - StringValue* col_data = - reinterpret_cast(_mem_pool->allocate(size * sizeof(StringValue))); - col_vector->set_col_data(col_data); - - char* string_buffer = reinterpret_cast(_mem_pool->allocate(60)); - memset(string_buffer, 0, 60); - for (int i = 0; i < size; ++i) { - for (int j = 0; j <= 5; ++j) { - string_buffer[j] = 'a' + i; - } - (*(col_data + i)).len = 5; - (*(col_data + i)).ptr = string_buffer; - string_buffer += 5; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 3); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(*(col_data + sel[0]), value1); - EXPECT_EQ(*(col_data + sel[1]), value2); - EXPECT_EQ(*(col_data + sel[2]), value3); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); auto select_size = _row_block->selected_size(); ColumnBlockView col_block_view(&col_block); - string_buffer = reinterpret_cast(_mem_pool->allocate(60)); + char* string_buffer = reinterpret_cast(_mem_pool->allocate(60)); memset(string_buffer, 0, 60); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { col_block_view.set_null_bits(1, false); @@ -620,32 +415,6 @@ TEST_F(TestInListPredicate, CHAR_COLUMN) { EXPECT_EQ(*(StringValue*)col_block.cell(_row_block->selection_vector()[1]).cell_ptr(), value2); EXPECT_EQ(*(StringValue*)col_block.cell(_row_block->selection_vector()[2]).cell_ptr(), value3); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - string_buffer = reinterpret_cast(_mem_pool->allocate(60)); - memset(string_buffer, 0, 60); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - for (int j = 0; j <= 5; ++j) { - string_buffer[j] = 'a' + i; - } - (*(col_data + i)).len = 5; - (*(col_data + i)).ptr = string_buffer; - } - string_buffer += 5; - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(*(col_data + sel[0]), value2); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); string_buffer = reinterpret_cast(_mem_pool->allocate(55)); @@ -701,36 +470,12 @@ TEST_F(TestInListPredicate, VARCHAR_COLUMN) { ColumnPredicate* pred = new InListPredicate(0, std::move(values)); - // for VectorizedBatch no null - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - StringValue* col_data = - reinterpret_cast(_mem_pool->allocate(size * sizeof(StringValue))); - col_vector->set_col_data(col_data); - - char* string_buffer = reinterpret_cast(_mem_pool->allocate(55)); - for (int i = 0; i < size; ++i) { - for (int j = 0; j <= i; ++j) { - string_buffer[j] = 'a' + i; - } - (*(col_data + i)).len = i + 1; - (*(col_data + i)).ptr = string_buffer; - string_buffer += i + 1; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 3); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(*(col_data + sel[0]), value1); - EXPECT_EQ(*(col_data + sel[1]), value2); - EXPECT_EQ(*(col_data + sel[2]), value3); - // for ColumnBlock no null init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); auto select_size = _row_block->selected_size(); ColumnBlockView col_block_view(&col_block); - string_buffer = reinterpret_cast(_mem_pool->allocate(60)); + char* string_buffer = reinterpret_cast(_mem_pool->allocate(60)); memset(string_buffer, 0, 60); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { col_block_view.set_null_bits(1, false); @@ -747,31 +492,6 @@ TEST_F(TestInListPredicate, VARCHAR_COLUMN) { EXPECT_EQ(*(StringValue*)col_block.cell(_row_block->selection_vector()[1]).cell_ptr(), value2); EXPECT_EQ(*(StringValue*)col_block.cell(_row_block->selection_vector()[2]).cell_ptr(), value3); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - string_buffer = reinterpret_cast(_mem_pool->allocate(55)); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - for (int j = 0; j <= i; ++j) { - string_buffer[j] = 'a' + i; - } - (*(col_data + i)).len = i + 1; - (*(col_data + i)).ptr = string_buffer; - } - string_buffer += i + 1; - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(*(col_data + sel[0]), value2); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); string_buffer = reinterpret_cast(_mem_pool->allocate(55)); @@ -816,13 +536,6 @@ TEST_F(TestInListPredicate, DATE_COLUMN) { values.insert(value3); ColumnPredicate* pred = new InListPredicate(0, std::move(values)); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - uint24_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint24_t))); - col_vector->set_col_data(col_data); - std::vector date_array; date_array.push_back("2017-09-07"); date_array.push_back("2017-09-08"); @@ -830,16 +543,6 @@ TEST_F(TestInListPredicate, DATE_COLUMN) { date_array.push_back("2017-09-10"); date_array.push_back("2017-09-11"); date_array.push_back("2017-09-12"); - for (int i = 0; i < size; ++i) { - uint24_t timestamp = datetime::timestamp_from_date(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 3); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(datetime::to_date_string(*(col_data + sel[0])), "2017-09-09"); - EXPECT_EQ(datetime::to_date_string(*(col_data + sel[1])), "2017-09-10"); - EXPECT_EQ(datetime::to_date_string(*(col_data + sel[2])), "2017-09-11"); // for ColumnBlock no nulls init_row_block(&tablet_schema, size); @@ -863,26 +566,6 @@ TEST_F(TestInListPredicate, DATE_COLUMN) { *(uint24_t*)col_block.cell(_row_block->selection_vector()[2]).cell_ptr()), "2017-09-11"); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - uint24_t timestamp = datetime::timestamp_from_date(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(datetime::to_date_string(*(col_data + sel[0])), "2017-09-10"); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -925,13 +608,6 @@ TEST_F(TestInListPredicate, DATE_V2_COLUMN) { values.insert(value3); ColumnPredicate* pred = new InListPredicate(0, std::move(values)); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - uint32_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint32_t))); - col_vector->set_col_data(col_data); - std::vector date_array; date_array.push_back("2017-09-07"); date_array.push_back("2017-09-08"); @@ -939,16 +615,6 @@ TEST_F(TestInListPredicate, DATE_V2_COLUMN) { date_array.push_back("2017-09-10"); date_array.push_back("2017-09-11"); date_array.push_back("2017-09-12"); - for (int i = 0; i < size; ++i) { - uint32_t timestamp = datetime::timestamp_from_date_v2(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 3); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[0])), "2017-09-09"); - EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[1])), "2017-09-10"); - EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[2])), "2017-09-11"); // for ColumnBlock no nulls init_row_block(&tablet_schema, size); @@ -972,26 +638,6 @@ TEST_F(TestInListPredicate, DATE_V2_COLUMN) { *(uint32_t*)col_block.cell(_row_block->selection_vector()[2]).cell_ptr()), "2017-09-11"); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - uint32_t timestamp = datetime::timestamp_from_date_v2(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(datetime::to_date_v2_string(*(col_data + sel[0])), "2017-09-10"); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -1035,13 +681,6 @@ TEST_F(TestInListPredicate, DATETIME_COLUMN) { ColumnPredicate* pred = new InListPredicate(0, std::move(values)); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - uint64_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint64_t))); - col_vector->set_col_data(col_data); - std::vector date_array; date_array.push_back("2017-09-07 00:00:00"); date_array.push_back("2017-09-08 00:01:00"); @@ -1049,16 +688,6 @@ TEST_F(TestInListPredicate, DATETIME_COLUMN) { date_array.push_back("2017-09-10 01:00:00"); date_array.push_back("2017-09-11 01:01:00"); date_array.push_back("2017-09-12 01:01:01"); - for (int i = 0; i < size; ++i) { - uint64_t timestamp = datetime::timestamp_from_datetime(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 3); - uint16_t* sel = _vectorized_batch->selected(); - EXPECT_EQ(datetime::to_datetime_string(*(col_data + sel[0])), "2017-09-09 00:00:01"); - EXPECT_EQ(datetime::to_datetime_string(*(col_data + sel[1])), "2017-09-10 01:00:00"); - EXPECT_EQ(datetime::to_datetime_string(*(col_data + sel[2])), "2017-09-11 01:01:00"); // for ColumnBlock no nulls init_row_block(&tablet_schema, size); @@ -1082,26 +711,6 @@ TEST_F(TestInListPredicate, DATETIME_COLUMN) { *(uint64_t*)col_block.cell(_row_block->selection_vector()[2]).cell_ptr()), "2017-09-11 01:01:00"); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - uint64_t timestamp = datetime::timestamp_from_datetime(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 1); - sel = _vectorized_batch->selected(); - EXPECT_EQ(datetime::to_datetime_string(*(col_data + sel[0])), "2017-09-10 01:00:00"); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { diff --git a/be/test/olap/null_predicate_test.cpp b/be/test/olap/null_predicate_test.cpp index 1fe189faf6..f837af19b9 100644 --- a/be/test/olap/null_predicate_test.cpp +++ b/be/test/olap/null_predicate_test.cpp @@ -26,7 +26,6 @@ #include "olap/row_block2.h" #include "runtime/mem_pool.h" #include "runtime/string_value.hpp" -#include "runtime/vectorized_row_batch.h" #include "util/logging.h" #include "vec/columns/column_nullable.h" #include "vec/core/block.h" @@ -52,32 +51,16 @@ static uint32_t to_date_v2_timestamp(const char* date_string) { return ((time_tm.tm_year + 1900) << 16) | ((time_tm.tm_mon + 1) << 8) | time_tm.tm_mday; } -static uint64_t to_datetime_timestamp(const std::string& value_string) { - tm time_tm; - strptime(value_string.c_str(), "%Y-%m-%d %H:%M:%S", &time_tm); - - uint64_t value = - ((time_tm.tm_year + 1900) * 10000L + (time_tm.tm_mon + 1) * 100L + time_tm.tm_mday) * - 1000000L + - time_tm.tm_hour * 10000L + time_tm.tm_min * 100L + time_tm.tm_sec; - - return value; -} - }; // namespace datetime class TestNullPredicate : public testing::Test { public: - TestNullPredicate() : _vectorized_batch(nullptr), _row_block(nullptr) { + TestNullPredicate() : _row_block(nullptr) { _mem_tracker.reset(new MemTracker(-1)); _mem_pool.reset(new MemPool(_mem_tracker.get())); } - ~TestNullPredicate() { - if (_vectorized_batch != nullptr) { - delete _vectorized_batch; - } - } + ~TestNullPredicate() {} void SetTabletSchema(std::string name, std::string type, std::string aggregation, uint32_t length, bool is_allow_null, bool is_key, @@ -99,12 +82,6 @@ public: tablet_schema->init_from_pb(tablet_schema_pb); } - void InitVectorizedBatch(const TabletSchema* tablet_schema, const std::vector& ids, - int size) { - _vectorized_batch = new VectorizedRowBatch(tablet_schema, ids, size); - _vectorized_batch->set_size(size); - } - void init_row_block(const TabletSchema* tablet_schema, int size) { _schema = std::make_unique(*tablet_schema); _row_block.reset(new RowBlockV2(*_schema, size)); @@ -112,7 +89,6 @@ public: std::shared_ptr _mem_tracker; std::unique_ptr _mem_pool; - VectorizedRowBatch* _vectorized_batch; std::unique_ptr _row_block; std::unique_ptr _schema; }; @@ -129,19 +105,6 @@ public: } \ std::unique_ptr pred(new NullPredicate(0, true)); \ \ - /* for VectorizedBatch nulls */ \ - InitVectorizedBatch(&tablet_schema, return_columns, size); \ - init_row_block(&tablet_schema, size); \ - ColumnVector* col_vector = _vectorized_batch->column(0); \ - col_vector->set_no_nulls(true); \ - TYPE* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(TYPE))); \ - col_vector->set_col_data(col_data); \ - for (int i = 0; i < size; ++i) { \ - *(col_data + i) = i; \ - } \ - pred->evaluate(_vectorized_batch); \ - EXPECT_EQ(_vectorized_batch->size(), 0); \ - \ /* for ColumnBlock nulls */ \ init_row_block(&tablet_schema, size); \ ColumnBlock col_block = _row_block->column_block(0); \ @@ -164,23 +127,6 @@ public: _row_block->selection_vector(), select_size); \ EXPECT_EQ(select_size, 0); \ \ - /* for has nulls */ \ - col_vector->set_no_nulls(false); \ - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); \ - memset(is_null, 0, size); \ - col_vector->set_is_null(is_null); \ - for (int i = 0; i < size; ++i) { \ - if (i % 2 == 0) { \ - is_null[i] = true; \ - } else { \ - *(col_data + i) = i; \ - } \ - } \ - _vectorized_batch->set_size(size); \ - _vectorized_batch->set_selected_in_use(false); \ - pred->evaluate(_vectorized_batch); \ - EXPECT_EQ(_vectorized_batch->size(), 5); \ - \ /* for ColumnBlock has nulls */ \ col_block_view = ColumnBlockView(&col_block); \ for (int i = 0; i < size; ++i, col_block_view.advance(1)) { \ @@ -224,18 +170,6 @@ TEST_F(TestNullPredicate, FLOAT_COLUMN) { } std::unique_ptr pred(new NullPredicate(0, true)); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - float* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(float))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - *(col_data + i) = i + 0.1; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 0); - // for ColumnBlock no nulls init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -258,23 +192,6 @@ TEST_F(TestNullPredicate, FLOAT_COLUMN) { _row_block->selection_vector(), select_size); EXPECT_EQ(select_size, 0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - *(col_data + i) = i + 0.1; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 5); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -312,18 +229,6 @@ TEST_F(TestNullPredicate, DOUBLE_COLUMN) { } std::unique_ptr pred(new NullPredicate(0, true)); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - double* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(double))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - *(col_data + i) = i + 0.1; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 0); - // for ColumnBlock no nulls init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -346,23 +251,6 @@ TEST_F(TestNullPredicate, DOUBLE_COLUMN) { _row_block->selection_vector(), select_size); EXPECT_EQ(select_size, 0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 2 == 0) { - is_null[i] = true; - } else { - *(col_data + i) = i + 0.1; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 5); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -400,20 +288,6 @@ TEST_F(TestNullPredicate, DECIMAL_COLUMN) { } std::unique_ptr pred(new NullPredicate(0, true)); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - decimal12_t* col_data = - reinterpret_cast(_mem_pool->allocate(size * sizeof(decimal12_t))); - col_vector->set_col_data(col_data); - for (int i = 0; i < size; ++i) { - (*(col_data + i)).integer = i; - (*(col_data + i)).fraction = i; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 0); - // for ColumnBlock no nulls init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); @@ -437,24 +311,6 @@ TEST_F(TestNullPredicate, DECIMAL_COLUMN) { _row_block->selection_vector(), select_size); EXPECT_EQ(select_size, 0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 3 == 0) { - is_null[i] = true; - } else { - (*(col_data + i)).integer = i; - (*(col_data + i)).fraction = i; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 4); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -493,33 +349,13 @@ TEST_F(TestNullPredicate, STRING_COLUMN) { } std::unique_ptr pred(new NullPredicate(0, true)); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - StringValue* col_data = - reinterpret_cast(_mem_pool->allocate(size * sizeof(StringValue))); - col_vector->set_col_data(col_data); - char* string_buffer = reinterpret_cast(_mem_pool->allocate(55)); - for (int i = 0; i < size; ++i) { - for (int j = 0; j <= i; ++j) { - string_buffer[j] = 'a' + i; - } - (*(col_data + i)).len = i + 1; - (*(col_data + i)).ptr = string_buffer; - string_buffer += i + 1; - } - EXPECT_EQ(_vectorized_batch->size(), 10); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 0); - // for ColumnBlock no nulls init_row_block(&tablet_schema, size); ColumnBlock col_block = _row_block->column_block(0); auto select_size = _row_block->selected_size(); ColumnBlockView col_block_view(&col_block); - string_buffer = reinterpret_cast(_mem_pool->allocate(55)); + char* string_buffer = reinterpret_cast(_mem_pool->allocate(55)); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { col_block_view.set_null_bits(1, false); for (int j = 0; j <= i; ++j) { @@ -542,29 +378,6 @@ TEST_F(TestNullPredicate, STRING_COLUMN) { _row_block->selection_vector(), select_size); EXPECT_EQ(select_size, 0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - string_buffer = reinterpret_cast(_mem_pool->allocate(55)); - for (int i = 0; i < size; ++i) { - if (i % 3 == 0) { - is_null[i] = true; - } else { - for (int j = 0; j <= i; ++j) { - string_buffer[j] = 'a' + i; - } - (*(col_data + i)).len = i + 1; - (*(col_data + i)).ptr = string_buffer; - } - string_buffer += i + 1; - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 4); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); string_buffer = reinterpret_cast(_mem_pool->allocate(55)); @@ -607,13 +420,6 @@ TEST_F(TestNullPredicate, DATE_COLUMN) { } std::unique_ptr pred(new NullPredicate(0, true)); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - uint24_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint24_t))); - col_vector->set_col_data(col_data); - std::vector date_array; date_array.push_back("2017-09-07"); date_array.push_back("2017-09-08"); @@ -621,12 +427,6 @@ TEST_F(TestNullPredicate, DATE_COLUMN) { date_array.push_back("2017-09-10"); date_array.push_back("2017-09-11"); date_array.push_back("2017-09-12"); - for (int i = 0; i < size; ++i) { - uint24_t timestamp = datetime::to_date_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 0); // for ColumnBlock no nulls init_row_block(&tablet_schema, size); @@ -651,24 +451,6 @@ TEST_F(TestNullPredicate, DATE_COLUMN) { _row_block->selection_vector(), select_size); EXPECT_EQ(select_size, 0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 3 == 0) { - is_null[i] = true; - } else { - uint24_t timestamp = datetime::to_date_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 2); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -707,25 +489,13 @@ TEST_F(TestNullPredicate, DATETIME_COLUMN) { } std::unique_ptr pred(new NullPredicate(0, true)); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - uint64_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint64_t))); - col_vector->set_col_data(col_data); std::vector date_array; - date_array.push_back("2017-09-07 00:00:00"); - date_array.push_back("2017-09-08 00:01:00"); - date_array.push_back("2017-09-09 00:00:01"); - date_array.push_back("2017-09-10 01:00:00"); - date_array.push_back("2017-09-11 01:01:00"); - date_array.push_back("2017-09-12 01:01:01"); - for (int i = 0; i < size; ++i) { - uint64_t timestamp = datetime::to_datetime_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 0); + date_array.push_back("2017-09-07"); + date_array.push_back("2017-09-08"); + date_array.push_back("2017-09-09"); + date_array.push_back("2017-09-10"); + date_array.push_back("2017-09-11"); + date_array.push_back("2017-09-12"); // for ColumnBlock no nulls init_row_block(&tablet_schema, size); @@ -750,24 +520,6 @@ TEST_F(TestNullPredicate, DATETIME_COLUMN) { _row_block->selection_vector(), select_size); EXPECT_EQ(select_size, 0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 3 == 0) { - is_null[i] = true; - } else { - uint64_t timestamp = datetime::to_datetime_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 2); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { @@ -806,13 +558,6 @@ TEST_F(TestNullPredicate, DATEV2_COLUMN) { } std::unique_ptr pred(new NullPredicate(0, true)); - // for VectorizedBatch no nulls - InitVectorizedBatch(&tablet_schema, return_columns, size); - ColumnVector* col_vector = _vectorized_batch->column(0); - col_vector->set_no_nulls(true); - uint32_t* col_data = reinterpret_cast(_mem_pool->allocate(size * sizeof(uint32_t))); - col_vector->set_col_data(col_data); - std::vector date_array; date_array.push_back("2017-09-07"); date_array.push_back("2017-09-08"); @@ -820,12 +565,6 @@ TEST_F(TestNullPredicate, DATEV2_COLUMN) { date_array.push_back("2017-09-10"); date_array.push_back("2017-09-11"); date_array.push_back("2017-09-12"); - for (int i = 0; i < size; ++i) { - uint32_t timestamp = datetime::to_date_v2_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 0); // for ColumnBlock no nulls init_row_block(&tablet_schema, size); @@ -850,24 +589,6 @@ TEST_F(TestNullPredicate, DATEV2_COLUMN) { _row_block->selection_vector(), select_size); EXPECT_EQ(select_size, 0); - // for VectorizedBatch has nulls - col_vector->set_no_nulls(false); - bool* is_null = reinterpret_cast(_mem_pool->allocate(size)); - memset(is_null, 0, size); - col_vector->set_is_null(is_null); - for (int i = 0; i < size; ++i) { - if (i % 3 == 0) { - is_null[i] = true; - } else { - uint32_t timestamp = datetime::to_date_v2_timestamp(date_array[i].c_str()); - *(col_data + i) = timestamp; - } - } - _vectorized_batch->set_size(size); - _vectorized_batch->set_selected_in_use(false); - pred->evaluate(_vectorized_batch); - EXPECT_EQ(_vectorized_batch->size(), 2); - // for ColumnBlock has nulls col_block_view = ColumnBlockView(&col_block); for (int i = 0; i < size; ++i, col_block_view.advance(1)) { diff --git a/be/test/olap/rowset/alpha_rowset_test.cpp b/be/test/olap/rowset/alpha_rowset_test.cpp deleted file mode 100644 index c5f82a100f..0000000000 --- a/be/test/olap/rowset/alpha_rowset_test.cpp +++ /dev/null @@ -1,333 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/rowset/alpha_rowset.h" - -#include -#include -#include -#include - -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "json2pb/json_to_pb.h" -#include "olap/data_dir.h" -#include "olap/olap_meta.h" -#include "olap/rowset/alpha_rowset_reader.h" -#include "olap/rowset/rowset_factory.h" -#include "olap/rowset/rowset_reader_context.h" -#include "olap/rowset/rowset_writer.h" -#include "olap/rowset/rowset_writer_context.h" -#include "olap/storage_engine.h" -#include "util/file_utils.h" -#include "util/logging.h" - -#ifndef BE_TEST -#define BE_TEST -#endif - -using ::testing::_; -using ::testing::Return; -using ::testing::SetArgPointee; -using std::string; - -namespace doris { - -static const uint32_t MAX_PATH_LEN = 1024; -static StorageEngine* k_engine = nullptr; - -void set_up() { - config::path_gc_check = false; - char buffer[MAX_PATH_LEN]; - EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr); - config::storage_root_path = std::string(buffer) + "/data_test"; - FileUtils::remove_all(config::storage_root_path); - EXPECT_TRUE(FileUtils::create_dir(config::storage_root_path).ok()); - std::vector paths; - paths.emplace_back(config::storage_root_path, -1); - - doris::EngineOptions options; - options.store_paths = paths; - Status s = doris::StorageEngine::open(options, &k_engine); - EXPECT_TRUE(s.ok()) << s.to_string(); - - ExecEnv* exec_env = doris::ExecEnv::GetInstance(); - exec_env->set_storage_engine(k_engine); - std::string data_path = config::storage_root_path + "/data"; - EXPECT_TRUE(FileUtils::create_dir(data_path).ok()); - std::string shard_path = data_path + "/0"; - EXPECT_TRUE(FileUtils::create_dir(shard_path).ok()); - std::string tablet_path = shard_path + "/12345"; - EXPECT_TRUE(FileUtils::create_dir(tablet_path).ok()); - std::string schema_hash_path = tablet_path + "/1111"; - EXPECT_TRUE(FileUtils::create_dir(schema_hash_path).ok()); -} - -void tear_down() { - if (k_engine != nullptr) { - k_engine->stop(); - delete k_engine; - k_engine = nullptr; - } -} - -void create_rowset_writer_context(TabletSchema* tablet_schema, - RowsetWriterContext* rowset_writer_context) { - RowsetId rowset_id; - rowset_id.init(10000); - rowset_writer_context->rowset_id = rowset_id; - rowset_writer_context->tablet_id = 12345; - rowset_writer_context->tablet_schema_hash = 1111; - rowset_writer_context->partition_id = 10; - rowset_writer_context->rowset_type = ALPHA_ROWSET; - rowset_writer_context->path_desc.filepath = config::storage_root_path + "/data/0/12345/1111"; - rowset_writer_context->rowset_state = VISIBLE; - rowset_writer_context->tablet_schema = tablet_schema; - rowset_writer_context->version.first = 0; - rowset_writer_context->version.second = 1; -} - -void create_rowset_reader_context(TabletSchema* tablet_schema, - const std::vector* return_columns, - const DeleteHandler* delete_handler, - std::vector* predicates, - std::set* load_bf_columns, Conditions* conditions, - RowsetReaderContext* rowset_reader_context) { - rowset_reader_context->reader_type = READER_ALTER_TABLE; - rowset_reader_context->tablet_schema = tablet_schema; - rowset_reader_context->need_ordered_result = true; - rowset_reader_context->return_columns = return_columns; - rowset_reader_context->delete_handler = delete_handler; - rowset_reader_context->lower_bound_keys = nullptr; - rowset_reader_context->is_lower_keys_included = nullptr; - rowset_reader_context->upper_bound_keys = nullptr; - rowset_reader_context->is_upper_keys_included = nullptr; - rowset_reader_context->predicates = predicates; - rowset_reader_context->load_bf_columns = load_bf_columns; - rowset_reader_context->conditions = conditions; -} - -static void create_tablet_schema(KeysType keys_type, TabletSchema* tablet_schema) { - TabletSchemaPB tablet_schema_pb; - tablet_schema_pb.set_keys_type(keys_type); - tablet_schema_pb.set_num_short_key_columns(2); - tablet_schema_pb.set_num_rows_per_row_block(1024); - tablet_schema_pb.set_compress_kind(COMPRESS_NONE); - tablet_schema_pb.set_next_column_unique_id(4); - - ColumnPB* column_1 = tablet_schema_pb.add_column(); - column_1->set_unique_id(1); - column_1->set_name("k1"); - column_1->set_type("INT"); - column_1->set_is_key(true); - column_1->set_length(4); - column_1->set_index_length(4); - column_1->set_is_nullable(false); - column_1->set_is_bf_column(false); - - ColumnPB* column_2 = tablet_schema_pb.add_column(); - column_2->set_unique_id(2); - column_2->set_name("k2"); - column_2->set_type("VARCHAR"); - column_2->set_length(20); - column_2->set_index_length(20); - column_2->set_is_key(true); - column_2->set_is_nullable(false); - column_2->set_is_bf_column(false); - - ColumnPB* column_3 = tablet_schema_pb.add_column(); - column_3->set_unique_id(3); - column_3->set_name("v1"); - column_3->set_type("INT"); - column_3->set_length(4); - column_3->set_is_key(false); - column_3->set_is_nullable(false); - column_3->set_is_bf_column(false); - column_3->set_aggregation("SUM"); - - tablet_schema->init_from_pb(tablet_schema_pb); -} - -class AlphaRowsetTest : public testing::Test { -public: - virtual void SetUp() { - set_up(); - _mem_tracker.reset(new MemTracker(-1)); - _mem_pool.reset(new MemPool(_mem_tracker.get())); - } - virtual void TearDown() { tear_down(); } - -private: - std::unique_ptr _alpha_rowset_writer; - std::shared_ptr _mem_tracker; - std::unique_ptr _mem_pool; -}; - -TEST_F(AlphaRowsetTest, TestAlphaRowsetWriter) { - TabletSchema tablet_schema; - create_tablet_schema(AGG_KEYS, &tablet_schema); - RowsetWriterContext rowset_writer_context; - create_rowset_writer_context(&tablet_schema, &rowset_writer_context); - EXPECT_EQ(Status::OK(), - RowsetFactory::create_rowset_writer(rowset_writer_context, &_alpha_rowset_writer)); - RowCursor row; - Status res = row.init(tablet_schema); - EXPECT_EQ(Status::OK(), res); - - int32_t field_0 = 10; - row.set_field_content(0, reinterpret_cast(&field_0), _mem_pool.get()); - Slice field_1("well"); - row.set_field_content(1, reinterpret_cast(&field_1), _mem_pool.get()); - int32_t field_2 = 100; - row.set_field_content(2, reinterpret_cast(&field_2), _mem_pool.get()); - _alpha_rowset_writer->add_row(row); - _alpha_rowset_writer->flush(); - RowsetSharedPtr alpha_rowset = _alpha_rowset_writer->build(); - EXPECT_TRUE(alpha_rowset != nullptr); - RowsetId rowset_id; - rowset_id.init(10000); - EXPECT_EQ(rowset_id, alpha_rowset->rowset_id()); - EXPECT_EQ(1, alpha_rowset->num_rows()); -} - -TEST_F(AlphaRowsetTest, TestAlphaRowsetReader) { - TabletSchema tablet_schema; - create_tablet_schema(AGG_KEYS, &tablet_schema); - RowsetWriterContext rowset_writer_context; - create_rowset_writer_context(&tablet_schema, &rowset_writer_context); - - EXPECT_EQ(Status::OK(), - RowsetFactory::create_rowset_writer(rowset_writer_context, &_alpha_rowset_writer)); - - RowCursor row; - Status res = row.init(tablet_schema); - EXPECT_EQ(Status::OK(), res); - - int32_t field_0 = 10; - row.set_not_null(0); - row.set_field_content(0, reinterpret_cast(&field_0), _mem_pool.get()); - Slice field_1("well"); - row.set_not_null(1); - row.set_field_content(1, reinterpret_cast(&field_1), _mem_pool.get()); - int32_t field_2 = 100; - row.set_not_null(2); - row.set_field_content(2, reinterpret_cast(&field_2), _mem_pool.get()); - res = _alpha_rowset_writer->add_row(row); - EXPECT_EQ(Status::OK(), res); - res = _alpha_rowset_writer->flush(); - EXPECT_EQ(Status::OK(), res); - RowsetSharedPtr alpha_rowset = _alpha_rowset_writer->build(); - EXPECT_TRUE(alpha_rowset != nullptr); - RowsetId rowset_id; - rowset_id.init(10000); - EXPECT_EQ(rowset_id, alpha_rowset->rowset_id()); - EXPECT_EQ(1, alpha_rowset->num_rows()); - RowsetReaderSharedPtr rowset_reader; - res = alpha_rowset->create_reader(&rowset_reader); - EXPECT_EQ(Status::OK(), res); - std::vector return_columns; - for (int i = 0; i < tablet_schema.num_columns(); ++i) { - return_columns.push_back(i); - } - DeleteHandler delete_handler; - DelPredicateArray predicate_array; - res = delete_handler.init(tablet_schema, predicate_array, 4); - EXPECT_EQ(Status::OK(), res); - RowsetReaderContext rowset_reader_context; - - std::set load_bf_columns; - std::vector predicates; - Conditions conditions; - create_rowset_reader_context(&tablet_schema, &return_columns, &delete_handler, &predicates, - &load_bf_columns, &conditions, &rowset_reader_context); - res = rowset_reader->init(&rowset_reader_context); - EXPECT_EQ(Status::OK(), res); - RowBlock* row_block = nullptr; - res = rowset_reader->next_block(&row_block); - EXPECT_EQ(Status::OK(), res); - EXPECT_EQ(1, row_block->remaining()); -} - -TEST_F(AlphaRowsetTest, TestRowCursorWithOrdinal) { - TabletSchema tablet_schema; - create_tablet_schema(AGG_KEYS, &tablet_schema); - - RowCursor* row1 = new (std::nothrow) RowCursor(); // 10, "well", 100 - row1->init(tablet_schema); - int32_t field1_0 = 10; - row1->set_not_null(0); - row1->set_field_content(0, reinterpret_cast(&field1_0), _mem_pool.get()); - Slice field1_1("well"); - row1->set_not_null(1); - row1->set_field_content(1, reinterpret_cast(&field1_1), _mem_pool.get()); - int32_t field1_2 = 100; - row1->set_not_null(2); - row1->set_field_content(2, reinterpret_cast(&field1_2), _mem_pool.get()); - - RowCursor* row2 = new (std::nothrow) RowCursor(); // 11, "well", 100 - row2->init(tablet_schema); - int32_t field2_0 = 11; - row2->set_not_null(0); - row2->set_field_content(0, reinterpret_cast(&field2_0), _mem_pool.get()); - Slice field2_1("well"); - row2->set_not_null(1); - row2->set_field_content(1, reinterpret_cast(&field2_1), _mem_pool.get()); - int32_t field2_2 = 100; - row2->set_not_null(2); - row2->set_field_content(2, reinterpret_cast(&field2_2), _mem_pool.get()); - - RowCursor* row3 = new (std::nothrow) RowCursor(); // 11, "good", 100 - row3->init(tablet_schema); - int32_t field3_0 = 11; - row3->set_not_null(0); - row3->set_field_content(0, reinterpret_cast(&field3_0), _mem_pool.get()); - Slice field3_1("good"); - row3->set_not_null(1); - row3->set_field_content(1, reinterpret_cast(&field3_1), _mem_pool.get()); - int32_t field3_2 = 100; - row3->set_not_null(2); - row3->set_field_content(2, reinterpret_cast(&field3_2), _mem_pool.get()); - - std::priority_queue, - AlphaMergeContextComparator> - queue; - AlphaMergeContext ctx1; - ctx1.row_cursor.reset(row1); - AlphaMergeContext ctx2; - ctx2.row_cursor.reset(row2); - AlphaMergeContext ctx3; - ctx3.row_cursor.reset(row3); - - queue.push(&ctx1); - queue.push(&ctx2); - queue.push(&ctx3); - - // should be: - // row1, row3, row2 - AlphaMergeContext* top1 = queue.top(); - EXPECT_EQ(top1, &ctx1); - queue.pop(); - AlphaMergeContext* top2 = queue.top(); - EXPECT_EQ(top2, &ctx3); - queue.pop(); - AlphaMergeContext* top3 = queue.top(); - EXPECT_EQ(top3, &ctx2); - queue.pop(); - EXPECT_TRUE(queue.empty()); -} - -} // namespace doris diff --git a/be/test/olap/run_length_byte_test.cpp b/be/test/olap/run_length_byte_test.cpp deleted file mode 100644 index 7e5cc3445d..0000000000 --- a/be/test/olap/run_length_byte_test.cpp +++ /dev/null @@ -1,834 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include "olap/byte_buffer.h" -#include "olap/file_stream.h" -#include "olap/in_stream.h" -#include "olap/out_stream.h" -#include "olap/rowset/column_reader.h" -#include "olap/rowset/run_length_byte_reader.h" -#include "olap/rowset/run_length_byte_writer.h" -#include "olap/stream_index_reader.h" -#include "olap/stream_index_writer.h" -#include "util/logging.h" - -namespace doris { - -using namespace testing; - -TEST(TestStream, UncompressOutStream) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, nullptr); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor == nullptr); - - out_stream->write(0x5a); - out_stream->flush(); - - EXPECT_EQ(out_stream->get_stream_length(), sizeof(StreamHead) + 1); - - EXPECT_EQ(out_stream->output_buffers().size(), 1); - - std::vector::const_iterator it = out_stream->output_buffers().begin(); - EXPECT_EQ((*it)->position(), 0); - StreamHead head; - (*it)->get((char*)&head, sizeof(head)); - EXPECT_EQ(head.type, StreamHead::UNCOMPRESSED); - EXPECT_EQ(head.length, 1); - char data; - EXPECT_EQ(Status::OK(), (*it)->get((char*)&data)); - EXPECT_EQ(0x5A, data); - EXPECT_NE(Status::OK(), (*it)->get((char*)&data)); - - SAFE_DELETE(out_stream); -} - -TEST(TestStream, UncompressOutStream2) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, nullptr); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor == nullptr); - - for (int32_t i = 0; i < OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE; i++) { - out_stream->write(0x5a); - } - out_stream->write(0x5a); - out_stream->flush(); - - uint64_t stream_length = sizeof(StreamHead) * 2 + OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE + 1; - EXPECT_EQ(out_stream->get_stream_length(), stream_length); - - EXPECT_EQ(out_stream->output_buffers().size(), 2); - - std::vector inputs; - for (const auto& it : out_stream->output_buffers()) { - inputs.push_back(StorageByteBuffer::reference_buffer(it, 0, it->limit())); - } - - std::vector offsets; - offsets.push_back(0); - offsets.push_back(sizeof(StreamHead) + OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), nullptr, - out_stream->get_total_buffer_size()); - - char data; - for (int32_t i = 0; i < OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE - 1; i++) { - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - } - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - - EXPECT_NE(in_stream->read(&data), Status::OK()); - - SAFE_DELETE(out_stream); - SAFE_DELETE(in_stream); - for (auto input : inputs) { - delete input; - } -} - -TEST(TestStream, UncompressOutStream3) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, nullptr); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor == nullptr); - - for (int32_t i = 0; i < OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE; i++) { - out_stream->write(0x5a); - } - char write_data[2] = {0x5a, 0x5a}; - out_stream->write(write_data, 2); - out_stream->flush(); - - uint64_t stream_length = sizeof(StreamHead) * 2 + OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE + 2; - EXPECT_EQ(out_stream->get_stream_length(), stream_length); - - EXPECT_EQ(out_stream->output_buffers().size(), 2); - - std::vector inputs; - for (const auto& it : out_stream->output_buffers()) { - inputs.push_back(StorageByteBuffer::reference_buffer(it, 0, it->limit())); - } - - std::vector offsets; - offsets.push_back(0); - offsets.push_back(sizeof(StreamHead) + OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), nullptr, - out_stream->get_total_buffer_size()); - - char data; - for (int32_t i = 0; i < OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE; i++) { - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - } - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - - EXPECT_NE(in_stream->read(&data), Status::OK()); - - SAFE_DELETE(in_stream); - SAFE_DELETE(out_stream); - for (auto input : inputs) { - delete input; - } -} - -TEST(TestStream, UncompressInStream) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, nullptr); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor == nullptr); - - out_stream->write(0x5a); - out_stream->flush(); - - // read data - std::vector inputs; - const auto& it = out_stream->output_buffers().begin(); - EXPECT_NE(it, out_stream->output_buffers().end()); - inputs.push_back(StorageByteBuffer::reference_buffer(*it, 0, (*it)->capacity())); - - std::vector offsets; - offsets.assign(inputs.size(), 0); - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), nullptr, - out_stream->get_total_buffer_size()); - SAFE_DELETE(out_stream); - - EXPECT_EQ(in_stream->available(), 1); - char data; - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - - SAFE_DELETE(in_stream); - for (auto input : inputs) { - delete input; - } -} - -// the length after compress must be smaller than original stream, then the compressor will be called. -TEST(TestStream, CompressOutStream) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, lz4_compress); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor != nullptr); - - char* write_data = new char[OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE]; - memset(write_data, 0x5a, OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); - - out_stream->write(write_data, OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); - out_stream->flush(); - - //EXPECT_EQ(out_stream->get_stream_length(), sizeof(StreamHead) + 2); - - //EXPECT_EQ(out_stream->output_buffers().size(), 1); - - std::vector::const_iterator it = out_stream->output_buffers().begin(); - - StreamHead head; - (*it)->get((char*)&head, sizeof(head)); - EXPECT_EQ(head.type, StreamHead::COMPRESSED); - // if lzo, this should be 49 - EXPECT_EQ(51, head.length); - - SAFE_DELETE_ARRAY(write_data); - SAFE_DELETE(out_stream); -} - -TEST(TestStream, CompressOutStream2) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, lz4_compress); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor != nullptr); - - for (int32_t i = 0; i < OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE; i++) { - out_stream->write(0x5a); - } - out_stream->write(0x5a); - out_stream->flush(); - - std::vector inputs; - for (const auto& it : out_stream->output_buffers()) { - inputs.push_back(StorageByteBuffer::reference_buffer(it, 0, it->limit())); - } - - std::vector offsets; - offsets.push_back(0); - offsets.push_back(59); // if lzo, this should be 57 - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), - lz4_decompress, out_stream->get_total_buffer_size()); - - char data; - for (int32_t i = 0; i < OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE; i++) { - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - } - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - - EXPECT_NE(in_stream->read(&data), Status::OK()); - - SAFE_DELETE(in_stream); - SAFE_DELETE(out_stream); - for (auto input : inputs) { - delete input; - } -} - -TEST(TestStream, CompressOutStream3) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, lz4_compress); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor != nullptr); - - for (int32_t i = 0; i < OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE; i++) { - out_stream->write(0x5a); - } - char write_data[100]; - for (int32_t i = 0; i < sizeof(write_data); i++) { - write_data[i] = 0x5a; - } - out_stream->write(write_data, sizeof(write_data)); - out_stream->flush(); - - std::vector inputs; - for (const auto& it : out_stream->output_buffers()) { - inputs.push_back(StorageByteBuffer::reference_buffer(it, 0, it->limit())); - } - - std::vector offsets; - offsets.push_back(0); - offsets.push_back(57); - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), - lz4_decompress, out_stream->get_total_buffer_size()); - - char data; - for (int32_t i = 0; i < OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE; i++) { - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - } - for (int32_t i = 0; i < sizeof(write_data); i++) { - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, write_data[i]); - } - - EXPECT_NE(in_stream->read(&data), Status::OK()); - - SAFE_DELETE(in_stream); - SAFE_DELETE(out_stream); - for (auto input : inputs) { - delete input; - } -} - -//test for _slice() in [while (len > 0 && m_current_range < m_inputs.size())] -TEST(TestStream, CompressOutStream4) { - // write data - OutStream* out_stream = new (std::nothrow) OutStream(18, lz4_compress); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor != nullptr); - - for (int32_t i = 0; i < 15; i++) { - out_stream->write(0x5a); - } - out_stream->_spill(); - - for (int32_t i = 0; i < 12; i++) { - out_stream->write(0x5a); - } - for (int32_t i = 0; i < 6; i++) { - out_stream->write(i); - } - out_stream->flush(); - - std::vector inputs; - for (const auto& it : out_stream->output_buffers()) { - inputs.push_back(StorageByteBuffer::reference_buffer(it, 0, it->limit())); - } - - std::vector offsets; - offsets.push_back(0); - offsets.push_back(16); - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), - lz4_decompress, out_stream->get_total_buffer_size()); - - char data; - for (int32_t i = 0; i < 15; i++) { - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - } - - for (int32_t i = 0; i < 12; i++) { - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - } - - for (int32_t i = 0; i < 6; i++) { - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, i); - } - - EXPECT_NE(in_stream->read(&data), Status::OK()); - - SAFE_DELETE(in_stream); - SAFE_DELETE(out_stream); - for (auto input : inputs) { - delete input; - } -} - -TEST(TestStream, CompressMassOutStream) { - // write data - OutStream* out_stream = new (std::nothrow) OutStream(100, lz4_compress); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor != nullptr); - - for (int32_t i = 0; i < 100; i++) { - out_stream->write(0x5a); - } - //out_stream->write(0); - - for (int32_t i = 0; i < 100; i++) { - out_stream->write(i); - } - //out_stream->write(100); - out_stream->flush(); - - std::vector inputs; - for (const auto& it : out_stream->output_buffers()) { - inputs.push_back(StorageByteBuffer::reference_buffer(it, 0, it->limit())); - } - std::vector offsets; - offsets.push_back(0); - offsets.push_back(19); // if lzo, this should be 17 - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), - lz4_decompress, out_stream->get_total_buffer_size()); - SAFE_DELETE(out_stream); - - char data; - for (int32_t i = 0; i < 100; i++) { - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - } - for (int32_t i = 0; i < 100; i++) { - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, i); - } - - EXPECT_NE(in_stream->read(&data), Status::OK()); - - SAFE_DELETE(in_stream); - for (auto input : inputs) { - delete input; - } -} - -TEST(TestStream, CompressInStream) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, lz4_compress); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor != nullptr); - - char* write_data = new char[OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE]; - memset(write_data, 0x5a, OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); - - out_stream->write(write_data, OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); - out_stream->flush(); - - // read data - std::vector inputs; - std::vector::const_iterator it = out_stream->output_buffers().begin(); - EXPECT_NE(it, out_stream->output_buffers().end()); - StorageByteBuffer* tmp_byte_buffer = - StorageByteBuffer::reference_buffer(*it, 0, (*it)->capacity()); - inputs.push_back(tmp_byte_buffer); - - std::vector offsets; - offsets.assign(inputs.size(), 0); - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), - lz4_decompress, out_stream->get_total_buffer_size()); - EXPECT_EQ(in_stream->available(), OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); - char data; - for (int32_t i = 0; i < OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE - 1; ++i) { - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - } - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5a); - EXPECT_NE(in_stream->read(&data), Status::OK()); - - SAFE_DELETE_ARRAY(write_data); - SAFE_DELETE(out_stream); - SAFE_DELETE(in_stream); - for (auto input : inputs) { - delete input; - } -} - -TEST(TestStream, SeekUncompress) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, nullptr); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor == nullptr); - - out_stream->write(0x5a); - - PositionEntryWriter index_entry; - out_stream->get_position(&index_entry); - out_stream->write(0x5b); - EXPECT_EQ(index_entry.positions_count(), 2); - EXPECT_EQ(index_entry.positions(0), 0); - EXPECT_EQ(index_entry.positions(1), 1); - out_stream->flush(); - - // read data - std::vector inputs; - std::vector::const_iterator it = out_stream->output_buffers().begin(); - EXPECT_NE(it, out_stream->output_buffers().end()); - StorageByteBuffer* tmp_byte_buffer = - StorageByteBuffer::reference_buffer(*it, 0, (*it)->capacity()); - inputs.push_back(tmp_byte_buffer); - - std::vector offsets; - offsets.assign(inputs.size(), 0); - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), nullptr, - out_stream->get_total_buffer_size()); - EXPECT_EQ(in_stream->available(), 2); - - char buffer[256]; - index_entry.write_to_buffer(buffer); - StreamIndexHeader header; - header.position_format = index_entry.positions_count(); - header.statistic_format = OLAP_FIELD_TYPE_TINYINT; - PositionEntryReader entry; - entry.init(&header, OLAP_FIELD_TYPE_TINYINT, false); - entry.attach(buffer); - PositionProvider position(&entry); - - EXPECT_EQ(entry.positions_count(), 2); - EXPECT_EQ(entry.positions(0), 0); - EXPECT_EQ(entry.positions(1), 1); - - in_stream->seek(&position); - char data; - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5b); - SAFE_DELETE(out_stream); - SAFE_DELETE(in_stream); - for (auto input : inputs) { - delete input; - } -} - -TEST(TestStream, SkipUncompress) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, nullptr); - EXPECT_TRUE(out_stream != nullptr); - EXPECT_TRUE(out_stream->_compressor == nullptr); - - char write_data[] = {0x5a, 0x5b, 0x5c, 0x5d}; - for (int32_t i = 0; i < sizeof(write_data); i++) { - out_stream->write(write_data[i]); - } - out_stream->write(0x5e); - out_stream->flush(); - - // read data - std::vector inputs; - std::vector::const_iterator it = out_stream->output_buffers().begin(); - EXPECT_NE(it, out_stream->output_buffers().end()); - StorageByteBuffer* tmp_byte_buffer = - StorageByteBuffer::reference_buffer(*it, 0, (*it)->capacity()); - inputs.push_back(tmp_byte_buffer); - - std::vector offsets; - offsets.assign(inputs.size(), 0); - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), nullptr, - out_stream->get_total_buffer_size()); - EXPECT_EQ(in_stream->available(), sizeof(write_data) + 1); - in_stream->skip(sizeof(write_data) - 1); - char data; - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, write_data[sizeof(write_data) - 1]); - SAFE_DELETE(out_stream); - SAFE_DELETE(in_stream); - for (auto input : inputs) { - delete input; - } -} - -TEST(TestStream, SeekCompress) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, lz4_compress); - EXPECT_TRUE(out_stream != nullptr); - - for (int32_t i = 0; i < 10; i++) { - out_stream->write(0x5a); - } - - PositionEntryWriter index_entry; - out_stream->get_position(&index_entry); - out_stream->write(0x5b); - EXPECT_EQ(index_entry.positions_count(), 2); - EXPECT_EQ(index_entry.positions(0), 0); - EXPECT_EQ(index_entry.positions(1), 10); - out_stream->flush(); - - // read data - std::vector inputs; - std::vector::const_iterator it = out_stream->output_buffers().begin(); - EXPECT_NE(it, out_stream->output_buffers().end()); - StorageByteBuffer* tmp_byte_buffer = - StorageByteBuffer::reference_buffer(*it, 0, (*it)->capacity()); - inputs.push_back(tmp_byte_buffer); - - std::vector offsets; - offsets.assign(inputs.size(), 0); - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), - lz4_decompress, out_stream->get_total_buffer_size()); - //EXPECT_EQ(in_stream->available(), 2); - char buffer[256]; - index_entry.write_to_buffer(buffer); - StreamIndexHeader header; - header.position_format = index_entry.positions_count(); - header.statistic_format = OLAP_FIELD_TYPE_TINYINT; - PositionEntryReader entry; - entry.init(&header, OLAP_FIELD_TYPE_TINYINT, false); - entry.attach(buffer); - - PositionProvider position(&entry); - in_stream->seek(&position); - char data; - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5b); - SAFE_DELETE(out_stream); - SAFE_DELETE(in_stream); - for (auto input : inputs) { - delete input; - } -} - -TEST(TestStream, SkipCompress) { - // write data - OutStream* out_stream = - new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, lz4_compress); - EXPECT_TRUE(out_stream != nullptr); - - for (int32_t i = 0; i < 10; i++) { - out_stream->write(0x5a); - } - out_stream->write(0x5e); - out_stream->flush(); - - // read data - std::vector inputs; - std::vector::const_iterator it = out_stream->output_buffers().begin(); - EXPECT_NE(it, out_stream->output_buffers().end()); - StorageByteBuffer* tmp_byte_buffer = - StorageByteBuffer::reference_buffer(*it, 0, (*it)->capacity()); - inputs.push_back(tmp_byte_buffer); - - std::vector offsets; - offsets.assign(inputs.size(), 0); - InStream* in_stream = - new (std::nothrow) InStream(&inputs, offsets, out_stream->get_stream_length(), - lz4_decompress, out_stream->get_total_buffer_size()); - - in_stream->skip(10); - char data; - EXPECT_EQ(in_stream->read(&data), Status::OK()); - EXPECT_EQ(data, 0x5e); - - SAFE_DELETE(out_stream); - SAFE_DELETE(in_stream); - for (auto input : inputs) { - delete input; - } -} - -class TestRunLengthByte : public testing::Test { -public: - TestRunLengthByte() {} - - virtual ~TestRunLengthByte() {} - - virtual void SetUp() { - EXPECT_EQ(system("mkdir -p ./ut_dir"), 0); - EXPECT_EQ(system("rm -rf ./ut_dir/tmp_file"), 0); - _out_stream = new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, nullptr); - EXPECT_TRUE(_out_stream != nullptr); - _writer = new (std::nothrow) RunLengthByteWriter(_out_stream); - EXPECT_TRUE(_writer != nullptr); - } - - virtual void TearDown() { - SAFE_DELETE(_reader); - SAFE_DELETE(_out_stream); - SAFE_DELETE(_writer); - SAFE_DELETE(_shared_buffer); - SAFE_DELETE(_stream); - } - - void CreateReader() { - EXPECT_EQ(Status::OK(), - helper.open_with_mode(_file_path.c_str(), O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR)); - _out_stream->write_to_file(&helper, 0); - helper.close(); - - EXPECT_EQ(Status::OK(), - helper.open_with_mode(_file_path.c_str(), O_RDONLY, S_IRUSR | S_IWUSR)); - - _shared_buffer = StorageByteBuffer::create(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE + - sizeof(StreamHead)); - EXPECT_TRUE(_shared_buffer != nullptr); - - _stream = new (std::nothrow) - ReadOnlyFileStream(&helper, &_shared_buffer, 0, helper.length(), nullptr, - OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, &_stats); - EXPECT_EQ(Status::OK(), _stream->init()); - - _reader = new (std::nothrow) RunLengthByteReader(_stream); - EXPECT_TRUE(_reader != nullptr); - } - - RunLengthByteReader* _reader; - OutStream* _out_stream; - RunLengthByteWriter* _writer; - FileHandler helper; - StorageByteBuffer* _shared_buffer; - ReadOnlyFileStream* _stream; - OlapReaderStatistics _stats; - - std::string _file_path = "./ut_dir/tmp_file"; -}; - -TEST_F(TestRunLengthByte, ReadWriteOneByte) { - _writer->write(0x5a); - _writer->flush(); - CreateReader(); - - EXPECT_TRUE(_reader->has_next()); - char value = 0xff; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 0X5A); - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthByte, ReadWriteMultiBytes) { - // write data - char write_data[] = {0x5a, 0x5b, 0x5c, 0x5d}; - for (int32_t i = 0; i < sizeof(write_data); i++) { - _writer->write(write_data[i]); - } - - _writer->flush(); - - // the stream contain head, control byte and four byte literal - EXPECT_EQ(_out_stream->get_stream_length(), sizeof(StreamHead) + 1 + 4); - - // read data - CreateReader(); - - for (int32_t i = 0; i < sizeof(write_data); i++) { - EXPECT_TRUE(_reader->has_next()); - char value = 0xff; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthByte, ReadWriteSameBytes) { - // write data - char write_data[] = {0x5a, 0x5a, 0x5a, 0x5a}; - for (int32_t i = 0; i < sizeof(write_data); i++) { - _writer->write(write_data[i]); - } - - _writer->flush(); - - // the stream contain head, control byte(4-3) and one byte literal - EXPECT_EQ(_out_stream->get_stream_length(), sizeof(StreamHead) + 1 + 1); - - // read data - CreateReader(); - - for (int32_t i = 0; i < sizeof(write_data); i++) { - EXPECT_TRUE(_reader->has_next()); - char value = 0xff; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthByte, Seek) { - // write data - char write_data[] = {0x5a, 0x5b, 0x5c, 0x5d}; - for (int32_t i = 0; i < sizeof(write_data); i++) { - _writer->write(write_data[i]); - } - - PositionEntryWriter index_entry; - _writer->get_position(&index_entry); - _writer->write(0x5e); - - _writer->write(0x5f); - _writer->write(0x60); - _writer->write(0x61); - - _writer->flush(); - - // read data - CreateReader(); - - char buffer[256]; - index_entry.write_to_buffer(buffer); - StreamIndexHeader header; - header.position_format = index_entry.positions_count(); - header.statistic_format = OLAP_FIELD_TYPE_TINYINT; - PositionEntryReader entry; - entry.init(&header, OLAP_FIELD_TYPE_TINYINT, false); - entry.attach(buffer); - - PositionProvider position(&entry); - _reader->seek(&position); - char value = 0xff; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 0x5e); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 0x5f); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 0x60); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 0x61); -} - -TEST_F(TestRunLengthByte, Skip) { - // write data - char write_data[] = {0x5a, 0x5b, 0x5c, 0x5d}; - for (int32_t i = 0; i < sizeof(write_data); i++) { - _writer->write(write_data[i]); - } - _writer->write(0x5e); - _writer->flush(); - - // read data - CreateReader(); - - _reader->skip(sizeof(write_data) - 1); - char value = 0xff; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[sizeof(write_data) - 1]); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 0x5e); -} - -} // namespace doris diff --git a/be/test/olap/run_length_integer_test.cpp b/be/test/olap/run_length_integer_test.cpp deleted file mode 100644 index 11149d4cf4..0000000000 --- a/be/test/olap/run_length_integer_test.cpp +++ /dev/null @@ -1,1479 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include "olap/byte_buffer.h" -#include "olap/in_stream.h" -#include "olap/out_stream.h" -#include "olap/rowset/run_length_integer_reader.h" -#include "olap/rowset/run_length_integer_writer.h" -#include "olap/stream_index_reader.h" -#include "olap/stream_index_writer.h" -#include "util/logging.h" - -namespace doris { - -class TestRunLengthUnsignInteger : public testing::Test { -public: - TestRunLengthUnsignInteger() {} - - virtual ~TestRunLengthUnsignInteger() {} - - virtual void SetUp() { - EXPECT_EQ(system("mkdir -p ./ut_dir"), 0); - EXPECT_EQ(system("rm -rf ./ut_dir/tmp_file"), 0); - _out_stream = new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, nullptr); - EXPECT_TRUE(_out_stream != nullptr); - _writer = new (std::nothrow) RunLengthIntegerWriter(_out_stream, false); - EXPECT_TRUE(_writer != nullptr); - } - - virtual void TearDown() { - SAFE_DELETE(_reader); - SAFE_DELETE(_out_stream); - SAFE_DELETE(_writer); - SAFE_DELETE(_shared_buffer); - SAFE_DELETE(_stream); - } - - void CreateReader() { - EXPECT_EQ(Status::OK(), - helper.open_with_mode(_file_path.c_str(), O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR)); - _out_stream->write_to_file(&helper, 0); - helper.close(); - - EXPECT_EQ(Status::OK(), - helper.open_with_mode(_file_path.c_str(), O_RDONLY, S_IRUSR | S_IWUSR)); - - _shared_buffer = StorageByteBuffer::create(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE + - sizeof(StreamHead)); - EXPECT_TRUE(_shared_buffer != nullptr); - - _stream = new (std::nothrow) - ReadOnlyFileStream(&helper, &_shared_buffer, 0, helper.length(), nullptr, - OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, &_stats); - EXPECT_EQ(Status::OK(), _stream->init()); - - _reader = new (std::nothrow) RunLengthIntegerReader(_stream, false); - EXPECT_TRUE(_reader != nullptr); - } - - RunLengthIntegerReader* _reader; - OutStream* _out_stream; - RunLengthIntegerWriter* _writer; - FileHandler helper; - StorageByteBuffer* _shared_buffer; - ReadOnlyFileStream* _stream; - OlapReaderStatistics _stats; - - std::string _file_path = "./ut_dir/tmp_file"; -}; - -TEST_F(TestRunLengthUnsignInteger, ReadWriteOneInteger) { - // write data - EXPECT_EQ(Status::OK(), _writer->write(100)); - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 100); - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthUnsignInteger, ReadWriteMultiInteger) { - // write data - int64_t write_data[] = {100, 102, 105, 106}; - for (int32_t i = 0; i < 4; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 4; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthUnsignInteger, seek) { - // write data - int64_t write_data[] = {100, 102, 105, 106}; - for (int32_t i = 0; i < 4; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - - PositionEntryWriter index_entry; - _writer->get_position(&index_entry, false); - _writer->write(107); - - _writer->write(108); - _writer->write(109); - _writer->write(110); - - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - PositionEntryReader entry; - entry._positions = index_entry._positions; - entry._positions_count = index_entry._positions_count; - entry._statistics.init(OLAP_FIELD_TYPE_INT, false); - - PositionProvider position(&entry); - _reader->seek(&position); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 107); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 108); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 109); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 110); -} - -TEST_F(TestRunLengthUnsignInteger, skip) { - // write data - int64_t write_data[] = {100, 102, 105, 106}; - for (int32_t i = 0; i < 4; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - _reader->skip(2); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 105); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 106); - EXPECT_NE(Status::OK(), _reader->next(&value)); -} - -TEST_F(TestRunLengthUnsignInteger, ShortRepeatEncoding) { - // write data - int64_t write_data[] = {100, 100, 100, 100}; - for (int32_t i = 0; i < 4; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 4; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthUnsignInteger, ShortRepeatEncoding2) { - // write data - int64_t write_data[] = {876012345678912, 876012345678912, 876012345678912, 876012345678912}; - for (int32_t i = 0; i < 4; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 4; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthUnsignInteger, ShortRepeatEncoding3) { - // write data - int64_t write_data[] = {876012345678912}; - for (int32_t i = 0; i < 1; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 1; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthUnsignInteger, DirectEncoding) { - // write data - int64_t write_data[] = {1703, 6054, 8760, 902}; - for (int32_t i = 0; i < 4; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 4; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthUnsignInteger, DirectEncoding2) { - // write data - int64_t write_data[] = {1703, 6054, 876012345678912, 902, 9292, 184932, 873624, 827364, 999, 8}; - for (int32_t i = 0; i < 10; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 10; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthUnsignInteger, PatchedBaseEncoding1) { - // write data - int64_t write_data[] = { - 1703, 6054, 876012345678912, 902, 9292, 184932, 873624, 827364, 999, 8, - 1, 3323, 432232523, 90982, 9, 223234, 5, 44, 5, 3}; - for (int32_t i = 0; i < 20; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 20; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthUnsignInteger, PatchedBaseEncoding2) { - // write data - int64_t write_data[] = { - 1703, 6054, 902, 9292, 184932, 873624, 827364, 999, 8, 1, - 3323, 432232523, 90982, 9, 223234, 5, 876012345678912, 44, 5, 3}; - for (int32_t i = 0; i < 20; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 20; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -class TestRunLengthSignInteger : public testing::Test { -public: - TestRunLengthSignInteger() {} - - virtual ~TestRunLengthSignInteger() {} - - virtual void SetUp() { - EXPECT_EQ(system("mkdir -p ./ut_dir"), 0); - EXPECT_EQ(system("rm ./ut_dir/tmp_file"), 0); - _out_stream = new (std::nothrow) OutStream(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, nullptr); - EXPECT_TRUE(_out_stream != nullptr); - _writer = new (std::nothrow) RunLengthIntegerWriter(_out_stream, true); - EXPECT_TRUE(_writer != nullptr); - } - - virtual void TearDown() { - SAFE_DELETE(_reader); - SAFE_DELETE(_out_stream); - SAFE_DELETE(_writer); - SAFE_DELETE(_shared_buffer); - SAFE_DELETE(_stream); - } - - void CreateReader() { - EXPECT_EQ(Status::OK(), - helper.open_with_mode(_file_path.c_str(), O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR)); - _out_stream->write_to_file(&helper, 0); - helper.close(); - - EXPECT_EQ(Status::OK(), - helper.open_with_mode(_file_path.c_str(), O_RDONLY, S_IRUSR | S_IWUSR)); - - _shared_buffer = StorageByteBuffer::create(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE + - sizeof(StreamHead)); - EXPECT_TRUE(_shared_buffer != nullptr); - - _stream = new (std::nothrow) - ReadOnlyFileStream(&helper, &_shared_buffer, 0, helper.length(), nullptr, - OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE, &_stats); - EXPECT_EQ(Status::OK(), _stream->init()); - - _reader = new (std::nothrow) RunLengthIntegerReader(_stream, true); - EXPECT_TRUE(_reader != nullptr); - } - - RunLengthIntegerReader* _reader; - OutStream* _out_stream; - RunLengthIntegerWriter* _writer; - FileHandler helper; - StorageByteBuffer* _shared_buffer; - ReadOnlyFileStream* _stream; - OlapReaderStatistics _stats; - std::string _file_path = "./ut_dir/tmp_file"; -}; - -TEST_F(TestRunLengthSignInteger, ReadWriteOneInteger) { - // write data - EXPECT_EQ(Status::OK(), _writer->write(100)); - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 100); - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthSignInteger, ReadWriteOneInteger2) { - // write data - EXPECT_EQ(Status::OK(), _writer->write(1234567800)); - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 1234567800); - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthSignInteger, ReadWriteMultiInteger) { - // write data - int64_t write_data[] = {100, 101, 104, 107}; - for (int32_t i = 0; i < 4; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 4; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthSignInteger, seek) { - // write data - int64_t write_data[] = {100, 102, 105, 106}; - for (int32_t i = 0; i < 4; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - - PositionEntryWriter index_entry; - _writer->get_position(&index_entry, false); - _writer->write(107); - - _writer->write(108); - _writer->write(109); - _writer->write(110); - - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - PositionEntryReader entry; - entry._positions = index_entry._positions; - entry._positions_count = index_entry._positions_count; - entry._statistics.init(OLAP_FIELD_TYPE_INT, false); - - PositionProvider position(&entry); - _reader->seek(&position); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 107); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 108); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 109); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 110); -} - -TEST_F(TestRunLengthSignInteger, skip) { - // write data - int64_t write_data[] = {100, 102, 105, 106}; - for (int32_t i = 0; i < 4; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - _reader->skip(2); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 105); - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, 106); - EXPECT_NE(Status::OK(), _reader->next(&value)); -} - -TEST_F(TestRunLengthSignInteger, ShortRepeatEncoding) { - // write data - int64_t write_data[] = {-100, -100, -100, -100}; - for (int32_t i = 0; i < 4; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 4; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthSignInteger, DirectEncoding) { - // write data - int64_t write_data[] = {-1703, -6054, -8760, -902}; - for (int32_t i = 0; i < 4; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 4; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthUnsignInteger, ReadWriteMassInteger) { - // write data - for (int64_t i = 0; i < 100000; i++) { - EXPECT_EQ(Status::OK(), _writer->write(i)); - } - - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int64_t i = 0; i < 100000; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, i); - } -} - -TEST_F(TestRunLengthSignInteger, PatchedBaseEncoding1) { - // write data - int64_t write_data[] = { - 1703, 6054, -876012345678912, 902, 9292, 184932, 873624, 827364, 999, 8, - 1, 3323, 432232523, -90982, 9, 223234, 5, 44, 5, 3}; - for (int32_t i = 0; i < 20; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 20; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthSignInteger, PatchedBaseEncoding2) { - // write data - int64_t write_data[] = { - -1703, -6054, -876012345678912, -902, -9292, -184932, -873624, -827364, -999, -8, - -1, -3323, -432232523, -90982, -9, -223234, -5, -44, -5, -3}; - for (int32_t i = 0; i < 20; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 20; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -// case for T in [if ((base & mask) != 0)] -TEST_F(TestRunLengthSignInteger, PatchedBaseEncoding3) { - // write data - int64_t write_data[] = { - 1703, 6054, 876012345678912, 902, 9292, 184932, 873624, 827364, 999, 888, - -300, 3323, 432232523, -90982, 450, 223234, 690, 444, 555, 333}; - for (int32_t i = 0; i < 20; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 20; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -// test fo gap > 255 -TEST_F(TestRunLengthSignInteger, PatchedBaseEncoding4) { - // write data - int64_t write_data[] = {300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 300, - 6054, - 902, - 9292, - 184932, - 873624, - 827364, - 999, - 888, - 1703, - 3323, - 432232523, - 90982, - 450, - 223234, - 690, - 444, - 555, - 333, - 232, - 876012345678912}; - - for (int32_t i = 0; i < 281; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 281; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -// test for [if (max_gap == 511)] -TEST_F(TestRunLengthSignInteger, PatchedBaseEncoding5) { - // write data - int64_t write_data[] = { - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 432232523, 90982, 450, 223234, 690, 444, 555, 333, 232, - 300, 6054, 902, 9292, 184932, 873624, 827364, 999, 888, 1703, - 3323, 876012345678912}; - - for (int32_t i = 0; i < 512; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 512; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -// this case use to test large negative number. -// The minimum of data sequence is -84742859065569280, -// the positive form is 84742859065569280. -// It is a 57 bit width integer and used 8 byte to encoding it. -// The byte number is encoding as (8-1) = 7, in 111 binary form. -TEST_F(TestRunLengthSignInteger, PatchedBaseEncoding6) { - // write data - int64_t write_data[] = { - -17887939293638656, -15605417571528704, -15605417571528704, -13322895849418752, - -13322895849418752, -84742859065569280, -15605417571528704, -13322895849418752, - -13322895849418752, -15605417571528704, -13322895849418752, -13322895849418752, - -15605417571528704, -15605417571528704, -13322895849418752, -13322895849418752, - -15605417571528704, -15605417571528704, -13322895849418752, -13322895849418752, - -11040374127308800, -15605417571528704, -13322895849418752, -13322895849418752, - -15605417571528704, -15605417571528704, -13322895849418752, -13322895849418752, - -15605417571528704, -13322895849418752}; - for (int32_t i = 0; i < 30; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 30; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthSignInteger, DirectEncodingForDeltaOverflows1) { - // write data - int64_t write_data[] = {4513343538618202711, 2911390882471569739, -9181829309989854913}; - for (int32_t i = 0; i < 3; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 3; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -TEST_F(TestRunLengthSignInteger, DirectEncodingForDeltaOverflows2) { - // write data - int64_t write_data[388] = {-10655366027330, - -8, - -8, - 17062988027150, - 33791751739027, - 11226586316144, - 28085455533984, - 3809037113342, - 4519464927340, - 7057442420685, - 14594401530747, - 51297588056932, - 22756638885750, - 33503880074496, - 37462526094279, - 2768357503914, - 10013218679000, - 9782540355504, - 14191567385952, - 7500449144168, - 52986427404273, - 34061223404544, - 41158355187480, - 30231130071318, - 59156080720623, - 5229947056790, - 15537038856150, - 63126198469920, - 3285383272432, - 35686846610326, - 1408668367602, - 3506863240250, - 38486027140122, - 74694461322614, - 5631517623023, - 1071278551640, - 60280539267507, - 55998488499421, - 15148183576420, - 9345266527488, - 92610339730326, - 4760991023496, - 5809785829925, - 4727943945615, - 45024286170204, - 9430751906642, - 9104898240000, - 1310787469233, - 19485266345637, - 5211360444059, - 13671501255270, - 29025205412982, - 16589693255688, - 5550737611815, - 55740247673420, - 4416290119582, - 5285248047529, - 36719742782763, - 25526635256587, - 469124468760, - 23302498025820, - 15213356460134, - 39178120907751, - 3663022204884, - 12383207229348, - 26476189044888, - 3016879997537, - 40542785494057, - 27113842491675, - 22895214591884, - 10417880862400, - 41668786286070, - 9444820760988, - 4868267611350, - 95075547166521, - 31102600536052, - 3984468754330, - 18604973210352, - 14995789569131, - 31920766994781, - 1683419132059, - 23882256010288, - 31188556951272, - 21582097820895, - 3392728705744, - 49134949965522, - 71017521225855, - 2954016986940, - 39104716002146, - 849942233098, - 13757131707844, - 134503055610, - 3609965787162, - 5092516507104, - 15030981322468, - 13161768499404, - 21216129648768, - 37101803372970, - 37101803372970, - 2847403596440, - 8709398658448, - 17409277451720, - 1765292210160, - 12794111172864, - 59395030132284, - 25700056122869, - 3617518482435, - 10096266209660, - 603359572452, - 14638639926830, - 47440669759700, - 541260539624, - 30978788863282, - 696405704526, - 18657155830038, - 8940665549457, - 48888857992884, - 61600635324852, - 62348216229282, - 36737996974800, - 23412718588072, - 39733566705126, - 24349931484564, - 8726400511488, - 53728914318592, - 33230740766440, - 5598927206656, - 62807536003975, - 59361934257815, - 12281154081596, - 60614594920391, - 55896003656892, - 12410592819768, - 26900986612464, - 27212849083404, - 4081661384017, - 62807128313880, - 10390915208885, - 19067862634200, - 54951638814317, - 6813715526592, - 55668241923840, - 10385258308992, - 449478352872, - 5773207377695, - 7951085473750, - 8075739133609, - 3474440407650, - 12804432172290, - 54059206249468, - 41060766380882, - 32370112924240, - 14089711019310, - 20756475257231, - 12027575432880, - 18022828219993, - 2876759174844, - 3428645976585, - 10428044444020, - 83396752046512, - 53899236887790, - 2436412807160, - 58240905226752, - 49366625937140, - 37105861727280, - 8612969542385, - 73033645852629, - 43369958815872, - 7261355608605, - 39209192629850, - 52609810587480, - 43476360891080, - 5041062521194, - 38576540661093, - 68017667314375, - 13432761019896, - 5464942964090, - 43050750861745, - 19350242905500, - 75097467074637, - 2556614854921, - 37718643408480, - 213620237510, - 6724311130250, - 25457500052952, - 489516376431, - 11514465298138, - 717063515668, - 26446350217560, - 6756064528605, - 33085247961173, - 42923416365802, - 24304783775635, - 38572198977000, - 30768510004640, - 15169698546850, - 11126988953456, - 19972411935195, - 3128825910344, - 41008728739248, - 9593284980500, - 71039982484735, - 32594940903700, - 1833067856312, - 30457700317302, - 1581398691678, - 20232754466668, - 1176823804850, - 15276320639355, - 8945917123286, - 18919341601824, - 21678108350498, - 11552006037852, - 23919732805330, - 11335293921846, - 42481509203406, - 2122540078032, - 12644802041058, - 57724114297590, - 4260375471943, - 22854679188447, - 1679805810144, - 20920530725100, - 79680508970004, - 4822078070025, - 21021229791528, - 51167989737960, - 165090524909, - 25873285104027, - 46513810563360, - 3575825256938, - 34059047115978, - 19806749124512, - 78800618138484, - 28322771215728, - 1706728554744, - 3278090395233, - 23320617333774, - 12703326279678, - 2607629313708, - 2539395442752, - 28015716713200, - 10300326647150, - 30128043086820, - 54595159345500, - 13060160042787, - 10655366027330, - 17062988027150, - 57029084625870, - 7756250581905, - 76771656574558, - 4510750676015, - 21874347427140, - 3200391993042, - 44103009370141, - 57277463673026, - 1018187154816, - 46506687921519, - 34421816854665, - 2676955231764, - 47301429307040, - 3547510780001, - 46639125075628, - 17055804956334, - 1646477501284, - 2478078885625, - 54398069002768, - 31340183949950, - 49221902712600, - 36419457248232, - 13122452465580, - 10185432189606, - 74024622661290, - 17301062652176, - 76970388164508, - 62373552781812, - 6841468343550, - 59793356788000, - 19225236462716, - 32388898413606, - 6187897297975, - 6187897297975, - 17420000862880, - 53547183179125, - 28894759120032, - 24398977402333, - 24813328904455, - 28037560402170, - 8374290597420, - 4350223211799, - 17455283786332, - 63688413624321, - 57562099045760, - 33975883005192, - 18354742508025, - 16639404261926, - 58997385417308, - 6803575242456, - 37764812164226, - 84538481394123, - 4965525257607, - 121662980560, - 7029409232880, - 31220248346100, - 68137270368096, - 26037399196380, - 44447384771070, - 10695793870605, - 25448023325214, - 92664298126772, - 7339073401512, - 8807332093230, - 10372293259380, - 54867075540795, - 497219901781, - 69741834972537, - 18310069153200, - 3089795662998, - 2555983074092, - 27136123982275, - 45588802330695, - 784447977666, - 8592621213364, - 16042005794780, - 48914398341200, - 39352870745600, - 13745169930000, - 22896230019150, - 3674063255112, - 6936424673580, - 20715482731050, - 962371484361, - 59719596050289, - 24836189653956, - 27890432977132, - 10416225687015, - 37371930733881, - 16641855346480, - 1243415213082, - 7024777702423, - 15056326461072, - 237522450780, - 8654097255216, - 1935692634400, - 48149720268700, - 22018394580560, - 4353414553674, - 233342798248, - 1689195367328, - 73349430633813, - 16579193709760, - 47254775444604, - 5751774863052, - 4168272591177, - 56466991266759, - 10403807615696, - 30368152985625, - 23805779365764, - 1751347115141, - 686411646452, - 10942582463904, - 37051912941344, - 66573514373520, - 5193629914880, - 10276936700310, - 45333683755358, - 5369872937250, - 19870592423202, - 44901818802729, - 1984995522276, - 38210789175216, - 13140877390832, - 36472949862774, - 11003144677000, - 34026969817136, - 68246909413281, - 9480783308290, - 8927412760982, - 44662728145200, - 4559499822921, - 13378660270086, - 50432177305629, - 33536986417717, - 8548419242610, - 43216481118384, - 37549778463076, - 2, - 9223372036854775807}; - - for (int32_t i = 0; i < 388; i++) { - EXPECT_EQ(Status::OK(), _writer->write(write_data[i])); - } - EXPECT_EQ(Status::OK(), _writer->flush()); - - // read data - CreateReader(); - - for (int32_t i = 0; i < 388; i++) { - EXPECT_TRUE(_reader->has_next()); - int64_t value = 0; - EXPECT_EQ(Status::OK(), _reader->next(&value)); - EXPECT_EQ(value, write_data[i]); - } - - EXPECT_FALSE(_reader->has_next()); -} - -} // namespace doris diff --git a/be/test/olap/schema_change_test.cpp b/be/test/olap/schema_change_test.cpp deleted file mode 100644 index e9383f5763..0000000000 --- a/be/test/olap/schema_change_test.cpp +++ /dev/null @@ -1,1462 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -#include "olap/schema_change.h" - -#include - -#include "olap/byte_buffer.h" -#include "olap/field.h" -#include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/row_block.h" -#include "olap/row_cursor.h" -#include "olap/rowset/column_reader.h" -#include "olap/rowset/column_writer.h" -#include "olap/stream_name.h" -#include "runtime/mem_pool.h" -#include "runtime/vectorized_row_batch.h" -#include "util/logging.h" - -using std::string; - -namespace doris { - -class TestColumn : public testing::Test { -public: - TestColumn() : _column_writer(nullptr), _column_reader(nullptr), _stream_factory(nullptr) { - _offsets.clear(); - _map_in_streams.clear(); - _present_buffers.clear(); - _data_buffers.clear(); - _second_buffers.clear(); - _dictionary_buffers.clear(); - _length_buffers.clear(); - _mem_tracker.reset(new MemTracker(-1)); - _mem_pool.reset(new MemPool(_mem_tracker.get())); - } - - virtual ~TestColumn() { - SAFE_DELETE(_column_writer); - SAFE_DELETE(_column_reader); - SAFE_DELETE(_stream_factory); - } - - virtual void SetUp() { - _offsets.push_back(0); - _stream_factory = new (std::nothrow) - OutStreamFactory(COMPRESS_LZ4, OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE); - EXPECT_TRUE(_stream_factory != nullptr); - config::column_dictionary_key_ratio_threshold = 30; - config::column_dictionary_key_size_threshold = 1000; - } - - virtual void TearDown() { - SAFE_DELETE(_column_writer); - SAFE_DELETE(_column_reader); - SAFE_DELETE(_stream_factory); - SAFE_DELETE(_shared_buffer); - - _offsets.clear(); - for (auto in_stream : _map_in_streams) { - delete in_stream.second; - } - _map_in_streams.clear(); - _present_buffers.clear(); - _data_buffers.clear(); - _second_buffers.clear(); - _dictionary_buffers.clear(); - _length_buffers.clear(); - } - - void create_column_writer(const TabletSchema& tablet_schema) { - _column_writer = ColumnWriter::create(0, tablet_schema, _stream_factory, 1024, - BLOOM_FILTER_DEFAULT_FPP); - EXPECT_TRUE(_column_writer != nullptr); - EXPECT_EQ(_column_writer->init(), Status::OK()); - } - - void create_column_reader(const TabletSchema& tablet_schema) { - UniqueIdEncodingMap encodings; - encodings[0] = ColumnEncodingMessage(); - encodings[0].set_kind(ColumnEncodingMessage::DIRECT); - encodings[0].set_dictionary_size(1); - create_column_reader(tablet_schema, encodings); - } - - void create_column_reader(const TabletSchema& tablet_schema, UniqueIdEncodingMap& encodings) { - UniqueIdToColumnIdMap included; - included[0] = 0; - UniqueIdToColumnIdMap segment_included; - segment_included[0] = 0; - - SAFE_DELETE(_column_reader); - _column_reader = - ColumnReader::create(0, tablet_schema, included, segment_included, encodings); - - EXPECT_TRUE(_column_reader != nullptr); - - EXPECT_EQ(system("mkdir -p ./ut_dir"), 0); - EXPECT_EQ(system("rm -f ./ut_dir/tmp_file"), 0); - - EXPECT_EQ(Status::OK(), - helper.open_with_mode("./ut_dir/tmp_file", O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR)); - std::vector off; - std::vector length; - std::vector buffer_size; - std::vector name; - - std::map::const_iterator it = _stream_factory->streams().begin(); - for (; it != _stream_factory->streams().end(); ++it) { - StreamName stream_name = it->first; - OutStream* out_stream = it->second; - std::vector* buffers = nullptr; - - if (out_stream->is_suppressed()) { - continue; - } - if (stream_name.kind() == StreamInfoMessage::ROW_INDEX) { - continue; - } else if (stream_name.kind() == StreamInfoMessage::PRESENT) { - buffers = &_present_buffers; - } else if (stream_name.kind() == StreamInfoMessage::DATA) { - buffers = &_data_buffers; - } else if (stream_name.kind() == StreamInfoMessage::SECONDARY) { - buffers = &_second_buffers; - } else if (stream_name.kind() == StreamInfoMessage::DICTIONARY_DATA) { - buffers = &_dictionary_buffers; - } else if (stream_name.kind() == StreamInfoMessage::LENGTH) { - buffers = &_length_buffers; - } else { - EXPECT_TRUE(false); - } - - EXPECT_TRUE(buffers != nullptr); - off.push_back(helper.tell()); - out_stream->write_to_file(&helper, 0); - length.push_back(out_stream->get_stream_length()); - buffer_size.push_back(out_stream->get_total_buffer_size()); - name.push_back(stream_name); - } - helper.close(); - - EXPECT_EQ(Status::OK(), - helper.open_with_mode("./ut_dir/tmp_file", O_RDONLY, S_IRUSR | S_IWUSR)); - - SAFE_DELETE(_shared_buffer); - _shared_buffer = StorageByteBuffer::create(OLAP_DEFAULT_COLUMN_STREAM_BUFFER_SIZE + - sizeof(StreamHead)); - EXPECT_TRUE(_shared_buffer != nullptr); - - for (auto in_stream : _map_in_streams) { - delete in_stream.second; - } - _map_in_streams.clear(); - - for (int i = 0; i < off.size(); ++i) { - ReadOnlyFileStream* in_stream = new (std::nothrow) - ReadOnlyFileStream(&helper, &_shared_buffer, off[i], length[i], lz4_decompress, - buffer_size[i], &_stats); - EXPECT_EQ(Status::OK(), in_stream->init()); - _map_in_streams[name[i]] = in_stream; - } - - EXPECT_EQ(_column_reader->init(&_map_in_streams, 1024, _mem_pool.get(), &_stats), - Status::OK()); - } - - void set_tablet_schema(const std::string& name, const std::string& type, - const std::string& aggregation, uint32_t length, bool is_allow_null, - bool is_key, TabletSchema* tablet_schema) { - TabletSchemaPB tablet_schema_pb; - ColumnPB* column = tablet_schema_pb.add_column(); - column->set_unique_id(0); - column->set_name(name); - column->set_type(type); - column->set_is_key(is_key); - column->set_is_nullable(is_allow_null); - column->set_length(length); - column->set_aggregation(aggregation); - tablet_schema->init_from_pb(tablet_schema_pb); - } - - void create_and_save_last_position() { - EXPECT_EQ(_column_writer->create_row_index_entry(), Status::OK()); - } - - template - void test_convert_to_varchar(const std::string& type_name, int type_size, T val, - const std::string& expected_val, Status expected_st, - int varchar_len = 255) { - TabletSchema src_tablet_schema; - set_tablet_schema("ConvertColumn", type_name, "REPLACE", type_size, false, false, - &src_tablet_schema); - create_column_writer(src_tablet_schema); - - RowCursor write_row; - write_row.init(src_tablet_schema); - RowBlock block(&src_tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - write_row.set_field_content(0, reinterpret_cast(&val), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - helper.close(); - - TabletSchema dst_tablet_schema; - set_tablet_schema("VarcharColumn", "VARCHAR", "REPLACE", varchar_len, false, false, - &dst_tablet_schema); - create_column_reader(src_tablet_schema); - RowCursor read_row; - read_row.init(dst_tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - auto st = read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), - _mem_pool.get()); - EXPECT_EQ(st, expected_st); - if (st == Status::OK()) { - std::string dst_str = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); - EXPECT_TRUE(dst_str.compare(0, expected_val.size(), expected_val) == 0); - } - - const auto* tp = get_scalar_type_info(); - st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get()); - EXPECT_EQ(st, Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); - } - - void test_convert_from_varchar(const std::string& type_name, int type_size, - const std::string& value, Status expected_st) { - TabletSchema tablet_schema; - set_tablet_schema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - Slice normal_str(value); - write_row.set_field_content(0, reinterpret_cast(&normal_str), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - helper.close(); - - TabletSchema converted_tablet_schema; - set_tablet_schema("ConvertColumn", type_name, "REPLACE", type_size, false, false, - &converted_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(converted_tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - auto st = read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), - _mem_pool.get()); - EXPECT_EQ(st, expected_st); - if (st == Status::OK()) { - std::string dst_str = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); - EXPECT_TRUE(dst_str.compare(0, value.size(), value) == 0); - } - - const auto* tp = get_scalar_type_info(); - st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get()); - EXPECT_EQ(st, Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); - } - - ColumnWriter* _column_writer; - - ColumnReader* _column_reader; - std::shared_ptr _mem_tracker; - std::unique_ptr _mem_pool; - std::unique_ptr _col_vector; - - OutStreamFactory* _stream_factory; - - std::vector _offsets; - std::vector _present_buffers; - std::vector _data_buffers; - std::vector _second_buffers; - std::vector _dictionary_buffers; - std::vector _length_buffers; - StorageByteBuffer* _shared_buffer = nullptr; - std::map _map_in_streams; - FileHandler helper; - OlapReaderStatistics _stats; -}; - -TEST_F(TestColumn, ConvertFloatToDouble) { - TabletSchema tablet_schema; - set_tablet_schema("FloatColumn", "FLOAT", "REPLACE", 4, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - float value = 1.234; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - value = 3.234; - write_row.set_field_content(0, reinterpret_cast(&value), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - TabletSchema convert_tablet_schema; - set_tablet_schema("DoubleColumn", "DOUBLE", "REPLACE", 4, false, false, &convert_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(convert_tablet_schema); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 2, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); - //float val1 = *reinterpret_cast(read_row.cell_ptr(0)); - double val2 = *reinterpret_cast(read_row.cell_ptr(0)); - - char buf[64]; - memset(buf, 0, sizeof(buf)); - sprintf(buf, "%f", val2); - char* tg; - double v2 = strtod(buf, &tg); - EXPECT_EQ(v2, 1.234); - - //test not support type - const auto* tp = get_scalar_type_info(); - Status st = read_row.convert_from(0, data, tp, _mem_pool.get()); - EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertDatetimeToDate) { - TabletSchema tablet_schema; - set_tablet_schema("DatetimeColumn", "DATETIME", "REPLACE", 8, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - std::string origin_val = "2019-11-25 19:07:00"; - val_string_array.emplace_back(origin_val); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - TabletSchema convert_tablet_schema; - set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &convert_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(convert_tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); - std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); - EXPECT_TRUE(strncmp(dest_string.c_str(), "2019-11-25", strlen("2019-11-25")) == 0); - - //test not support type - const auto* tp = get_scalar_type_info(); - Status st = read_row.convert_from(0, data, tp, _mem_pool.get()); - EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertDatetimeToDateV2) { - TabletSchema tablet_schema; - set_tablet_schema("DatetimeColumn", "DATETIME", "REPLACE", 8, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - std::string origin_val = "2019-11-25 19:07:00"; - val_string_array.emplace_back(origin_val); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - TabletSchema convert_tablet_schema; - set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &convert_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(convert_tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); - std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); - EXPECT_TRUE(strncmp(dest_string.c_str(), "2019-11-25", strlen("2019-11-25")) == 0); - - //test not support type - const auto* tp = get_scalar_type_info(); - Status st = read_row.convert_from(0, data, tp, _mem_pool.get()); - EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertDateV2ToDateTime) { - TabletSchema tablet_schema; - set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - std::string origin_val = "2019-11-25"; - val_string_array.emplace_back(origin_val); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // read data - TabletSchema convert_tablet_schema; - set_tablet_schema("DateTimeColumn", "DATETIME", "REPLACE", 8, false, false, - &convert_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(convert_tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); - std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); - EXPECT_TRUE(strncmp(dest_string.c_str(), "2019-11-25 00:00:00", - strlen("2019-11-25 00:00:00")) == 0); - - //test not support type - const auto* tp = get_scalar_type_info(); - Status st = read_row.convert_from(0, data, tp, _mem_pool.get()); - EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertDateToDatetime) { - TabletSchema tablet_schema; - set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - std::string origin_val = "2019-12-04"; - val_string_array.emplace_back(origin_val); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header_message; - EXPECT_EQ(_column_writer->finalize(&header_message), Status::OK()); - - TabletSchema convert_tablet_schema; - set_tablet_schema("DateTimeColumn", "DATETIME", "REPLACE", 8, false, false, - &convert_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(convert_tablet_schema); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.set_field_content(0, data, _mem_pool.get()); - read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); - std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); - EXPECT_TRUE(dest_string.compare("2019-12-04 00:00:00") == 0); - - //test not support type - const auto* tp = get_scalar_type_info(); - Status st = read_row.convert_from(0, data, tp, _mem_pool.get()); - EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertDateToDateV2) { - TabletSchema tablet_schema; - set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - std::string origin_val = "2019-12-04"; - val_string_array.emplace_back(origin_val); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header_message; - EXPECT_EQ(_column_writer->finalize(&header_message), Status::OK()); - - TabletSchema convert_tablet_schema; - set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &convert_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(convert_tablet_schema); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.set_field_content(0, data, _mem_pool.get()); - read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); - std::string expected = "2019-12-04"; - std::string dest_string = - read_row.column_schema(0)->to_string(read_row.cell_ptr(0)).substr(0, expected.length()); - EXPECT_TRUE(dest_string.compare("2019-12-04") == 0); - - //test not support type - const auto* tp = get_scalar_type_info(); - Status st = read_row.convert_from(0, data, tp, _mem_pool.get()); - EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertDateV2ToDate) { - TabletSchema tablet_schema; - set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - std::string origin_val = "2019-12-04"; - val_string_array.emplace_back(origin_val); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header_message; - EXPECT_EQ(_column_writer->finalize(&header_message), Status::OK()); - - TabletSchema convert_tablet_schema; - set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &convert_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(convert_tablet_schema); - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.set_field_content(0, data, _mem_pool.get()); - read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); - std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); - EXPECT_TRUE(dest_string.compare("2019-12-04") == 0); - - //test not support type - const auto* tp = get_scalar_type_info(); - Status st = read_row.convert_from(0, data, tp, _mem_pool.get()); - EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertIntToDate) { - TabletSchema tablet_schema; - set_tablet_schema("IntColumn", "INT", "REPLACE", 4, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - int time_val = 20191205; - write_row.set_field_content(0, reinterpret_cast(&time_val), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - TabletSchema convert_tablet_schema; - set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &convert_tablet_schema); - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(convert_tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); - std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); - EXPECT_TRUE(strncmp(dest_string.c_str(), "2019-12-05", strlen("2019-12-05")) == 0); - - //test not support type - const auto* tp = get_scalar_type_info(); - Status st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get()); - EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertIntToDateV2) { - TabletSchema tablet_schema; - set_tablet_schema("IntColumn", "INT", "REPLACE", 4, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - int time_val = 20191205; - write_row.set_field_content(0, reinterpret_cast(&time_val), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - TabletSchema convert_tablet_schema; - set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 4, false, false, &convert_tablet_schema); - create_column_reader(tablet_schema); - - RowCursor read_row; - read_row.init(convert_tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); - std::string dest_string = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); - EXPECT_TRUE(strncmp(dest_string.c_str(), "2019-12-05", strlen("2019-12-05")) == 0); - - //test not support type - const auto* tp = get_scalar_type_info(); - Status st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get()); - EXPECT_TRUE(st == Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertVarcharToDate) { - TabletSchema tablet_schema; - set_tablet_schema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - // test valid format convert - std::vector valid_src_strs = { - "2019-12-17", "19-12-17", "20191217", "191217", "2019/12/17", "19/12/17", - }; - std::string expected_val("2019-12-17"); - for (auto src_str : valid_src_strs) { - write_row.set_field_content(0, reinterpret_cast(&src_str), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // because file_helper is reused in this case, we should close it. - helper.close(); - TabletSchema convert_tablet_schema; - set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &convert_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(convert_tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); - std::string dst_str = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); - EXPECT_EQ(expected_val, dst_str); - } - helper.close(); - TabletSchema convert_tablet_schema; - set_tablet_schema("DateColumn", "DATE", "REPLACE", 3, false, false, &convert_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(convert_tablet_schema); - - //test not support type - const auto* tp = get_scalar_type_info(); - Status st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get()); - EXPECT_EQ(st, Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertVarcharToDateV2) { - TabletSchema tablet_schema; - set_tablet_schema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema); - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - // test valid format convert - std::vector valid_src_strs = { - "2019-12-17", "19-12-17", "20191217", "191217", "2019/12/17", "19/12/17", - }; - std::string expected_val("2019-12-17"); - for (auto src_str : valid_src_strs) { - write_row.set_field_content(0, reinterpret_cast(&src_str), _mem_pool.get()); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - // because file_helper is reused in this case, we should close it. - helper.close(); - TabletSchema convert_tablet_schema; - set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 3, false, false, - &convert_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(convert_tablet_schema); - - _col_vector.reset(new ColumnVector()); - EXPECT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), Status::OK()); - char* data = reinterpret_cast(_col_vector->col_data()); - read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); - std::string dst_str = read_row.column_schema(0) - ->to_string(read_row.cell_ptr(0)) - .substr(0, expected_val.length()); - EXPECT_EQ(expected_val, dst_str); - } - helper.close(); - TabletSchema convert_tablet_schema; - set_tablet_schema("DateV2Column", "DATEV2", "REPLACE", 3, false, false, &convert_tablet_schema); - create_column_reader(tablet_schema); - RowCursor read_row; - read_row.init(convert_tablet_schema); - - //test not support type - const auto* tp = get_scalar_type_info(); - Status st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get()); - EXPECT_EQ(st, Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertVarcharToTinyInt1) { - test_convert_from_varchar("TINYINT", 1, "127", Status::OK()); -} - -TEST_F(TestColumn, ConvertVarcharToTinyInt2) { - test_convert_from_varchar("TINYINT", 1, "128", - Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertVarcharToSmallInt1) { - test_convert_from_varchar("SMALLINT", 2, "32767", Status::OK()); -} - -TEST_F(TestColumn, ConvertVarcharToSmallInt2) { - test_convert_from_varchar("SMALLINT", 2, "32768", - Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertVarcharToInt1) { - test_convert_from_varchar("INT", 4, "2147483647", Status::OK()); -} - -TEST_F(TestColumn, ConvertVarcharToInt2) { - test_convert_from_varchar("INT", 4, "2147483648", - Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertVarcharToBigInt1) { - test_convert_from_varchar("BIGINT", 8, "9223372036854775807", Status::OK()); -} - -TEST_F(TestColumn, ConvertVarcharToBigInt2) { - test_convert_from_varchar("BIGINT", 8, "9223372036854775808", - Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertVarcharToLargeInt1) { - test_convert_from_varchar("LARGEINT", 16, "170141183460469000000000000000000000000", - Status::OK()); -} - -TEST_F(TestColumn, ConvertVarcharToLargeInt2) { - test_convert_from_varchar("LARGEINT", 16, "1701411834604690000000000000000000000000", - Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertVarcharToFloat1) { - test_convert_from_varchar("FLOAT", 4, "3.40282e+38", Status::OK()); -} - -TEST_F(TestColumn, ConvertVarcharToFloat2) { - test_convert_from_varchar( - "FLOAT", 4, - "17976900000000000632303049213894264349303303643368533621541098328912643414890628994061" - "52996321966094455338163203127744334848599000464911410516510916727344709727599413825823" - "04802812882753059262973637182942535982636884444611376868582636745405553206881859340916" - "3400929532301499014067384276511218551077374242324480.999", - Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertVarcharToDouble1) { - test_convert_from_varchar("DOUBLE", 8, "123.456", Status::OK()); -} - -TEST_F(TestColumn, ConvertVarcharToDouble2) { - test_convert_from_varchar( - "DOUBLE", 8, - "17976900000000000632303049213894264349303303643368533621541098328912643414890628994061" - "52996321966094455338163203127744334848599000464911410516510916727344709727599413825823" - "04802812882753059262973637182942535982636884444611376868582636745405553206881859340916" - "3400929532301499014067384276511218551077374242324480.0000000000", - Status::OLAPInternalError(OLAP_ERR_INVALID_SCHEMA)); -} - -TEST_F(TestColumn, ConvertTinyIntToVarchar3) { - test_convert_to_varchar("TINYINT", 1, 127, "", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 3); -} - -TEST_F(TestColumn, ConvertTinyIntToVarchar5) { - test_convert_to_varchar("TINYINT", 1, 127, "127", Status::OK(), 3 + 2); -} - -TEST_F(TestColumn, ConvertTinyIntToVarchar4) { - test_convert_to_varchar("TINYINT", 1, -127, "", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 4); -} - -TEST_F(TestColumn, ConvertTinyIntToVarchar6) { - // 4: tinyint digit count + minus symbol, +2 for var len bytes - test_convert_to_varchar("TINYINT", 1, -127, "-127", Status::OK(), 4 + 2); -} - -TEST_F(TestColumn, ConvertSmallIntToVarchar5) { - test_convert_to_varchar("SMALLINT", 2, 32767, "", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 5); -} - -TEST_F(TestColumn, ConvertSmallIntToVarchar7) { - test_convert_to_varchar("SMALLINT", 2, 32767, "32767", Status::OK(), 7); -} - -TEST_F(TestColumn, ConvertSmallIntToVarchar6) { - test_convert_to_varchar("SMALLINT", 2, -32767, "", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 6); -} - -TEST_F(TestColumn, ConvertSmallIntToVarchar8) { - test_convert_to_varchar("SMALLINT", 2, -32767, "-32767", Status::OK(), 8); -} - -TEST_F(TestColumn, ConvertIntToVarchar10) { - test_convert_to_varchar("INT", 4, 2147483647, "", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 10); -} - -TEST_F(TestColumn, ConvertIntToVarchar12) { - test_convert_to_varchar("INT", 4, 2147483647, "2147483647", Status::OK(), 12); -} - -TEST_F(TestColumn, ConvertIntToVarchar11) { - test_convert_to_varchar("INT", 4, -2147483647, "", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 11); -} - -TEST_F(TestColumn, ConvertIntToVarchar13) { - test_convert_to_varchar("INT", 4, -2147483647, "-2147483647", Status::OK(), 13); -} - -TEST_F(TestColumn, ConvertBigIntToVarchar19) { - test_convert_to_varchar("BIGINT", 8, 9223372036854775807, "", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 19); -} - -TEST_F(TestColumn, ConvertBigIntToVarchar21) { - test_convert_to_varchar("BIGINT", 8, 9223372036854775807, "9223372036854775807", - Status::OK(), 21); -} - -TEST_F(TestColumn, ConvertBigIntToVarchar20) { - test_convert_to_varchar("BIGINT", 8, -9223372036854775807, "", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 20); -} - -TEST_F(TestColumn, ConvertBigIntToVarchar22) { - test_convert_to_varchar("BIGINT", 8, -9223372036854775807, "-9223372036854775807", - Status::OK(), 22); -} - -TEST_F(TestColumn, ConvertLargeIntToVarchar39) { - std::string str_val("170141183460469231731687303715884105727"); - StringParser::ParseResult result; - int128_t int128_val = - StringParser::string_to_int(str_val.c_str(), str_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - test_convert_to_varchar("LARGEINT", 16, int128_val, "", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), - 39); -} - -TEST_F(TestColumn, ConvertLargeIntToVarchar41) { - std::string str_val("170141183460469231731687303715884105727"); - StringParser::ParseResult result; - int128_t int128_val = - StringParser::string_to_int(str_val.c_str(), str_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - test_convert_to_varchar("LARGEINT", 16, int128_val, str_val, Status::OK(), 41); -} - -TEST_F(TestColumn, ConvertLargeIntToVarchar40) { - std::string str_val = "-170141183460469231731687303715884105727"; - StringParser::ParseResult result; - int128_t int128_val = - StringParser::string_to_int(str_val.c_str(), str_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - test_convert_to_varchar("LARGEINT", 16, int128_val, "", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), - 40); -} - -TEST_F(TestColumn, ConvertLargeIntToVarchar46) { - std::string str_val = "-170141183460469231731687303715884105727"; - StringParser::ParseResult result; - int128_t int128_val = - StringParser::string_to_int(str_val.c_str(), str_val.length(), &result); - DCHECK(result == StringParser::PARSE_SUCCESS); - test_convert_to_varchar("LARGEINT", 16, int128_val, str_val, Status::OK(), 42); -} - -TEST_F(TestColumn, ConvertFloatToVarchar11) { - test_convert_to_varchar("FLOAT", 4, 3.40282e+38, "3.40282e+38", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 11); -} - -TEST_F(TestColumn, ConvertFloatToVarchar13) { - test_convert_to_varchar("FLOAT", 4, 3.40282e+38, "3.40282e+38", Status::OK(), 13); -} - -TEST_F(TestColumn, ConvertFloatToVarchar13_2) { - test_convert_to_varchar("FLOAT", 4, 3402820000000000000.0, "3.40282e+18", Status::OK(), - 13); -} - -TEST_F(TestColumn, ConvertFloatToVarchar12) { - test_convert_to_varchar("FLOAT", 4, -3.40282e+38, "-3.40282e+38", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 12); -} - -TEST_F(TestColumn, ConvertFloatToVarchar14) { - test_convert_to_varchar("FLOAT", 4, -3.40282e+38, "-3.40282e+38", Status::OK(), 14); -} - -TEST_F(TestColumn, ConvertFloatToVarchar14_2) { - test_convert_to_varchar("FLOAT", 4, -3402820000000000000.0, "-3.40282e+18", Status::OK(), - 14); -} - -TEST_F(TestColumn, ConvertFloatToVarchar13_3) { - test_convert_to_varchar("FLOAT", 4, 1.17549435082228750796873653722224568e-38F, - "1.1754944e-38", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 13); -} - -TEST_F(TestColumn, ConvertFloatToVarchar15) { - test_convert_to_varchar("FLOAT", 4, 1.17549435082228750796873653722224568e-38F, - "1.1754944e-38", Status::OK(), 15); -} - -TEST_F(TestColumn, ConvertFloatToVarchar14_3) { - test_convert_to_varchar("FLOAT", 4, -1.17549435082228750796873653722224568e-38F, - "-1.1754944e-38", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 14); -} - -TEST_F(TestColumn, ConvertFloatToVarchar16) { - test_convert_to_varchar("FLOAT", 4, -1.17549435082228750796873653722224568e-38F, - "-1.1754944e-38", Status::OK(), 16); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar7) { - test_convert_to_varchar("DOUBLE", 8, 123.456, "123.456", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 7); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar9) { - test_convert_to_varchar("DOUBLE", 8, 123.456, "123.456", Status::OK(), 9); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar23) { - test_convert_to_varchar("DOUBLE", 8, 1.79769313486231570814527423731704357e+308, - "1.7976931348623157e+308", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 23); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar25) { - test_convert_to_varchar("DOUBLE", 8, 1.79769313486231570814527423731704357e+308, - "1.7976931348623157e+308", Status::OK(), 25); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar22) { - test_convert_to_varchar("DOUBLE", 8, 1797693134862315708.0, "1.7976931348623158e+18", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 22); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar24) { - test_convert_to_varchar("DOUBLE", 8, 1797693134862315708.0, "1.7976931348623158e+18", - Status::OK(), 24); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar23_2) { - test_convert_to_varchar("DOUBLE", 8, -1797693134862315708.0, "-1.7976931348623158e+18", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 23); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar25_2) { - test_convert_to_varchar("DOUBLE", 8, -1797693134862315708.0, "-1.7976931348623158e+18", - Status::OK(), 25); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar23_3) { - test_convert_to_varchar("DOUBLE", 8, 2.22507385850720138309023271733240406e-308, - "2.2250738585072014e-308", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 23); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar25_3) { - test_convert_to_varchar("DOUBLE", 8, 2.22507385850720138309023271733240406e-308, - "2.2250738585072014e-308", Status::OK(), 25); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar24_2) { - test_convert_to_varchar("DOUBLE", 8, -2.22507385850720138309023271733240406e-308, - "-2.2250738585072014e-308", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 24); -} - -TEST_F(TestColumn, ConvertDoubleToVarchar26) { - test_convert_to_varchar("DOUBLE", 8, -2.22507385850720138309023271733240406e-308, - "-2.2250738585072014e-308", Status::OK(), 26); -} - -TEST_F(TestColumn, ConvertDecimalToVarchar13) { - decimal12_t val = {456, 789000000}; - test_convert_to_varchar("Decimal", 12, val, "456.789000000", - Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), - 13); -} - -TEST_F(TestColumn, ConvertDecimalToVarchar15) { - decimal12_t val = {456, 789000000}; - test_convert_to_varchar("Decimal", 12, val, "456.789000000", Status::OK(), 15); -} - -TEST_F(TestColumn, ConvertDecimalToVarchar28) { - decimal12_t val = {999999999999999999, 999999999}; - test_convert_to_varchar( - "Decimal", 12, val, "", Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 28); -} - -TEST_F(TestColumn, ConvertDecimalToVarchar30) { - decimal12_t val = {999999999999999999, 999999999}; - test_convert_to_varchar("Decimal", 12, val, "999999999999999999.999999999", - Status::OK(), 30); -} - -TEST_F(TestColumn, ConvertDecimalToVarchar29) { - decimal12_t val = {-999999999999999999, 999999999}; - test_convert_to_varchar( - "Decimal", 12, val, "", Status::OLAPInternalError(OLAP_ERR_INPUT_PARAMETER_ERROR), 29); -} - -TEST_F(TestColumn, ConvertDecimalToVarchar31) { - decimal12_t val = {-999999999999999999, 999999999}; - test_convert_to_varchar("Decimal", 12, val, "-999999999999999999.999999999", - Status::OK(), 31); -} - -void CreateTabletSchema(TabletSchema& tablet_schema) { - TabletSchemaPB tablet_schema_pb; - tablet_schema_pb.set_keys_type(KeysType::AGG_KEYS); - tablet_schema_pb.set_num_short_key_columns(2); - tablet_schema_pb.set_num_rows_per_row_block(1024); - tablet_schema_pb.set_compress_kind(COMPRESS_NONE); - tablet_schema_pb.set_next_column_unique_id(4); - - ColumnPB* column_1 = tablet_schema_pb.add_column(); - column_1->set_unique_id(1); - column_1->set_name("k1"); - column_1->set_type("INT"); - column_1->set_is_key(true); - column_1->set_length(4); - column_1->set_index_length(4); - column_1->set_is_nullable(false); - column_1->set_is_bf_column(false); - - ColumnPB* column_2 = tablet_schema_pb.add_column(); - column_2->set_unique_id(2); - column_2->set_name("k2"); - column_2->set_type("VARCHAR"); - column_2->set_length(20); - column_2->set_index_length(20); - column_2->set_is_key(true); - column_2->set_is_nullable(false); - column_2->set_is_bf_column(false); - - ColumnPB* column_3 = tablet_schema_pb.add_column(); - column_3->set_unique_id(3); - column_3->set_name("k3"); - column_3->set_type("INT"); - column_3->set_is_key(true); - column_3->set_length(4); - column_3->set_index_length(4); - column_3->set_is_nullable(false); - column_3->set_is_bf_column(false); - - ColumnPB* column_4 = tablet_schema_pb.add_column(); - column_4->set_unique_id(4); - column_4->set_name("v1"); - column_4->set_type("INT"); - column_4->set_length(4); - column_4->set_is_key(false); - column_4->set_is_nullable(false); - column_4->set_is_bf_column(false); - column_4->set_aggregation("SUM"); - - tablet_schema.init_from_pb(tablet_schema_pb); -} - -TEST_F(TestColumn, ConvertIntToBitmap) { - //Base Tablet - TabletSchema tablet_schema; - CreateTabletSchema(tablet_schema); - //Base row block - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - val_string_array.emplace_back("5"); - val_string_array.emplace_back("4"); - val_string_array.emplace_back("2"); - val_string_array.emplace_back("3"); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - //Materialized View tablet schema - TabletSchemaPB mv_tablet_schema_pb; - mv_tablet_schema_pb.set_keys_type(KeysType::AGG_KEYS); - mv_tablet_schema_pb.set_num_short_key_columns(2); - mv_tablet_schema_pb.set_num_rows_per_row_block(1024); - mv_tablet_schema_pb.set_compress_kind(COMPRESS_NONE); - mv_tablet_schema_pb.set_next_column_unique_id(3); - - ColumnPB* mv_column_1 = mv_tablet_schema_pb.add_column(); - mv_column_1->set_unique_id(1); - mv_column_1->set_name("k1"); - mv_column_1->set_type("INT"); - mv_column_1->set_is_key(true); - mv_column_1->set_length(4); - mv_column_1->set_index_length(4); - mv_column_1->set_is_nullable(false); - mv_column_1->set_is_bf_column(false); - - ColumnPB* mv_column_2 = mv_tablet_schema_pb.add_column(); - mv_column_2->set_unique_id(2); - mv_column_2->set_name("v1"); - mv_column_2->set_type("OBJECT"); - mv_column_2->set_length(8); - mv_column_2->set_is_key(false); - mv_column_2->set_is_nullable(false); - mv_column_2->set_is_bf_column(false); - mv_column_2->set_aggregation("BITMAP_UNION"); - - TabletSchema mv_tablet_schema; - mv_tablet_schema.init_from_pb(mv_tablet_schema_pb); - - RowBlockChanger row_block_changer(mv_tablet_schema, DescriptorTbl()); - ColumnMapping* column_mapping = row_block_changer.get_mutable_column_mapping(0); - column_mapping->ref_column = 0; - column_mapping = row_block_changer.get_mutable_column_mapping(1); - column_mapping->ref_column = 2; - column_mapping->materialized_function = "to_bitmap"; - - RowBlock mutable_block(&mv_tablet_schema); - mutable_block.init(block_info); - uint64_t filtered_rows = 0; - row_block_changer.change_row_block(&block, 0, &mutable_block, &filtered_rows); - - RowCursor mv_row_cursor; - mv_row_cursor.init(mv_tablet_schema); - mutable_block.get_row(0, &mv_row_cursor); - - auto dst_slice = reinterpret_cast(mv_row_cursor.cell_ptr(1)); - BitmapValue bitmapValue(dst_slice->data); - EXPECT_EQ(bitmapValue.cardinality(), 1); -} - -TEST_F(TestColumn, ConvertCharToHLL) { - //Base Tablet - TabletSchema tablet_schema; - CreateTabletSchema(tablet_schema); - - //Base row block - create_column_writer(tablet_schema); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - std::vector val_string_array; - //std::string origin_val = "2019-11-25 19:07:00"; - //val_string_array.emplace_back(origin_val); - val_string_array.emplace_back("1"); - val_string_array.emplace_back("1"); - val_string_array.emplace_back("2"); - val_string_array.emplace_back("3"); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - block.set_row(0, write_row); - block.finalize(1); - EXPECT_EQ(_column_writer->write_batch(&block, &write_row), Status::OK()); - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - //Materialized View tablet schema - TabletSchemaPB mv_tablet_schema_pb; - mv_tablet_schema_pb.set_keys_type(KeysType::AGG_KEYS); - mv_tablet_schema_pb.set_num_short_key_columns(2); - mv_tablet_schema_pb.set_num_rows_per_row_block(1024); - mv_tablet_schema_pb.set_compress_kind(COMPRESS_NONE); - mv_tablet_schema_pb.set_next_column_unique_id(3); - - ColumnPB* mv_column_1 = mv_tablet_schema_pb.add_column(); - mv_column_1->set_unique_id(1); - mv_column_1->set_name("k1"); - mv_column_1->set_type("INT"); - mv_column_1->set_is_key(true); - mv_column_1->set_length(4); - mv_column_1->set_index_length(4); - mv_column_1->set_is_nullable(false); - mv_column_1->set_is_bf_column(false); - - ColumnPB* mv_column_2 = mv_tablet_schema_pb.add_column(); - mv_column_2->set_unique_id(2); - mv_column_2->set_name("v1"); - mv_column_2->set_type("HLL"); - mv_column_2->set_length(4); - mv_column_2->set_is_key(false); - mv_column_2->set_is_nullable(false); - mv_column_2->set_is_bf_column(false); - mv_column_2->set_aggregation("HLL_UNION"); - - TabletSchema mv_tablet_schema; - mv_tablet_schema.init_from_pb(mv_tablet_schema_pb); - - RowBlockChanger row_block_changer(mv_tablet_schema, DescriptorTbl()); - ColumnMapping* column_mapping = row_block_changer.get_mutable_column_mapping(0); - column_mapping->ref_column = 0; - column_mapping = row_block_changer.get_mutable_column_mapping(1); - column_mapping->ref_column = 1; - column_mapping->materialized_function = "hll_hash"; - - RowBlock mutable_block(&mv_tablet_schema); - mutable_block.init(block_info); - uint64_t filtered_rows = 0; - row_block_changer.change_row_block(&block, 0, &mutable_block, &filtered_rows); - - RowCursor mv_row_cursor; - mv_row_cursor.init(mv_tablet_schema); - mutable_block.get_row(0, &mv_row_cursor); - - auto dst_slice = reinterpret_cast(mv_row_cursor.cell_ptr(1)); - HyperLogLog hll(*dst_slice); - EXPECT_EQ(hll.estimate_cardinality(), 1); -} - -TEST_F(TestColumn, ConvertCharToCount) { - //Base Tablet - TabletSchema tablet_schema; - CreateTabletSchema(tablet_schema); - - //Base row block - create_column_writer(tablet_schema); - - RowBlock block(&tablet_schema); - RowBlockInfo block_info; - block_info.row_num = 10000; - block.init(block_info); - - RowCursor write_row; - write_row.init(tablet_schema); - write_row.allocate_memory_for_string_type(tablet_schema); - std::vector val_string_array; - val_string_array.emplace_back("1"); - val_string_array.emplace_back("1"); - val_string_array.emplace_back("2"); - val_string_array.emplace_back("3"); - OlapTuple tuple(val_string_array); - write_row.from_tuple(tuple); - block.set_row(0, write_row); - - block.finalize(1); - ColumnDataHeaderMessage header; - EXPECT_EQ(_column_writer->finalize(&header), Status::OK()); - - //Materialized View tablet schema - TabletSchemaPB mv_tablet_schema_pb; - mv_tablet_schema_pb.set_keys_type(KeysType::AGG_KEYS); - mv_tablet_schema_pb.set_num_short_key_columns(2); - mv_tablet_schema_pb.set_num_rows_per_row_block(1024); - mv_tablet_schema_pb.set_compress_kind(COMPRESS_NONE); - mv_tablet_schema_pb.set_next_column_unique_id(3); - - ColumnPB* mv_column_1 = mv_tablet_schema_pb.add_column(); - mv_column_1->set_unique_id(1); - mv_column_1->set_name("k1"); - mv_column_1->set_type("INT"); - mv_column_1->set_is_key(true); - mv_column_1->set_length(4); - mv_column_1->set_index_length(4); - mv_column_1->set_is_nullable(false); - mv_column_1->set_is_bf_column(false); - - ColumnPB* mv_column_2 = mv_tablet_schema_pb.add_column(); - mv_column_2->set_unique_id(2); - mv_column_2->set_name("v1"); - mv_column_2->set_type("BIGINT"); - mv_column_2->set_length(4); - mv_column_2->set_is_key(false); - mv_column_2->set_is_nullable(false); - mv_column_2->set_is_bf_column(false); - mv_column_2->set_aggregation("SUM"); - - TabletSchema mv_tablet_schema; - mv_tablet_schema.init_from_pb(mv_tablet_schema_pb); - - RowBlockChanger row_block_changer(mv_tablet_schema, DescriptorTbl()); - ColumnMapping* column_mapping = row_block_changer.get_mutable_column_mapping(0); - column_mapping->ref_column = 0; - column_mapping = row_block_changer.get_mutable_column_mapping(1); - column_mapping->ref_column = 1; - column_mapping->materialized_function = "count_field"; - - RowBlock mutable_block(&mv_tablet_schema); - mutable_block.init(block_info); - uint64_t filtered_rows = 0; - row_block_changer.change_row_block(&block, 0, &mutable_block, &filtered_rows); - - RowCursor mv_row_cursor; - mv_row_cursor.init(mv_tablet_schema); - mutable_block.get_row(0, &mv_row_cursor); - - auto dst = mv_row_cursor.cell_ptr(1); - EXPECT_EQ(*(int64_t*)dst, 1); -} -} // namespace doris diff --git a/be/test/olap/serialize_test.cpp b/be/test/olap/serialize_test.cpp deleted file mode 100644 index 65a13fe245..0000000000 --- a/be/test/olap/serialize_test.cpp +++ /dev/null @@ -1,234 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/serialize.h" - -#include - -namespace doris { -namespace ser { - -class SerializeTest : public testing::Test { -public: - SerializeTest() {} - virtual ~SerializeTest() {} -}; - -TEST_F(SerializeTest, get_closet_fixed_bits) { - EXPECT_EQ(1, get_closet_fixed_bits(0)); - for (int i = 1; i <= 24; ++i) { - EXPECT_EQ(i, get_closet_fixed_bits(i)); - } - for (int i = 25; i <= 26; ++i) { - EXPECT_EQ(26, get_closet_fixed_bits(i)); - } - for (int i = 27; i <= 28; ++i) { - EXPECT_EQ(28, get_closet_fixed_bits(i)); - } - for (int i = 29; i <= 30; ++i) { - EXPECT_EQ(30, get_closet_fixed_bits(i)); - } - for (int i = 31; i <= 32; ++i) { - EXPECT_EQ(32, get_closet_fixed_bits(i)); - } - for (int i = 33; i <= 40; ++i) { - EXPECT_EQ(40, get_closet_fixed_bits(i)); - } - for (int i = 41; i <= 48; ++i) { - EXPECT_EQ(48, get_closet_fixed_bits(i)); - } - for (int i = 49; i <= 56; ++i) { - EXPECT_EQ(56, get_closet_fixed_bits(i)); - } - for (int i = 57; i <= 64; ++i) { - EXPECT_EQ(64, get_closet_fixed_bits(i)); - } -} - -TEST_F(SerializeTest, find_closet_num_bits) { - EXPECT_EQ(1, find_closet_num_bits(0)); - for (int i = 1; i <= 24; ++i) { - uint64_t val = (1l << (i - 1)); - EXPECT_EQ(i, find_closet_num_bits(val)); - } - for (int i = 25; i <= 26; ++i) { - uint64_t val = (1l << (i - 1)); - EXPECT_EQ(26, find_closet_num_bits(val)); - } - for (int i = 27; i <= 28; ++i) { - uint64_t val = (1l << (i - 1)); - EXPECT_EQ(28, find_closet_num_bits(val)); - } - for (int i = 29; i <= 30; ++i) { - uint64_t val = (1l << (i - 1)); - EXPECT_EQ(30, find_closet_num_bits(val)); - } - for (int i = 31; i <= 32; ++i) { - uint64_t val = (1l << (i - 1)); - EXPECT_EQ(32, find_closet_num_bits(val)); - } - for (int i = 33; i <= 40; ++i) { - uint64_t val = (1l << (i - 1)); - EXPECT_EQ(40, find_closet_num_bits(val)); - } - for (int i = 41; i <= 48; ++i) { - uint64_t val = (1l << (i - 1)); - EXPECT_EQ(48, find_closet_num_bits(val)); - } - for (int i = 49; i <= 56; ++i) { - uint64_t val = (1l << (i - 1)); - EXPECT_EQ(56, find_closet_num_bits(val)); - } - for (int i = 57; i <= 64; ++i) { - uint64_t val = (1l << (i - 1)); - EXPECT_EQ(64, find_closet_num_bits(val)); - } -} - -TEST_F(SerializeTest, encode_bit_width) { - EXPECT_EQ(ONE, encode_bit_width(0)); - for (int i = 1; i <= 24; ++i) { - EXPECT_EQ(i - 1, encode_bit_width(i)); - } - for (int i = 25; i <= 26; ++i) { - EXPECT_EQ(TWENTYSIX, encode_bit_width(i)); - } - for (int i = 27; i <= 28; ++i) { - EXPECT_EQ(TWENTYEIGHT, encode_bit_width(i)); - } - for (int i = 29; i <= 30; ++i) { - EXPECT_EQ(THIRTY, encode_bit_width(i)); - } - for (int i = 31; i <= 32; ++i) { - EXPECT_EQ(THIRTYTWO, encode_bit_width(i)); - } - for (int i = 33; i <= 40; ++i) { - EXPECT_EQ(FORTY, encode_bit_width(i)); - } - for (int i = 41; i <= 48; ++i) { - EXPECT_EQ(FORTYEIGHT, encode_bit_width(i)); - } - for (int i = 49; i <= 56; ++i) { - EXPECT_EQ(FIFTYSIX, encode_bit_width(i)); - } - for (int i = 57; i <= 64; ++i) { - EXPECT_EQ(SIXTYFOUR, encode_bit_width(i)); - } -} - -TEST_F(SerializeTest, decode_bit_width) { - for (int i = 0; i <= TWENTYFOUR; ++i) { - EXPECT_EQ(i + 1, decode_bit_width(i)); - } - EXPECT_EQ(26, decode_bit_width(TWENTYSIX)); - EXPECT_EQ(28, decode_bit_width(TWENTYEIGHT)); - EXPECT_EQ(30, decode_bit_width(THIRTY)); - EXPECT_EQ(32, decode_bit_width(THIRTYTWO)); - EXPECT_EQ(40, decode_bit_width(FORTY)); - EXPECT_EQ(48, decode_bit_width(FORTYEIGHT)); - EXPECT_EQ(56, decode_bit_width(FIFTYSIX)); - EXPECT_EQ(64, decode_bit_width(SIXTYFOUR)); -} - -TEST_F(SerializeTest, percentile_bits) { - int64_t data[100]; - - { - for (int i = 0; i < 5; ++i) { - data[i] = (1l << 58); - } - for (int i = 5; i < 100; ++i) { - data[i] = 1; - } - EXPECT_EQ(0, percentile_bits(data, 100, 0.0)); - EXPECT_EQ(1, percentile_bits(data, 100, 0.95)); - EXPECT_EQ(64, percentile_bits(data, 100, 0.99)); - EXPECT_EQ(64, percentile_bits(data, 100, 1.0)); - } - { - for (int i = 0; i < 11; ++i) { - data[i] = (1l << 26); - } - for (int i = 11; i < 100; ++i) { - data[i] = 1; - } - EXPECT_EQ(0, percentile_bits(data, 100, 0.0)); - EXPECT_EQ(1, percentile_bits(data, 100, 0.8)); - EXPECT_EQ(28, percentile_bits(data, 100, 0.9)); - } - { - for (int i = 0; i < 11; ++i) { - data[i] = (1l << 26); - } - for (int i = 11; i < 100; ++i) { - data[i] = 0; - } - EXPECT_EQ(0, percentile_bits(data, 100, 0.0)); - EXPECT_EQ(1, percentile_bits(data, 100, 0.1)); - EXPECT_EQ(1, percentile_bits(data, 100, 0.8)); - EXPECT_EQ(28, percentile_bits(data, 100, 0.9)); - } -} - -TEST_F(SerializeTest, new_percentile_bits) { - int64_t data[100]; - - { - for (int i = 0; i < 5; ++i) { - data[i] = (1l << 58); - } - for (int i = 5; i < 100; ++i) { - data[i] = 1; - } - uint16_t hists[65]; - compute_hists(data, 100, hists); - EXPECT_EQ(0, percentile_bits_with_hist(hists, 100, 0.0)); - EXPECT_EQ(1, percentile_bits_with_hist(hists, 100, 0.95)); - EXPECT_EQ(64, percentile_bits_with_hist(hists, 100, 0.99)); - EXPECT_EQ(64, percentile_bits_with_hist(hists, 100, 1.0)); - } - { - for (int i = 0; i < 11; ++i) { - data[i] = (1l << 26); - } - for (int i = 11; i < 100; ++i) { - data[i] = 1; - } - uint16_t hists[65]; - compute_hists(data, 100, hists); - EXPECT_EQ(0, percentile_bits_with_hist(hists, 100, 0.0)); - EXPECT_EQ(1, percentile_bits_with_hist(hists, 100, 0.8)); - EXPECT_EQ(28, percentile_bits_with_hist(hists, 100, 0.9)); - } - { - for (int i = 0; i < 11; ++i) { - data[i] = (1l << 26); - } - for (int i = 11; i < 100; ++i) { - data[i] = 0; - } - uint16_t hists[65]; - compute_hists(data, 100, hists); - EXPECT_EQ(0, percentile_bits_with_hist(hists, 100, 0.0)); - EXPECT_EQ(1, percentile_bits_with_hist(hists, 100, 0.1)); - EXPECT_EQ(1, percentile_bits_with_hist(hists, 100, 0.8)); - EXPECT_EQ(28, percentile_bits_with_hist(hists, 100, 0.9)); - } -} - -} // namespace ser -} // namespace doris diff --git a/be/test/olap/stream_index_test.cpp b/be/test/olap/stream_index_test.cpp deleted file mode 100644 index 75191378d1..0000000000 --- a/be/test/olap/stream_index_test.cpp +++ /dev/null @@ -1,385 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include - -#include "olap/file_stream.h" -#include "olap/olap_common.h" -#include "olap/olap_cond.h" -#include "olap/olap_define.h" -#include "olap/row_cursor.h" -#include "olap/storage_engine.h" -#include "olap/stream_index_common.h" -#include "olap/stream_index_reader.h" -#include "olap/stream_index_writer.h" -#include "olap/tablet.h" -#include "olap/tablet_meta.h" -#include "olap/wrapper_field.h" -#include "util/logging.h" - -using namespace std; - -namespace doris { - -class TestStreamIndex : public testing::Test { -public: - virtual ~TestStreamIndex() {} - - virtual void SetUp() {} - - virtual void TearDown() {} -}; - -TEST_F(TestStreamIndex, index_write) { - StreamIndexWriter writer(OLAP_FIELD_TYPE_INT); - PositionEntryWriter entry; - ColumnStatistics stat; - stat.init(OLAP_FIELD_TYPE_INT, true); - - EXPECT_EQ(Status::OK(), stat.init(OLAP_FIELD_TYPE_INT, true)); - - static const uint32_t loop = 10; - uint32_t i = 0; - for (; i < loop; i++) { - entry.add_position(i); - entry.add_position(i * 2); - entry.add_position(i * 3); - - entry.set_statistic(&stat); - writer.add_index_entry(entry); - entry.reset_write_offset(); - } - - size_t output_size = sizeof(StreamIndexHeader) + i * sizeof(uint32_t) * 3; - // for statistics - output_size += (sizeof(int) + 1) * 2 * loop; - - EXPECT_EQ(output_size, writer.output_size()); - - char* buffer = new char[output_size]; - - EXPECT_EQ(Status::OK(), writer.write_to_buffer(buffer, output_size)); - - StreamIndexReader reader; - EXPECT_EQ(Status::OK(), reader.init(buffer, output_size, OLAP_FIELD_TYPE_INT, true, true)); - - EXPECT_EQ(loop, reader.entry_count()); - - for (i = 0; i < loop; i++) { - const PositionEntryReader& e = reader.entry(i); - EXPECT_EQ(e.positions(0), i); - EXPECT_EQ(e.positions(1), i * 2); - EXPECT_EQ(e.positions(2), i * 3); - } - delete[] buffer; -} - -TEST_F(TestStreamIndex, remove_written_position) { - StreamIndexWriter writer(OLAP_FIELD_TYPE_INT); - PositionEntryWriter entry; - ColumnStatistics stat; - stat.init(OLAP_FIELD_TYPE_INT, true); - - static const uint32_t loop = 10; - //test 1 - { - uint32_t i = 0; - for (; i < loop; i++) { - entry.add_position(i); - entry.add_position(i * 2); - entry.add_position(i * 3); - entry.add_position(i * 4); - entry.add_position(i * 5); - entry.add_position(i * 6); - entry.add_position(i * 7); - - entry.set_statistic(&stat); - writer.add_index_entry(entry); - entry.reset_write_offset(); - } - - for (i = 0; i < loop; i++) { - PositionEntryWriter* e = writer.mutable_entry(i); - e->remove_written_position(0, 4); - } - - size_t output_size = writer.output_size(); - char* buffer = new char[output_size]; - - EXPECT_EQ(Status::OK(), writer.write_to_buffer(buffer, output_size)); - - StreamIndexReader reader; - EXPECT_EQ(Status::OK(), reader.init(buffer, output_size, OLAP_FIELD_TYPE_INT, true, true)); - - EXPECT_EQ(loop, reader.entry_count()); - - for (i = 0; i < loop; i++) { - const PositionEntryReader& e = reader.entry(i); - EXPECT_EQ(e.positions(0), i * 5); - EXPECT_EQ(e.positions(1), i * 6); - EXPECT_EQ(e.positions(2), i * 7); - } - delete[] buffer; - } - writer.reset(); - - // test 2 - { - uint32_t i = 0; - for (; i < loop; i++) { - entry.add_position(i); - entry.add_position(i * 2); - entry.add_position(i * 3); - entry.add_position(i * 4); - entry.add_position(i * 5); - entry.add_position(i * 6); - entry.add_position(i * 7); - - entry.set_statistic(&stat); - writer.add_index_entry(entry); - entry.reset_write_offset(); - } - - for (i = 0; i < loop; i++) { - PositionEntryWriter* e = writer.mutable_entry(i); - e->remove_written_position(0, 2); - } - - size_t output_size = writer.output_size(); - char* buffer = new char[output_size]; - - EXPECT_EQ(Status::OK(), writer.write_to_buffer(buffer, output_size)); - - StreamIndexReader reader; - EXPECT_EQ(Status::OK(), reader.init(buffer, output_size, OLAP_FIELD_TYPE_INT, true, true)); - - EXPECT_EQ(loop, reader.entry_count()); - - for (i = 0; i < loop; i++) { - const PositionEntryReader& e = reader.entry(i); - EXPECT_EQ(e.positions(0), i * 3); - EXPECT_EQ(e.positions(1), i * 4); - EXPECT_EQ(e.positions(2), i * 5); - EXPECT_EQ(e.positions(3), i * 6); - EXPECT_EQ(e.positions(4), i * 7); - } - delete[] buffer; - } - writer.reset(); - // test 3 - { - uint32_t i = 0; - for (; i < loop; i++) { - entry.add_position(i); - entry.add_position(i * 2); - entry.add_position(i * 3); - entry.add_position(i * 4); - entry.add_position(i * 5); - entry.add_position(i * 6); - entry.add_position(i * 7); - - entry.set_statistic(&stat); - writer.add_index_entry(entry); - entry.reset_write_offset(); - } - - for (i = 0; i < loop; i++) { - PositionEntryWriter* e = writer.mutable_entry(i); - e->remove_written_position(3, 2); - } - - size_t output_size = writer.output_size(); - char* buffer = new char[output_size]; - - EXPECT_EQ(Status::OK(), writer.write_to_buffer(buffer, output_size)); - - StreamIndexReader reader; - EXPECT_EQ(Status::OK(), reader.init(buffer, output_size, OLAP_FIELD_TYPE_INT, true, true)); - - EXPECT_EQ(loop, reader.entry_count()); - - for (i = 0; i < loop; i++) { - const PositionEntryReader& e = reader.entry(i); - EXPECT_EQ(e.positions(0), i * 1); - EXPECT_EQ(e.positions(1), i * 2); - EXPECT_EQ(e.positions(2), i * 3); - EXPECT_EQ(e.positions(3), i * 6); - EXPECT_EQ(e.positions(4), i * 7); - } - delete[] buffer; - } - writer.reset(); - // test 4 - { - uint32_t i = 0; - for (; i < loop; i++) { - entry.add_position(i); - entry.add_position(i * 2); - entry.add_position(i * 3); - entry.add_position(i * 4); - entry.add_position(i * 5); - entry.add_position(i * 6); - entry.add_position(i * 7); - - entry.set_statistic(&stat); - writer.add_index_entry(entry); - entry.reset_write_offset(); - } - - for (i = 0; i < loop; i++) { - PositionEntryWriter* e = writer.mutable_entry(i); - e->remove_written_position(4, 3); - } - - size_t output_size = writer.output_size(); - - char* buffer = new char[output_size]; - EXPECT_EQ(Status::OK(), writer.write_to_buffer(buffer, output_size)); - - StreamIndexReader reader; - EXPECT_EQ(Status::OK(), reader.init(buffer, output_size, OLAP_FIELD_TYPE_INT, true, true)); - - EXPECT_EQ(loop, reader.entry_count()); - - for (i = 0; i < loop; i++) { - const PositionEntryReader& e = reader.entry(i); - EXPECT_EQ(e.positions(0), i * 1); - EXPECT_EQ(e.positions(1), i * 2); - EXPECT_EQ(e.positions(2), i * 3); - EXPECT_EQ(e.positions(3), i * 4); - } - delete[] buffer; - } - writer.reset(); -} - -TEST_F(TestStreamIndex, test_statistic) { - ColumnStatistics stat; - EXPECT_EQ(Status::OK(), stat.init(OLAP_FIELD_TYPE_INT, true)); - - WrapperField* field = WrapperField::create_by_type(OLAP_FIELD_TYPE_INT); - - // start - EXPECT_STREQ(stat.minimum()->to_string().c_str(), "2147483647"); - EXPECT_STREQ(stat.maximum()->to_string().c_str(), "-2147483648"); - - // 1 - field->from_string("3"); - stat.add(*field); - EXPECT_STREQ(stat.minimum()->to_string().c_str(), "3"); - EXPECT_STREQ(stat.maximum()->to_string().c_str(), "3"); - - // 2 - field->from_string("5"); - stat.add(*field); - EXPECT_STREQ(stat.minimum()->to_string().c_str(), "3"); - EXPECT_STREQ(stat.maximum()->to_string().c_str(), "5"); - - // 3 - field->from_string("899"); - stat.add(*field); - EXPECT_STREQ(stat.minimum()->to_string().c_str(), "3"); - EXPECT_STREQ(stat.maximum()->to_string().c_str(), "899"); - - // 4 - field->from_string("-111"); - stat.add(*field); - EXPECT_STREQ(stat.minimum()->to_string().c_str(), "-111"); - EXPECT_STREQ(stat.maximum()->to_string().c_str(), "899"); - - stat.reset(); - // start - EXPECT_STREQ(stat.minimum()->to_string().c_str(), "2147483647"); - EXPECT_STREQ(stat.maximum()->to_string().c_str(), "-2147483648"); - - field->from_string("3"); - stat.add(*field); - field->from_string("6"); - stat.add(*field); - EXPECT_STREQ(stat.minimum()->to_string().c_str(), "3"); - EXPECT_STREQ(stat.maximum()->to_string().c_str(), "6"); - - ColumnStatistics stat2; - EXPECT_EQ(Status::OK(), stat2.init(OLAP_FIELD_TYPE_INT, true)); - - char buf[256]; - stat.write_to_buffer(buf, sizeof(buf)); - stat2.attach(buf); - - EXPECT_STREQ(stat2.minimum()->to_string().c_str(), "3"); - EXPECT_STREQ(stat2.maximum()->to_string().c_str(), "6"); - delete field; -} - -TEST_F(TestStreamIndex, statistic) { - StreamIndexWriter writer(OLAP_FIELD_TYPE_INT); - PositionEntryWriter entry; - ColumnStatistics stat; - - EXPECT_EQ(Status::OK(), stat.init(OLAP_FIELD_TYPE_INT, true)); - - WrapperField* field = WrapperField::create_by_type(OLAP_FIELD_TYPE_INT); - EXPECT_TRUE(nullptr != field); - char string_buffer[256]; - - static const uint32_t loop = 10; - uint32_t i = 0; - for (; i < loop; i++) { - entry.add_position(i); - entry.add_position(i * 2); - entry.add_position(i * 3); - - snprintf(string_buffer, sizeof(string_buffer), "%d", i * 9); - field->from_string(string_buffer); - stat.add(*field); - - snprintf(string_buffer, sizeof(string_buffer), "%d", i * 2); - field->from_string(string_buffer); - stat.add(*field); - - entry.set_statistic(&stat); - - writer.add_index_entry(entry); - entry.reset_write_offset(); - } - - size_t output_size = sizeof(StreamIndexHeader) + loop * sizeof(uint32_t) * 3 + - (1 + sizeof(int32_t)) * loop * 2; - EXPECT_EQ(output_size, writer.output_size()); - - char* buffer = new char[output_size]; - - EXPECT_EQ(Status::OK(), writer.write_to_buffer(buffer, output_size)); - - StreamIndexReader reader; - EXPECT_EQ(Status::OK(), reader.init(buffer, output_size, OLAP_FIELD_TYPE_INT, true, true)); - - EXPECT_EQ(loop, reader.entry_count()); - - for (i = 0; i < loop; i++) { - const PositionEntryReader& e = reader.entry(i); - EXPECT_EQ(e.positions(0), i); - EXPECT_EQ(e.positions(1), i * 2); - EXPECT_EQ(e.positions(2), i * 3); - } - delete[] buffer; - delete field; -} - -} // namespace doris diff --git a/be/test/olap/tablet_mgr_test.cpp b/be/test/olap/tablet_mgr_test.cpp index 6ac994e7cd..d0479949ba 100644 --- a/be/test/olap/tablet_mgr_test.cpp +++ b/be/test/olap/tablet_mgr_test.cpp @@ -24,7 +24,6 @@ #include "gtest/gtest.h" #include "json2pb/json_to_pb.h" #include "olap/olap_meta.h" -#include "olap/rowset/alpha_rowset.h" #include "olap/rowset/alpha_rowset_meta.h" #include "olap/rowset/rowset_meta_manager.h" #include "olap/storage_engine.h"