From 727f0374be166207c8e7dcc4df4ce81a5745b733 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Sun, 29 Sep 2024 09:19:17 +0800 Subject: [PATCH] [Refactor](inverted index) refactor inverted index compound predicates evaluate logic #38908 (#41385) cherry pick from #38908 --- .../segment_v2/inverted_index_reader.cpp | 47 ++ .../rowset/segment_v2/inverted_index_reader.h | 174 ++++- .../rowset/segment_v2/segment_iterator.cpp | 729 ++++-------------- .../olap/rowset/segment_v2/segment_iterator.h | 49 +- be/src/pipeline/exec/scan_operator.cpp | 20 - be/src/vec/exprs/vcompound_pred.h | 105 ++- be/src/vec/exprs/vectorized_fn_call.cpp | 56 +- be/src/vec/exprs/vectorized_fn_call.h | 6 +- be/src/vec/exprs/vexpr.cpp | 194 +++-- be/src/vec/exprs/vexpr.h | 20 +- be/src/vec/exprs/vexpr_context.cpp | 10 + be/src/vec/exprs/vexpr_context.h | 126 +++ be/src/vec/exprs/vin_predicate.cpp | 27 +- be/src/vec/exprs/vin_predicate.h | 2 +- be/src/vec/exprs/vliteral.h | 1 + be/src/vec/exprs/vmatch_predicate.cpp | 30 +- be/src/vec/exprs/vmatch_predicate.h | 3 +- .../functions/array/function_array_index.h | 86 +++ be/src/vec/functions/function.h | 39 +- be/src/vec/functions/function_multi_match.cpp | 119 +-- be/src/vec/functions/function_multi_match.h | 7 +- be/src/vec/functions/functions_comparison.h | 68 ++ be/src/vec/functions/in.h | 59 ++ be/src/vec/functions/match.cpp | 188 +++-- be/src/vec/functions/match.h | 5 + .../test_index_match_phrase_prefix.out | 24 +- .../test_all_index_hit_fault_injection.groovy | 6 +- .../test_index_inlist_fault_injection.groovy | 3 - ...est_index_lowercase_fault_injection.groovy | 1 + .../test_index_mow_fault_injection.groovy | 1 + ...test_need_read_data_fault_injection.groovy | 1 + .../test_topn_fault_injection.groovy | 1 + .../char_filter/test_char_replace.groovy | 2 +- .../test_index_compaction_dup_keys.groovy | 1 + .../test_index_compaction_null.groovy | 1 + .../test_index_compaction_unique_keys.groovy | 1 + ...ompaction_with_multi_index_segments.groovy | 1 + ...umulative_compaction_with_format_v2.groovy | 1 + .../test_drop_column_with_format_v2.groovy | 1 + .../test_drop_index_with_format_v2.groovy | 1 + .../test_mor_table_with_format_v2.groovy | 1 + .../test_mow_table_with_format_v2.groovy | 1 + .../test_rename_column_with_format_v2.groovy | 1 + ...e_replica_compaction_with_format_v2.groovy | 1 + .../test_storage_format_v1.groovy | 1 + ..._array_contains_with_inverted_index.groovy | 1 + .../inverted_index_p0/test_array_index.groovy | 1 + ..._array_with_inverted_index_all_type.groovy | 1 + .../test_chinese_analyzer.groovy | 1 + .../inverted_index_p0/test_compound.groovy | 1 + .../inverted_index_p0/test_compound_1.groovy | 1 + .../test_compound_inlist.groovy | 1 + .../test_count_on_index.groovy | 1 + .../test_count_on_index_2.groovy | 1 + .../inverted_index_p0/test_delete.groovy | 1 + .../test_equal_on_fulltext.groovy | 1 + .../test_index_chinese_column.groovy | 1 + .../test_index_complex_match.groovy | 1 + .../test_index_delete.groovy | 1 + .../test_index_empty_string.groovy | 2 +- .../test_index_equal_select.groovy | 1 + .../test_index_key_match_select.groovy | 1 + .../test_index_match_phrase.groovy | 1 + .../test_index_match_phrase_edge.groovy | 6 +- .../test_index_match_phrase_ordered.groovy | 6 +- .../test_index_match_phrase_prefix.groovy | 18 +- .../test_index_match_phrase_prefix_1.groovy | 6 +- .../test_index_match_phrase_slop.groovy | 6 +- .../test_index_match_regexp.groovy | 6 +- .../test_index_match_select.groovy | 1 + .../test_index_multi_match.groovy | 2 +- .../test_index_no_need_read_data.groovy | 1 + .../inverted_index_p0/test_index_null.groovy | 2 +- .../test_index_range_between_select.groovy | 1 + ...index_range_bigger_and_equal_select.groovy | 2 +- .../test_index_range_bigger_select.groovy | 2 +- .../test_index_range_in_select.groovy | 1 + .../test_index_range_not_in_select.groovy | 1 + ...ndex_range_smaller_and_equal_select.groovy | 2 +- .../test_index_range_smaller_select.groovy | 2 +- .../test_index_rqg_bug.groovy | 2 +- .../test_index_rqg_bug2.groovy | 2 +- .../test_index_rqg_bug3.groovy | 2 +- .../test_index_rqg_bug4.groovy | 1 + .../test_index_skip_read_data.groovy | 2 +- .../test_inverted_index.groovy | 1 + .../test_inverted_index_keyword.groovy | 2 +- .../test_inverted_index_mor.groovy | 2 +- .../test_inverted_index_null.groovy | 1 + .../test_inverted_index_null_ram_dir.groovy | 1 + .../inverted_index_p0/test_lowercase.groovy | 1 + .../test_match_query_without_index.groovy | 2 +- .../test_match_without_index.groovy | 1 + .../test_need_read_data.groovy | 1 + .../inverted_index_p0/test_null_index.groovy | 2 +- .../test_or_not_match.groovy | 1 + .../test_pk_no_need_read_data.groovy | 1 + .../inverted_index_p0/test_stopwords.groovy | 1 + .../inverted_index_p0/topn_clear_block.groovy | 1 + .../test_mow_with_null_sequence.groovy | 1 + .../unique_with_mow/test_pk_uk_case.groovy | 1 + .../test_primary_key_simple_case.groovy | 1 + .../test_unique_mow_sequence.groovy | 1 + .../inverted_index.groovy | 1 + .../suites/variant_p0/with_index/load.groovy | 3 +- .../variant_p0/with_index/var_index.groovy | 1 + 106 files changed, 1333 insertions(+), 1008 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index 99cb699062..1a20f84a1b 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -73,6 +73,47 @@ namespace doris::segment_v2 { +template +Status InvertedIndexQueryParamFactory::create_query_value( + const void* value, std::unique_ptr& result_param) { + using CPP_TYPE = typename PrimitiveTypeTraits::CppType; + std::unique_ptr> param = + InvertedIndexQueryParam::create_unique(); + auto&& storage_val = PrimitiveTypeConvertor::to_storage_field_type( + *reinterpret_cast(value)); + param->set_value(&storage_val); + result_param = std::move(param); + return Status::OK(); +}; + +#define CREATE_QUERY_VALUE_TEMPLATE(PT) \ + template Status InvertedIndexQueryParamFactory::create_query_value( \ + const void* value, std::unique_ptr& result_param); + +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_BOOLEAN) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_TINYINT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_SMALLINT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_INT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_BIGINT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_LARGEINT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_FLOAT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DOUBLE) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_VARCHAR) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATE) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATEV2) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATETIME) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATETIMEV2) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_CHAR) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMALV2) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL32) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL64) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL128I) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL256) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_HLL) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_STRING) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_IPV4) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_IPV6) + std::unique_ptr InvertedIndexReader::create_analyzer( InvertedIndexCtx* inverted_index_ctx) { std::unique_ptr analyzer; @@ -115,6 +156,10 @@ std::unique_ptr InvertedIndexReader::create_reader( return reader; } +std::string InvertedIndexReader::get_index_file_path() { + return _inverted_index_file_reader->get_index_file_path(&_index_meta); +} + void InvertedIndexReader::get_analyse_result(std::vector& analyse_result, lucene::util::Reader* reader, lucene::analysis::Analyzer* analyzer, @@ -182,6 +227,8 @@ Status InvertedIndexReader::read_null_bitmap(OlapReaderStatistics* stats, null_bitmap->runOptimize(); cache->insert(cache_key, null_bitmap, cache_handle); FINALIZE_INPUT(null_bitmap_in); + } else { + cache->insert(cache_key, null_bitmap, cache_handle); } if (owned_dir) { FINALIZE_INPUT(dir); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h index cd5d89d491..e8395903d7 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h @@ -33,6 +33,7 @@ #include "olap/rowset/segment_v2/inverted_index_desc.h" #include "olap/rowset/segment_v2/inverted_index_query_type.h" #include "olap/tablet_schema.h" +#include "runtime/primitive_type.h" #include "util/once.h" #define FINALIZE_INPUT(x) \ @@ -72,6 +73,104 @@ class InvertedIndexIterator; class InvertedIndexQueryCacheHandle; class InvertedIndexFileReader; struct InvertedIndexQueryInfo; +class InvertedIndexResultBitmap { +private: + std::shared_ptr _data_bitmap = nullptr; + std::shared_ptr _null_bitmap = nullptr; + +public: + // Default constructor + InvertedIndexResultBitmap() = default; + ~InvertedIndexResultBitmap() = default; + + // Constructor with arguments + InvertedIndexResultBitmap(std::shared_ptr data_bitmap, + std::shared_ptr null_bitmap) + : _data_bitmap(std::move(data_bitmap)), _null_bitmap(std::move(null_bitmap)) {} + + // Copy constructor + InvertedIndexResultBitmap(const InvertedIndexResultBitmap& other) + : _data_bitmap(std::make_shared(*other._data_bitmap)), + _null_bitmap(std::make_shared(*other._null_bitmap)) {} + + // Move constructor + InvertedIndexResultBitmap(InvertedIndexResultBitmap&& other) noexcept + : _data_bitmap(std::move(other._data_bitmap)), + _null_bitmap(std::move(other._null_bitmap)) {} + + // Copy assignment operator + InvertedIndexResultBitmap& operator=(const InvertedIndexResultBitmap& other) { + if (this != &other) { // Prevent self-assignment + _data_bitmap = std::make_shared(*other._data_bitmap); + _null_bitmap = std::make_shared(*other._null_bitmap); + } + return *this; + } + + // Move assignment operator + InvertedIndexResultBitmap& operator=(InvertedIndexResultBitmap&& other) noexcept { + if (this != &other) { // Prevent self-assignment + _data_bitmap = std::move(other._data_bitmap); + _null_bitmap = std::move(other._null_bitmap); + } + return *this; + } + + // Operator &= + InvertedIndexResultBitmap& operator&=(const InvertedIndexResultBitmap& other) { + if (_data_bitmap && _null_bitmap && other._data_bitmap && other._null_bitmap) { + auto new_null_bitmap = (*_data_bitmap & *other._null_bitmap) | + (*_null_bitmap & *other._data_bitmap) | + (*_null_bitmap & *other._null_bitmap); + *_data_bitmap &= *other._data_bitmap; + *_null_bitmap = std::move(new_null_bitmap); + } + return *this; + } + + // Operator |= + InvertedIndexResultBitmap& operator|=(const InvertedIndexResultBitmap& other) { + if (_data_bitmap && _null_bitmap && other._data_bitmap && other._null_bitmap) { + auto new_null_bitmap = (*_null_bitmap | *other._null_bitmap) - *_data_bitmap; + *_data_bitmap |= *other._data_bitmap; + *_null_bitmap = std::move(new_null_bitmap); + } + return *this; + } + + // NOT operation + const InvertedIndexResultBitmap& op_not(const roaring::Roaring* universe) const { + if (_data_bitmap && _null_bitmap) { + *_data_bitmap = *universe - *_data_bitmap - *_null_bitmap; + // The _null_bitmap remains unchanged. + } + return *this; + } + + // Operator -= + InvertedIndexResultBitmap& operator-=(const InvertedIndexResultBitmap& other) { + if (_data_bitmap && _null_bitmap && other._data_bitmap && other._null_bitmap) { + *_data_bitmap -= *other._data_bitmap; + *_data_bitmap -= *other._null_bitmap; + *_null_bitmap -= *other._null_bitmap; + } + return *this; + } + + void mask_out_null() { + if (_data_bitmap && _null_bitmap) { + *_data_bitmap -= *_null_bitmap; + } + } + + const std::shared_ptr& get_data_bitmap() const { return _data_bitmap; } + + const std::shared_ptr& get_null_bitmap() const { return _null_bitmap; } + + // Check if both bitmaps are empty + bool is_empty() const { return (_data_bitmap == nullptr && _null_bitmap == nullptr); } +}; + class InvertedIndexReader : public std::enable_shared_from_this { public: explicit InvertedIndexReader( @@ -135,7 +234,7 @@ public: virtual Status handle_searcher_cache(InvertedIndexCacheHandle* inverted_index_cache_handle, OlapReaderStatistics* stats); - + std::string get_index_file_path(); static Status create_index_searcher(lucene::store::Directory* dir, IndexSearcherPtr* searcher, MemTracker* mem_tracker, InvertedIndexReaderType reader_type); @@ -282,6 +381,79 @@ private: const KeyCoder* _value_key_coder {}; }; +/** + * @brief InvertedIndexQueryParamFactory is a factory class to create QueryValue object. + * we need a template function to make predict class like in_list_predict template class to use. + * also need a function with primitive type parameter to create inverted index query value. like some function expr: function_array_index + * Now we just mapping field value in query engine to storage field value + */ +class InvertedIndexQueryParamFactory { + ENABLE_FACTORY_CREATOR(InvertedIndexQueryParamFactory); + +public: + virtual ~InvertedIndexQueryParamFactory() = default; + + template + static Status create_query_value(const void* value, + std::unique_ptr& result_param); + + static Status create_query_value( + const PrimitiveType& primitiveType, const void* value, + std::unique_ptr& result_param) { + switch (primitiveType) { +#define M(TYPE) \ + case TYPE: { \ + return create_query_value(value, result_param); \ + } + M(PrimitiveType::TYPE_BOOLEAN) + M(PrimitiveType::TYPE_TINYINT) + M(PrimitiveType::TYPE_SMALLINT) + M(PrimitiveType::TYPE_INT) + M(PrimitiveType::TYPE_BIGINT) + M(PrimitiveType::TYPE_LARGEINT) + M(PrimitiveType::TYPE_FLOAT) + M(PrimitiveType::TYPE_DOUBLE) + M(PrimitiveType::TYPE_DECIMALV2) + M(PrimitiveType::TYPE_DECIMAL32) + M(PrimitiveType::TYPE_DECIMAL64) + M(PrimitiveType::TYPE_DECIMAL128I) + M(PrimitiveType::TYPE_DECIMAL256) + M(PrimitiveType::TYPE_DATE) + M(PrimitiveType::TYPE_DATETIME) + M(PrimitiveType::TYPE_CHAR) + M(PrimitiveType::TYPE_VARCHAR) + M(PrimitiveType::TYPE_STRING) +#undef M + default: + return Status::NotSupported("Unsupported primitive type {} for inverted index reader", + primitiveType); + } + }; + + virtual const void* get_value() const { + LOG_FATAL( + "Execution reached an undefined behavior code path in " + "InvertedIndexQueryParamFactory"); + __builtin_unreachable(); + }; +}; + +template +class InvertedIndexQueryParam : public InvertedIndexQueryParamFactory { + ENABLE_FACTORY_CREATOR(InvertedIndexQueryParam); + using storage_val = typename PrimitiveTypeTraits::StorageFieldType; + +public: + void set_value(const storage_val* value) { + _value = *reinterpret_cast(value); + } + + const void* get_value() const override { return &_value; } + +private: + storage_val _value; +}; + class InvertedIndexIterator { ENABLE_FACTORY_CREATOR(InvertedIndexIterator); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index bfd8538441..f46347c3d2 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -297,26 +297,18 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { _block_rowids.resize(_opts.block_row_max); // compound predicates - _col_preds_except_leafnode_of_andnode.clear(); + //TODO: need to check can_apply_predicate_safely in expr + /*_col_preds_except_leafnode_of_andnode.clear(); for (const auto& predicate : opts.column_predicates_except_leafnode_of_andnode) { if (!_segment->can_apply_predicate_safely(predicate->column_id(), predicate, *_schema, _opts.io_ctx.reader_type)) { continue; } _col_preds_except_leafnode_of_andnode.push_back(predicate); - } + }*/ _remaining_conjunct_roots = opts.remaining_conjunct_roots; - _common_expr_ctxs_push_down = opts.common_expr_ctxs_push_down; - _enable_common_expr_pushdown = !_common_expr_ctxs_push_down.empty(); - _column_predicate_info.reset(new ColumnPredicateInfo()); - for (auto& expr : _remaining_conjunct_roots) { - _calculate_pred_in_remaining_conjunct_root(expr); - } - _calculate_func_in_remaining_conjunct_root(); - - _column_predicate_info.reset(new ColumnPredicateInfo()); if (_schema->rowid_col_idx() > 0) { _record_rowids = true; } @@ -366,8 +358,9 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) { } } + RETURN_IF_ERROR(_construct_compound_expr_context()); + _enable_common_expr_pushdown = !_common_expr_ctxs_push_down.empty(); _initialize_predicate_results(); - return Status::OK(); } @@ -375,33 +368,10 @@ void SegmentIterator::_initialize_predicate_results() { // Initialize from _col_predicates for (auto* pred : _col_predicates) { int cid = pred->column_id(); - std::string pred_sign = _gen_predicate_result_sign(pred); - _column_predicate_inverted_index_status[cid][pred_sign] = false; + _column_predicate_inverted_index_status[cid][pred] = false; } - // Initialize from _col_preds_except_leafnode_of_andnode - for (auto* pred : _col_preds_except_leafnode_of_andnode) { - int cid = pred->column_id(); - std::string pred_sign = _gen_predicate_result_sign(pred); - _column_predicate_inverted_index_status[cid][pred_sign] = false; - } - - // Initialize from _column_pred_in_remaining_vconjunct - for (auto& preds_in_remaining_vconjuct : _column_pred_in_remaining_vconjunct) { - for (auto& pred_info : preds_in_remaining_vconjuct.second) { - int cid = _schema->column_id(pred_info.column_id); - std::string pred_sign = _gen_predicate_result_sign(&pred_info); - _column_predicate_inverted_index_status[cid][pred_sign] = false; - } - } - - // Initialize from _func_name_to_result_sign - for (auto& iter : _func_name_to_result_sign) { - for (auto& pred_sign : iter.second) { - auto column_id = _opts.tablet_schema->field_index(iter.first); - _column_predicate_inverted_index_status[column_id][pred_sign] = false; - } - } + _calculate_expr_in_remaining_conjunct_root(); } Status SegmentIterator::init_iterators() { @@ -542,47 +512,34 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() { return Status::OK(); } - if (config::enable_index_apply_preds_except_leafnode_of_andnode) { - size_t input_rows = _row_bitmap.cardinality(); - RETURN_IF_ERROR(_apply_index_except_leafnode_of_andnode()); - if (_can_filter_by_preds_except_leafnode_of_andnode()) { - for (auto it = _remaining_conjunct_roots.begin(); - it != _remaining_conjunct_roots.end();) { - _pred_except_leafnode_of_andnode_evaluate_result.clear(); - auto res = _execute_predicates_except_leafnode_of_andnode(*it); - VLOG_DEBUG << "_execute_predicates_except_leafnode_of_andnode expr: " - << (*it)->debug_string() << " res: " << res; - if (res.ok() && _pred_except_leafnode_of_andnode_evaluate_result.size() == 1) { - _row_bitmap &= _pred_except_leafnode_of_andnode_evaluate_result[0]; - // Delete expr after it obtains the final result. - { - std::erase_if(_common_expr_ctxs_push_down, - [&it](const auto& iter) { return iter->root() == *it; }); - VLOG_DEBUG << "_remaining_conjunct_roots erase expr: " - << (*it)->debug_string(); - it = _remaining_conjunct_roots.erase(it); - } - } else { - ++it; - } - } - _col_preds_except_leafnode_of_andnode.clear(); - compound_func_exprs.clear(); - // 1. if all conditions in the compound hit the inverted index and there are no other expr to handle. - // 2. then there is no need to generate index_result_column. - if (_enable_common_expr_pushdown && _remaining_conjunct_roots.empty()) { - for (auto& iter : _rowid_result_for_index) { - iter.second.first = false; - } - } - } - _opts.stats->rows_inverted_index_filtered += (input_rows - _row_bitmap.cardinality()); - } - RETURN_IF_ERROR(_apply_bitmap_index()); RETURN_IF_ERROR(_apply_inverted_index()); + RETURN_IF_ERROR(_apply_index_expr()); + size_t input_rows = _row_bitmap.cardinality(); + for (auto it = _common_expr_ctxs_push_down.begin(); it != _common_expr_ctxs_push_down.end();) { + if ((*it)->all_expr_inverted_index_evaluated()) { + const auto* result = + (*it)->get_inverted_index_context()->get_inverted_index_result_for_expr( + (*it)->root().get()); + if (result != nullptr) { + _row_bitmap &= *result->get_data_bitmap(); + auto root = (*it)->root(); + auto iter_find = std::find(_remaining_conjunct_roots.begin(), + _remaining_conjunct_roots.end(), root); + if (iter_find != _remaining_conjunct_roots.end()) { + _remaining_conjunct_roots.erase(iter_find); + } + it = _common_expr_ctxs_push_down.erase(it); + } + } else { + ++it; + } + } + + _opts.stats->rows_inverted_index_filtered += (input_rows - _row_bitmap.cardinality()); for (auto cid : _schema->column_ids()) { - bool result_true = _check_all_predicates_passed_inverted_index_for_column(cid); + bool result_true = _check_all_conditions_passed_inverted_index_for_column(cid); + if (result_true) { _need_read_data_indices[cid] = false; } @@ -795,127 +752,6 @@ Status SegmentIterator::_extract_common_expr_columns(const vectorized::VExprSPtr return Status::OK(); } -Status SegmentIterator::_execute_predicates_except_leafnode_of_andnode( - const vectorized::VExprSPtr& expr) { - if (expr == nullptr) { - return Status::OK(); - } - - auto& children = expr->children(); - for (int i = 0; i < children.size(); ++i) { - RETURN_IF_ERROR(_execute_predicates_except_leafnode_of_andnode(children[i])); - } - - auto node_type = expr->node_type(); - if (node_type == TExprNodeType::SLOT_REF) { - auto slot_expr = std::dynamic_pointer_cast(expr); - _column_predicate_info->column_name = expr->expr_name(); - _column_predicate_info->column_id = slot_expr->column_id(); - } else if (_is_literal_node(node_type)) { - auto v_literal_expr = std::dynamic_pointer_cast(expr); - _column_predicate_info->query_values.insert(v_literal_expr->value()); - } else if (node_type == TExprNodeType::BINARY_PRED || node_type == TExprNodeType::MATCH_PRED || - node_type == TExprNodeType::IN_PRED || node_type == TExprNodeType::FUNCTION_CALL) { - std::string result_sign; - if (node_type == TExprNodeType::FUNCTION_CALL) { - result_sign = - BeConsts::BLOCK_TEMP_COLUMN_PREFIX + std::to_string(expr->index_unique_id()); - } else { - if (node_type == TExprNodeType::MATCH_PRED) { - _column_predicate_info->query_op = "match"; - } else if (node_type == TExprNodeType::IN_PRED) { - if (expr->op() == TExprOpcode::type::FILTER_IN) { - _column_predicate_info->query_op = "in"; - } else { - _column_predicate_info->query_op = "not_in"; - } - } else { - _column_predicate_info->query_op = expr->fn().name.function_name; - } - result_sign = _gen_predicate_result_sign(_column_predicate_info.get()); - } - - // get child condition result in compound conditions - _column_predicate_info.reset(new ColumnPredicateInfo()); - VLOG_DEBUG << "result_sign " << result_sign; - if (_rowid_result_for_index.count(result_sign) > 0 && - _rowid_result_for_index[result_sign].first) { - auto apply_result = _rowid_result_for_index[result_sign].second; - _pred_except_leafnode_of_andnode_evaluate_result.push_back(apply_result); - } else { - return Status::InvalidArgument( - "_execute_predicates_except_leafnode_of_andnode has no result for {}", - result_sign); - } - } else if (node_type == TExprNodeType::COMPOUND_PRED) { - auto function_name = expr->fn().name.function_name; - // execute logic function - RETURN_IF_ERROR(_execute_compound_fn(function_name)); - } else { - return Status::InvalidArgument( - "_execute_predicates_except_leafnode_of_andnode not supported for TExprNodeType:{}", - node_type); - } - - return Status::OK(); -} - -Status SegmentIterator::_execute_compound_fn(const std::string& function_name) { - auto size = _pred_except_leafnode_of_andnode_evaluate_result.size(); - if (function_name == "and") { - if (size < 2) { - return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 2", function_name, - size); - } - _pred_except_leafnode_of_andnode_evaluate_result.at(size - 2) &= - _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1); - _pred_except_leafnode_of_andnode_evaluate_result.pop_back(); - } else if (function_name == "or") { - if (size < 2) { - return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 2", function_name, - size); - } - _pred_except_leafnode_of_andnode_evaluate_result.at(size - 2) |= - _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1); - _pred_except_leafnode_of_andnode_evaluate_result.pop_back(); - } else if (function_name == "not") { - if (size < 1) { - return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 1", function_name, - size); - } - roaring::Roaring tmp = _row_bitmap; - tmp -= _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1); - _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1) = tmp; - } - return Status::OK(); -} - -bool SegmentIterator::_can_filter_by_preds_except_leafnode_of_andnode() { - // no compound predicates push down, so no need to filter - if (_col_preds_except_leafnode_of_andnode.empty() && compound_func_exprs.empty()) { - return false; - } - for (auto pred : _col_preds_except_leafnode_of_andnode) { - if (_not_apply_index_pred.count(pred->column_id()) || - (!_check_apply_by_inverted_index(pred, true))) { - return false; - } - // all predicates are evaluated by index, then true, else false - std::string pred_result_sign = _gen_predicate_result_sign(pred); - if (_rowid_result_for_index.count(pred_result_sign) == 0) { - return false; - } - } - for (const auto& expr : compound_func_exprs) { - std::string pred_result_sign = - BeConsts::BLOCK_TEMP_COLUMN_PREFIX + std::to_string(expr->index_unique_id()); - if (!_rowid_result_for_index.contains(pred_result_sign)) { - return false; - } - } - return true; -} - bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool pred_in_compound) { if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_inverted_index_query) { return false; @@ -973,74 +809,20 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool return true; } -Status SegmentIterator::_apply_inverted_index_except_leafnode_of_andnode( - ColumnPredicate* pred, roaring::Roaring* output_result) { - RETURN_IF_ERROR(pred->evaluate(_storage_name_and_type[pred->column_id()], - _inverted_index_iterators[pred->column_id()].get(), num_rows(), - output_result)); - return Status::OK(); -} - -Status SegmentIterator::_apply_index_except_leafnode_of_andnode() { - for (auto* pred : _col_preds_except_leafnode_of_andnode) { - auto column_id = pred->column_id(); - auto pred_type = pred->type(); - bool is_support = pred_type == PredicateType::EQ || pred_type == PredicateType::NE || - pred_type == PredicateType::LT || pred_type == PredicateType::LE || - pred_type == PredicateType::GT || pred_type == PredicateType::GE || - pred_type == PredicateType::MATCH || - pred_type == PredicateType::IN_LIST || - pred_type == PredicateType::NOT_IN_LIST; - if (!is_support) { - continue; - } - - bool can_apply_by_inverted_index = _check_apply_by_inverted_index(pred, true); - roaring::Roaring bitmap = _row_bitmap; - Status res = Status::OK(); - if (can_apply_by_inverted_index) { - res = _apply_inverted_index_except_leafnode_of_andnode(pred, &bitmap); - } else { - continue; - } - - bool need_remaining_after_evaluate = _column_has_fulltext_index(column_id) && - PredicateTypeTraits::is_equal_or_list(pred_type); - if (!res.ok()) { - if (_downgrade_without_index(res, need_remaining_after_evaluate)) { - // downgrade without index query - _not_apply_index_pred.insert(column_id); +Status SegmentIterator::_apply_index_expr() { + for (const auto& expr_ctx : _common_expr_ctxs_push_down) { + if (Status st = expr_ctx->evaluate_inverted_index(num_rows()); !st.ok()) { + if (_downgrade_without_index(st) || st.code() == ErrorCode::NOT_IMPLEMENTED_ERROR) { continue; - } - LOG(WARNING) << "failed to evaluate index" - << ", column predicate type: " << pred->pred_type_string(pred->type()) - << ", error msg: " << res.to_string(); - return res; - } - - std::string pred_result_sign = _gen_predicate_result_sign(pred); - _rowid_result_for_index.emplace(pred_result_sign, std::make_pair(true, std::move(bitmap))); - if (!pred->predicate_params()->marked_by_runtime_filter) { - _column_predicate_inverted_index_status[column_id][pred_result_sign] = true; - } - } - - for (const auto& expr : compound_func_exprs) { - roaring::Roaring bitmap = _row_bitmap; - auto result = std::make_shared(); - RETURN_IF_ERROR(execute_func_expr(expr, result)); - bitmap &= *result; - std::string result_sign = - BeConsts::BLOCK_TEMP_COLUMN_PREFIX + std::to_string(expr->index_unique_id()); - _rowid_result_for_index.emplace(result_sign, std::make_pair(true, std::move(bitmap))); - for (const auto& child_expr : expr->children()) { - if (child_expr->node_type() == TExprNodeType::type::SLOT_REF) { - auto column_id = _opts.tablet_schema->field_index(child_expr->expr_name()); - _column_predicate_inverted_index_status[column_id][result_sign] = true; + } else { + // other code is not to be handled, we should just break + LOG(WARNING) << "failed to evaluate inverted index for expr_ctx: " + << expr_ctx->root()->debug_string() + << ", error msg: " << st.to_string(); + return st; } } } - return Status::OK(); } @@ -1071,47 +853,6 @@ bool SegmentIterator::_downgrade_without_index(Status res, bool need_remaining) return false; } -std::string SegmentIterator::_gen_predicate_result_sign(ColumnPredicate* predicate) { - std::string pred_result_sign; - - auto column_desc = _schema->column(predicate->column_id()); - auto pred_type = predicate->type(); - auto predicate_params = predicate->predicate_params(); - - std::string col_name = column_desc->name(); - - if (column_desc->path() != nullptr) { - const static std::string pattern = "(CAST {}(Nullable(Variant)) TO {})"; - // indicate a subcolumn access for variant, using the expression pattern as pred result sign name - col_name = fmt::format(pattern, col_name, - _storage_name_and_type[predicate->column_id()].second->get_name()); - } - - pred_result_sign = BeConsts::BLOCK_TEMP_COLUMN_PREFIX + col_name + "_" + - predicate->pred_type_string(pred_type) + "_" + - join(predicate_params->values, ","); - VLOG_DEBUG << "_gen_predicate_result_sign: " << pred_result_sign; - return pred_result_sign; -} - -std::string SegmentIterator::_gen_predicate_result_sign(ColumnPredicateInfo* predicate_info) { - auto column_desc = _schema->column(_schema->column_id(predicate_info->column_id)); - std::string col_name = predicate_info->column_name; - if (column_desc->path() != nullptr) { - const static std::string pattern = "(CAST {}(Nullable(Variant)) TO {})"; - // indicate a subcolumn access for variant, using the expression pattern as pred result sign name - col_name = fmt::format(pattern, col_name, - _storage_name_and_type[_schema->column_id(predicate_info->column_id)] - .second->get_name()); - } - std::string pred_result_sign; - pred_result_sign = BeConsts::BLOCK_TEMP_COLUMN_PREFIX + col_name + "_" + - predicate_info->query_op + "_" + - boost::join(predicate_info->query_values, ","); - VLOG_DEBUG << "_gen_predicate_result_sign: " << pred_result_sign; - return pred_result_sign; -} - bool SegmentIterator::_column_has_fulltext_index(int32_t cid) { bool has_fulltext_index = _inverted_index_iterators[cid] != nullptr && _inverted_index_iterators[cid]->get_inverted_index_reader_type() == @@ -1124,10 +865,6 @@ inline bool SegmentIterator::_inverted_index_not_support_pred_type(const Predica return type == PredicateType::BF || type == PredicateType::BITMAP_FILTER; } -#define all_predicates_are_range_predicate(predicate_set) \ - std::all_of(predicate_set.begin(), predicate_set.end(), \ - [](const ColumnPredicate* p) { return PredicateTypeTraits::is_range(p->type()); }) - Status SegmentIterator::_apply_inverted_index_on_column_predicate( ColumnPredicate* pred, std::vector& remaining_predicates, bool* continue_apply) { @@ -1150,12 +887,6 @@ Status SegmentIterator::_apply_inverted_index_on_column_predicate( return res; } - auto pred_type = pred->type(); - if (pred_type == PredicateType::MATCH || pred_type == PredicateType::IN_LIST) { - std::string pred_result_sign = _gen_predicate_result_sign(pred); - _rowid_result_for_index.emplace(pred_result_sign, std::make_pair(false, _row_bitmap)); - } - if (_row_bitmap.isEmpty()) { // all rows have been pruned, no need to process further predicates *continue_apply = false; @@ -1166,52 +897,7 @@ Status SegmentIterator::_apply_inverted_index_on_column_predicate( return Status::OK(); } if (!pred->predicate_params()->marked_by_runtime_filter) { - std::string pred_result_sign = _gen_predicate_result_sign(pred); - _column_predicate_inverted_index_status[pred->column_id()][pred_result_sign] = true; - } - } - return Status::OK(); -} - -Status SegmentIterator::_apply_inverted_index_on_block_column_predicate( - ColumnId column_id, MutilColumnBlockPredicate* pred, - std::set& no_need_to_pass_column_predicate_set, - bool* continue_apply) { - bool handle_by_fulltext = _column_has_fulltext_index(column_id); - std::set predicate_set {}; - - pred->get_all_column_predicate(predicate_set); - - //four requirements here. - //1. Column has inverted index - //2. There are multiple predicates for this column. - //3. All the predicates are range predicate. - //4. if it's under fulltext parser type, we need to skip inverted index evaluate. - if (_inverted_index_iterators[column_id] != nullptr && predicate_set.size() > 1 && - all_predicates_are_range_predicate(predicate_set) && !handle_by_fulltext) { - roaring::Roaring output_result = _row_bitmap; - - std::string column_name = _schema->column(column_id)->name(); - - auto res = pred->evaluate(column_name, _inverted_index_iterators[column_id].get(), - num_rows(), &_row_bitmap); - - if (res.ok()) { - no_need_to_pass_column_predicate_set.insert(predicate_set.begin(), predicate_set.end()); - if (_row_bitmap.isEmpty()) { - // all rows have been pruned, no need to process further predicates - *continue_apply = false; - } - return res; - } else { - //TODO:mock until AndBlockColumnPredicate evaluate is ok. - if (res.code() == ErrorCode::INVERTED_INDEX_NOT_IMPLEMENTED) { - return Status::OK(); - } - LOG(WARNING) << "failed to evaluate index" - << ", column predicate type: range predicate" - << ", error msg: " << res; - return res; + _column_predicate_inverted_index_status[pred->column_id()][pred] = true; } } return Status::OK(); @@ -1310,63 +996,49 @@ Status SegmentIterator::_apply_inverted_index() { } } - // delete from _common_expr_ctxs_push_down if a MATCH predicate will be removed from _col_predicates - // since it's not necessary to eval it anymore to avoid index miss, which is added in _normalize_predicate - for (auto pred : _col_predicates) { - auto* match_pred = dynamic_cast(pred); - if (!match_pred || - !_is_match_predicate_and_not_remaining(match_pred, remaining_predicates)) { - continue; - } - - for (auto it = _common_expr_ctxs_push_down.begin(); - it != _common_expr_ctxs_push_down.end();) { - if (_is_target_expr_match_predicate((*it)->root(), match_pred, _schema.get())) { - _delete_expr_from_conjunct_roots((*it)->root(), _remaining_conjunct_roots); - it = _common_expr_ctxs_push_down.erase(it); - VLOG_DEBUG << "delete expr from _remaining_conjunct_roots " - << (*it)->root()->debug_string(); - } else { - ++it; - } - } - } - - for (const auto& expr : no_compound_func_exprs) { - auto result = std::make_shared(); - RETURN_IF_ERROR(execute_func_expr(expr, result)); - _row_bitmap &= *result; - for (auto it = _remaining_conjunct_roots.begin(); it != _remaining_conjunct_roots.end();) { - if (*it == expr) { - std::erase_if(_common_expr_ctxs_push_down, - [&it](const auto& iter) { return iter->root() == *it; }); - it = _remaining_conjunct_roots.erase(it); - } else { - ++it; - } - } - std::string result_sign = - BeConsts::BLOCK_TEMP_COLUMN_PREFIX + std::to_string(expr->index_unique_id()); - for (const auto& child_expr : expr->children()) { - if (child_expr->node_type() == TExprNodeType::type::SLOT_REF) { - auto column_id = _opts.tablet_schema->field_index(child_expr->expr_name()); - _column_predicate_inverted_index_status[column_id][result_sign] = true; - } - } - } - _col_predicates = std::move(remaining_predicates); _opts.stats->rows_inverted_index_filtered += (input_rows - _row_bitmap.cardinality()); return Status::OK(); } -bool SegmentIterator::_check_all_predicates_passed_inverted_index_for_column(ColumnId cid, +/** + * @brief Checks if all conditions related to a specific column have passed in both + * `_column_predicate_inverted_index_status` and `_common_expr_inverted_index_status`. + * + * This function first checks the conditions in `_column_predicate_inverted_index_status` + * for the given `ColumnId`. If all conditions pass, it sets `default_return` to `true`. + * It then checks the conditions in `_common_expr_inverted_index_status` for the same column. + * + * The function returns `true` if all conditions in both maps pass. If any condition fails + * in either map, the function immediately returns `false`. If the column does not exist + * in one of the maps, the function returns `default_return`. + * + * @param cid The ColumnId of the column to check. + * @param default_return The default value to return if the column is not found in the status maps. + * @return true if all conditions in both status maps pass, or if the column is not found + * and `default_return` is true. + * @return false if any condition in either status map fails, or if the column is not found + * and `default_return` is false. + */ +bool SegmentIterator::_check_all_conditions_passed_inverted_index_for_column(ColumnId cid, bool default_return) { - auto it = _column_predicate_inverted_index_status.find(cid); - if (it != _column_predicate_inverted_index_status.end()) { - const auto& pred_map = it->second; - return std::all_of(pred_map.begin(), pred_map.end(), - [](const auto& pred_entry) { return pred_entry.second; }); + auto pred_it = _column_predicate_inverted_index_status.find(cid); + if (pred_it != _column_predicate_inverted_index_status.end()) { + const auto& pred_map = pred_it->second; + bool pred_passed = std::all_of(pred_map.begin(), pred_map.end(), + [](const auto& pred_entry) { return pred_entry.second; }); + if (!pred_passed) { + return false; + } else { + default_return = true; + } + } + + auto expr_it = _common_expr_inverted_index_status.find(cid); + if (expr_it != _common_expr_inverted_index_status.end()) { + const auto& expr_map = expr_it->second; + return std::all_of(expr_map.begin(), expr_map.end(), + [](const auto& expr_entry) { return expr_entry.second; }); } return default_return; } @@ -2043,15 +1715,16 @@ Status SegmentIterator::_read_columns_by_index(uint32_t nrows_read_limit, uint32 } DBUG_EXECUTE_IF("segment_iterator._read_columns_by_index", { + auto col_name = _opts.tablet_schema->column(cid).name(); auto debug_col_name = DebugPoints::instance()->get_debug_param_or_default( "segment_iterator._read_columns_by_index", "column_name", ""); - if (debug_col_name.empty()) { - return Status::Error("does not need to read data"); + if (debug_col_name.empty() && col_name != "__DORIS_DELETE_SIGN__") { + return Status::Error("does not need to read data, {}", + col_name); } - auto col_name = _opts.tablet_schema->column(cid).name(); if (debug_col_name.find(col_name) != std::string::npos) { return Status::Error("does not need to read data, {}", - debug_col_name); + col_name); } }) @@ -2462,15 +2135,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) { return Status::EndOfFile("no more data in segment"); } - DBUG_EXECUTE_IF("segment_iterator._rowid_result_for_index", { - for (auto& iter : _rowid_result_for_index) { - if (iter.second.first) { - return Status::Error( - "_rowid_result_for_index exists true"); - } - } - }) - if (!_is_need_vec_eval && !_is_need_short_eval && !_is_need_expr_eval) { if (_non_predicate_columns.empty()) { return Status::InternalError("_non_predicate_columns is empty"); @@ -2478,9 +2142,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) { RETURN_IF_ERROR(_convert_to_expected_type(_first_read_column_ids)); RETURN_IF_ERROR(_convert_to_expected_type(_non_predicate_columns)); _output_non_pred_columns(block); - if (!_enable_common_expr_pushdown || !_remaining_conjunct_roots.empty()) { - _output_index_result_column(nullptr, 0, block); - } } else { uint16_t selected_size = _current_batch_rows_read; uint16_t sel_rowid_idx[selected_size]; @@ -2537,12 +2198,12 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) { auto col_const = vectorized::ColumnConst::create(std::move(res_column), selected_size); block->replace_by_position(0, std::move(col_const)); - _output_index_result_column(sel_rowid_idx, selected_size, block); + _output_index_result_column_for_expr(sel_rowid_idx, selected_size, block); block->shrink_char_type_column_suffix_zero(_char_type_idx_no_0); RETURN_IF_ERROR(_execute_common_expr(sel_rowid_idx, selected_size, block)); block->replace_by_position(0, std::move(col0)); } else { - _output_index_result_column(sel_rowid_idx, selected_size, block); + _output_index_result_column_for_expr(sel_rowid_idx, selected_size, block); block->shrink_char_type_column_suffix_zero(_char_type_idx); RETURN_IF_ERROR(_execute_common_expr(sel_rowid_idx, selected_size, block)); } @@ -2574,12 +2235,12 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) { auto col_const = vectorized::ColumnConst::create(std::move(res_column), selected_size); block->replace_by_position(0, std::move(col_const)); - _output_index_result_column(sel_rowid_idx, selected_size, block); + _output_index_result_column_for_expr(sel_rowid_idx, selected_size, block); block->shrink_char_type_column_suffix_zero(_char_type_idx_no_0); RETURN_IF_ERROR(_execute_common_expr(sel_rowid_idx, selected_size, block)); block->replace_by_position(0, std::move(col0)); } else { - _output_index_result_column(sel_rowid_idx, selected_size, block); + _output_index_result_column_for_expr(sel_rowid_idx, selected_size, block); block->shrink_char_type_column_suffix_zero(_char_type_idx); RETURN_IF_ERROR(_execute_common_expr(sel_rowid_idx, selected_size, block)); } @@ -2613,10 +2274,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) { RETURN_IF_ERROR(_convert_to_expected_type(_non_predicate_columns)); // step5: output columns _output_non_pred_columns(block); - - if (!_is_need_expr_eval) { - _output_index_result_column(sel_rowid_idx, selected_size, block); - } } // shrink char_type suffix zero data @@ -2693,53 +2350,42 @@ uint16_t SegmentIterator::_evaluate_common_expr_filter(uint16_t* sel_rowid_idx, } } -void SegmentIterator::_output_index_result_column(uint16_t* sel_rowid_idx, uint16_t select_size, - vectorized::Block* block) { +void SegmentIterator::_output_index_result_column_for_expr(uint16_t* sel_rowid_idx, + uint16_t select_size, + vectorized::Block* block) { SCOPED_RAW_TIMER(&_opts.stats->output_index_result_column_timer); if (block->rows() == 0) { return; } + for (auto& expr_ctx : _common_expr_ctxs_push_down) { + for (auto& inverted_index_result_bitmap_for_expr : + expr_ctx->get_inverted_index_context()->get_inverted_index_result_bitmap()) { + const auto* expr = inverted_index_result_bitmap_for_expr.first; + const auto& index_result_bitmap = + inverted_index_result_bitmap_for_expr.second.get_data_bitmap(); + auto index_result_column = vectorized::ColumnUInt8::create(); + vectorized::ColumnUInt8::Container& vec_match_pred = index_result_column->get_data(); + vec_match_pred.resize(block->rows()); + size_t idx_in_selected = 0; + roaring::BulkContext bulk_context; - for (auto& iter : _rowid_result_for_index) { - _columns_to_filter.push_back(block->columns()); - block->insert({vectorized::ColumnUInt8::create(), - std::make_shared(), iter.first}); - if (!iter.second.first) { - // predicate not in compound query - block->get_by_name(iter.first).column = - vectorized::DataTypeUInt8().create_column_const(block->rows(), (uint8_t)1); - continue; - } - _build_index_result_column(sel_rowid_idx, select_size, block, iter.first, - iter.second.second); - } -} - -void SegmentIterator::_build_index_result_column(const uint16_t* sel_rowid_idx, - uint16_t select_size, vectorized::Block* block, - const std::string& pred_result_sign, - const roaring::Roaring& index_result) { - auto index_result_column = vectorized::ColumnUInt8::create(); - vectorized::ColumnUInt8::Container& vec_match_pred = index_result_column->get_data(); - vec_match_pred.resize(block->rows()); - size_t idx_in_selected = 0; - roaring::BulkContext bulk_context; - - for (uint32_t i = 0; i < _current_batch_rows_read; i++) { - auto rowid = _block_rowids[i]; - if (sel_rowid_idx == nullptr || - (idx_in_selected < select_size && i == sel_rowid_idx[idx_in_selected])) { - if (index_result.containsBulk(bulk_context, rowid)) { - vec_match_pred[idx_in_selected] = true; - } else { - vec_match_pred[idx_in_selected] = false; + for (uint32_t i = 0; i < _current_batch_rows_read; i++) { + auto rowid = _block_rowids[i]; + if (sel_rowid_idx == nullptr || + (idx_in_selected < select_size && i == sel_rowid_idx[idx_in_selected])) { + if (index_result_bitmap->containsBulk(bulk_context, rowid)) { + vec_match_pred[idx_in_selected] = true; + } else { + vec_match_pred[idx_in_selected] = false; + } + idx_in_selected++; + } } - idx_in_selected++; + DCHECK(block->rows() == vec_match_pred.size()); + expr_ctx->get_inverted_index_context()->set_inverted_index_result_column_for_expr( + expr, std::move(index_result_column)); } } - DCHECK(block->rows() == vec_match_pred.size()); - auto index_result_position = block->get_position_by_name(pred_result_sign); - block->replace_by_position(index_result_position, std::move(index_result_column)); } void SegmentIterator::_convert_dict_code_for_predicate_if_necessary() { @@ -2790,125 +2436,45 @@ Status SegmentIterator::current_block_row_locations(std::vector* bl return Status::OK(); } -void SegmentIterator::_calculate_pred_in_remaining_conjunct_root( - const vectorized::VExprSPtr& expr) { - if (expr == nullptr) { - return; - } - - if (expr->fn().name.function_name == "multi_match") { - return; - } - - auto& children = expr->children(); - for (int i = 0; i < children.size(); ++i) { - _calculate_pred_in_remaining_conjunct_root(children[i]); - } - - auto node_type = expr->node_type(); - if (node_type == TExprNodeType::SLOT_REF) { - auto slot_expr = std::dynamic_pointer_cast(expr); - if (_column_predicate_info->column_name.empty()) { - _column_predicate_info->column_name = expr->expr_name(); - _column_predicate_info->column_id = slot_expr->column_id(); - } else { - // If column name already exists, create a new ColumnPredicateInfo - // if expr is columnA > columnB, then column name will exist, in this situation, we need to add it to _column_pred_in_remaining_vconjunct - auto new_column_pred_info = std::make_shared(); - new_column_pred_info->column_name = expr->expr_name(); - new_column_pred_info->column_id = slot_expr->column_id(); - _column_pred_in_remaining_vconjunct[new_column_pred_info->column_name].push_back( - *new_column_pred_info); - } - } else if (_is_literal_node(node_type)) { - auto v_literal_expr = static_cast(expr.get()); - _column_predicate_info->query_values.insert(v_literal_expr->value()); - } else if (node_type == TExprNodeType::NULL_LITERAL) { - if (!_column_predicate_info->column_name.empty()) { - auto v_literal_expr = static_cast(expr.get()); - _column_predicate_info->query_values.insert(v_literal_expr->value()); - } - } else { - if (node_type == TExprNodeType::MATCH_PRED) { - _column_predicate_info->query_op = "match"; - } else if (node_type == TExprNodeType::IN_PRED) { - if (expr->op() == TExprOpcode::type::FILTER_IN) { - _column_predicate_info->query_op = "in"; - } else { - _column_predicate_info->query_op = "not_in"; - } - } else if (node_type != TExprNodeType::COMPOUND_PRED) { - _column_predicate_info->query_op = expr->fn().name.function_name; - } - - if (!_column_predicate_info->is_empty()) { - _column_pred_in_remaining_vconjunct[_column_predicate_info->column_name].push_back( - *_column_predicate_info); - _column_predicate_info.reset(new ColumnPredicateInfo()); - } +Status SegmentIterator::_construct_compound_expr_context() { + auto inverted_index_context = std::make_shared( + _schema->column_ids(), _inverted_index_iterators, _storage_name_and_type, + _common_expr_inverted_index_status); + for (const auto& expr_ctx : _opts.common_expr_ctxs_push_down) { + vectorized::VExprContextSPtr context; + RETURN_IF_ERROR(expr_ctx->clone(_opts.runtime_state, context)); + context->set_inverted_index_context(inverted_index_context); + _common_expr_ctxs_push_down.emplace_back(context); } + return Status::OK(); } -void SegmentIterator::_calculate_func_in_remaining_conjunct_root() { - auto hash = [](const vectorized::VExprSPtr& expr) -> std::size_t { - return std::hash()(expr->expr_name()); - }; - auto equal = [](const vectorized::VExprSPtr& lhs, const vectorized::VExprSPtr& rhs) -> bool { - return lhs->equals(*rhs); - }; - - uint32_t next_id = 0; - std::unordered_map unique_map( - 0, hash, equal); - - auto gen_func_unique_id = [&unique_map, &next_id](const vectorized::VExprSPtr& expr) { - auto it = unique_map.find(expr); - if (it != unique_map.end()) { - return it->second; - } else { - unique_map[expr] = ++next_id; - return next_id; - } - }; - +void SegmentIterator::_calculate_expr_in_remaining_conjunct_root() { for (const auto& root_expr_ctx : _common_expr_ctxs_push_down) { const auto& root_expr = root_expr_ctx->root(); if (root_expr == nullptr) { continue; } - std::stack> stack; - stack.emplace(root_expr, false); + std::stack stack; + stack.emplace(root_expr); while (!stack.empty()) { - const auto& [expr, has_compound_pred] = stack.top(); + const auto& expr = stack.top(); stack.pop(); - bool current_has_compound_pred = - has_compound_pred || (expr->node_type() == TExprNodeType::COMPOUND_PRED); - - if (expr->fn().name.function_name == "multi_match") { - expr->set_index_unique_id(gen_func_unique_id(expr)); - if (current_has_compound_pred) { - compound_func_exprs.emplace_back(expr); - } else { - no_compound_func_exprs.emplace_back(expr); - } - - for (int32_t i = expr->get_num_children() - 1; i >= 0; i--) { - auto child_expr = expr->get_child(i); - if (child_expr->node_type() == TExprNodeType::type::SLOT_REF) { - std::string result_sign = BeConsts::BLOCK_TEMP_COLUMN_PREFIX + - std::to_string(expr->index_unique_id()); - _func_name_to_result_sign[child_expr->expr_name()].push_back(result_sign); - } + for (const auto& child : expr->children()) { + if (child->is_slot_ref()) { + auto* column_slot_ref = assert_cast(child.get()); + _common_expr_inverted_index_status[_schema->column_id( + column_slot_ref->column_id())][expr.get()] = false; } } const auto& children = expr->children(); for (int32_t i = children.size() - 1; i >= 0; --i) { if (!children[i]->children().empty()) { - stack.emplace(children[i], current_has_compound_pred); + stack.emplace(children[i]); } } } @@ -2935,7 +2501,7 @@ bool SegmentIterator::_no_need_read_key_data(ColumnId cid, vectorized::MutableCo return false; } - if (!_check_all_predicates_passed_inverted_index_for_column(cid)) { + if (!_check_all_conditions_passed_inverted_index_for_column(cid)) { return false; } @@ -2970,7 +2536,7 @@ bool SegmentIterator::_can_opt_topn_reads() { _opts.tablet_schema->column(cid).is_key()) { return true; } - if (_check_all_predicates_passed_inverted_index_for_column(cid, true)) { + if (_check_all_conditions_passed_inverted_index_for_column(cid, true)) { return true; } return false; @@ -2985,22 +2551,5 @@ bool SegmentIterator::_can_opt_topn_reads() { return all_true; } -Status SegmentIterator::execute_func_expr(const vectorized::VExprSPtr& expr, - std::shared_ptr& result) { - const auto& expr0 = expr->get_child(0); - if (!expr0 || expr0->node_type() != TExprNodeType::SLOT_REF) { - return Status::RuntimeError("cannot perform index filtering"); - } - - FuncExprParams params; - auto slot_expr = std::static_pointer_cast(expr0); - params._column_id = _schema->column_id(slot_expr->column_id()); - params._unique_id = _schema->unique_id(slot_expr->column_id()); - params._column_name = _opts.tablet_schema->column(params._column_id).name(); - params._segment_iterator = this; - - return expr->eval_inverted_index(params, result); -} - } // namespace segment_v2 } // namespace doris diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h index 44f71f23b0..7b42a3938d 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.h +++ b/be/src/olap/rowset/segment_v2/segment_iterator.h @@ -201,20 +201,10 @@ private: [[nodiscard]] Status _apply_inverted_index_on_column_predicate( ColumnPredicate* pred, std::vector& remaining_predicates, bool* continue_apply); - [[nodiscard]] Status _apply_inverted_index_on_block_column_predicate( - ColumnId column_id, MutilColumnBlockPredicate* pred, - std::set& no_need_to_pass_column_predicate_set, - bool* continue_apply); - [[nodiscard]] Status _apply_index_except_leafnode_of_andnode(); - [[nodiscard]] Status _apply_inverted_index_except_leafnode_of_andnode( - ColumnPredicate* pred, roaring::Roaring* output_result); + [[nodiscard]] Status _apply_index_expr(); bool _column_has_fulltext_index(int32_t cid); bool _downgrade_without_index(Status res, bool need_remaining = false); inline bool _inverted_index_not_support_pred_type(const PredicateType& type); - bool _can_filter_by_preds_except_leafnode_of_andnode(); - [[nodiscard]] Status _execute_predicates_except_leafnode_of_andnode( - const vectorized::VExprSPtr& expr); - [[nodiscard]] Status _execute_compound_fn(const std::string& function_name); bool _is_literal_node(const TExprNodeType::type& node_type); Status _vec_init_lazy_materialization(); @@ -298,6 +288,7 @@ private: bool _can_evaluated_by_vectorized(ColumnPredicate* predicate); [[nodiscard]] Status _extract_common_expr_columns(const vectorized::VExprSPtr& expr); + // same with _extract_common_expr_columns, but only extract columns that can be used for index [[nodiscard]] Status _execute_common_expr(uint16_t* sel_rowid_idx, uint16_t& selected_size, vectorized::Block* block); uint16_t _evaluate_common_expr_filter(uint16_t* sel_rowid_idx, uint16_t selected_size, @@ -310,24 +301,14 @@ private: bool _check_apply_by_inverted_index(ColumnPredicate* pred, bool pred_in_compound = false); - std::string _gen_predicate_result_sign(ColumnPredicate* predicate); - std::string _gen_predicate_result_sign(ColumnPredicateInfo* predicate_info); - - void _build_index_result_column(const uint16_t* sel_rowid_idx, uint16_t select_size, - vectorized::Block* block, const std::string& pred_result_sign, - const roaring::Roaring& index_result); - void _output_index_result_column(uint16_t* sel_rowid_idx, uint16_t select_size, - vectorized::Block* block); + void _output_index_result_column_for_expr(uint16_t* sel_rowid_idx, uint16_t select_size, + vectorized::Block* block); bool _need_read_data(ColumnId cid); bool _prune_column(ColumnId cid, vectorized::MutableColumnPtr& column, bool fill_defaults, size_t num_of_defaults); - // return true means one column's predicates all pushed down - bool _check_column_pred_all_push_down(const std::string& column_name, bool in_compound = false, - bool is_match = false); - void _calculate_pred_in_remaining_conjunct_root(const vectorized::VExprSPtr& expr); - void _calculate_func_in_remaining_conjunct_root(); + Status _construct_compound_expr_context(); // todo(wb) remove this method after RowCursor is removed void _convert_rowcursor_to_short_key(const RowCursor& key, size_t num_keys) { @@ -410,9 +391,10 @@ private: Status execute_func_expr(const vectorized::VExprSPtr& expr, std::shared_ptr& result); void _initialize_predicate_results(); - bool _check_all_predicates_passed_inverted_index_for_column(ColumnId cid, + bool _check_all_conditions_passed_inverted_index_for_column(ColumnId cid, bool default_return = false); + void _calculate_expr_in_remaining_conjunct_root(); class BitmapRangeIterator; class BackwardBitmapRangeIterator; @@ -427,8 +409,6 @@ private: std::vector> _inverted_index_iterators; // after init(), `_row_bitmap` contains all rowid to scan roaring::Roaring _row_bitmap; - // "column_name+operator+value-> - std::unordered_map> _rowid_result_for_index; // an iterator for `_row_bitmap` that can be used to extract row range to scan std::unique_ptr _range_iter; // the next rowid to read @@ -477,19 +457,9 @@ private: StorageReadOptions _opts; // make a copy of `_opts.column_predicates` in order to make local changes std::vector _col_predicates; - std::vector _col_preds_except_leafnode_of_andnode; - - std::vector no_compound_func_exprs; - std::vector compound_func_exprs; - vectorized::VExprContextSPtrs _common_expr_ctxs_push_down; bool _enable_common_expr_pushdown = false; std::vector _remaining_conjunct_roots; - std::vector _pred_except_leafnode_of_andnode_evaluate_result; - std::unique_ptr _column_predicate_info; - std::unordered_map> - _column_pred_in_remaining_vconjunct; - std::unordered_map> _func_name_to_result_sign; std::set _not_apply_index_pred; // row schema of the key to seek @@ -527,8 +497,11 @@ private: std::vector _ret_flags; - std::unordered_map> + std::unordered_map> _column_predicate_inverted_index_status; + + std::unordered_map> + _common_expr_inverted_index_status; }; } // namespace segment_v2 diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp index 8caf1e22cc..f1d34de2b9 100644 --- a/be/src/pipeline/exec/scan_operator.cpp +++ b/be/src/pipeline/exec/scan_operator.cpp @@ -348,9 +348,6 @@ Status ScanLocalState::_normalize_predicate( RETURN_IF_PUSH_DOWN(_normalize_noneq_binary_predicate( cur_expr, context, slot, value_range, &pdt), status); - RETURN_IF_PUSH_DOWN(_normalize_match_predicate(cur_expr, context, slot, - value_range, &pdt), - status); if (_is_key_column(slot->col_name())) { RETURN_IF_PUSH_DOWN( _normalize_bitmap_filter(cur_expr, context, slot, &pdt), @@ -368,23 +365,6 @@ Status ScanLocalState::_normalize_predicate( *range); RETURN_IF_ERROR(status); } - - if (pdt == vectorized::VScanNode::PushDownType::UNACCEPTABLE && - TExprNodeType::COMPOUND_PRED == cur_expr->node_type()) { - _normalize_compound_predicate(cur_expr, context, &pdt, _is_runtime_filter_predicate, - in_predicate_checker, eq_predicate_checker); - output_expr = conjunct_expr_root; // remaining in conjunct tree - return Status::OK(); - } - - if (pdt == vectorized::VScanNode::PushDownType::ACCEPTABLE && - TExprNodeType::MATCH_PRED == cur_expr->node_type()) { - // remaining it in the expr tree, in order to filter by function if the pushdown - // match_predicate failed to apply inverted index in the storage layer - output_expr = conjunct_expr_root; // remaining in conjunct tree - return Status::OK(); - } - if (pdt == vectorized::VScanNode::PushDownType::ACCEPTABLE && slotref != nullptr && slotref->type().is_variant_type()) { // remaining it in the expr tree, in order to filter by function if the pushdown diff --git a/be/src/vec/exprs/vcompound_pred.h b/be/src/vec/exprs/vcompound_pred.h index 2ede99cae6..faed6788ba 100644 --- a/be/src/vec/exprs/vcompound_pred.h +++ b/be/src/vec/exprs/vcompound_pred.h @@ -27,6 +27,7 @@ #include "vec/data_types/data_type_number.h" #include "vec/exprs/vectorized_fn_call.h" #include "vec/exprs/vexpr.h" +#include "vec/exprs/vexpr_context.h" namespace doris::vectorized { @@ -53,7 +54,107 @@ public: const std::string& expr_name() const override { return _expr_name; } + Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override { + segment_v2::InvertedIndexResultBitmap res; + bool all_pass = true; + + switch (_op) { + case TExprOpcode::COMPOUND_OR: { + for (const auto& child : _children) { + if (Status st = child->evaluate_inverted_index(context, segment_num_rows); + !st.ok()) { + LOG(ERROR) << "expr:" << child->expr_name() + << " evaluate_inverted_index error:" << st.to_string(); + all_pass = false; + continue; + } + if (context->get_inverted_index_context()->has_inverted_index_result_for_expr( + child.get())) { + const auto* index_result = + context->get_inverted_index_context() + ->get_inverted_index_result_for_expr(child.get()); + if (res.is_empty()) { + res = *index_result; + } else { + res |= *index_result; + } + if (res.get_data_bitmap()->cardinality() == segment_num_rows) { + break; // Early exit if result is full + } + } else { + all_pass = false; + } + } + break; + } + case TExprOpcode::COMPOUND_AND: { + for (const auto& child : _children) { + if (Status st = child->evaluate_inverted_index(context, segment_num_rows); + !st.ok()) { + LOG(ERROR) << "expr:" << child->expr_name() + << " evaluate_inverted_index error:" << st.to_string(); + all_pass = false; + continue; + } + if (context->get_inverted_index_context()->has_inverted_index_result_for_expr( + child.get())) { + const auto* index_result = + context->get_inverted_index_context() + ->get_inverted_index_result_for_expr(child.get()); + if (res.is_empty()) { + res = *index_result; + } else { + res &= *index_result; + } + + if (res.get_data_bitmap()->isEmpty()) { + break; // Early exit if result is empty + } + } else { + all_pass = false; + } + } + break; + } + case TExprOpcode::COMPOUND_NOT: { + const auto& child = _children[0]; + Status st = child->evaluate_inverted_index(context, segment_num_rows); + if (!st.ok()) { + LOG(ERROR) << "expr:" << child->expr_name() + << " evaluate_inverted_index error:" << st.to_string(); + return st; + } + + if (context->get_inverted_index_context()->has_inverted_index_result_for_expr( + child.get())) { + const auto* index_result = + context->get_inverted_index_context()->get_inverted_index_result_for_expr( + child.get()); + roaring::Roaring full_result; + full_result.addRange(0, segment_num_rows); + res = index_result->op_not(&full_result); + } else { + all_pass = false; + } + break; + } + default: + return Status::NotSupported( + "Compound operator must be AND, OR, or NOT to execute with inverted index."); + } + + if (all_pass && !res.is_empty()) { + // set fast_execute when expr evaluated by inverted index correctly + _can_fast_execute = true; + context->get_inverted_index_context()->set_inverted_index_result_for_expr(this, res); + } + return Status::OK(); + } + Status execute(VExprContext* context, Block* block, int* result_column_id) override { + if (_can_fast_execute && fast_execute(context, block, result_column_id)) { + return Status::OK(); + } if (children().size() == 1 || !_all_child_is_compound_and_not_const()) { return VectorizedFnCall::execute(context, block, result_column_id); } @@ -249,8 +350,8 @@ private: } std::pair _get_raw_data_and_null_map(ColumnPtr column, - bool nullable_column) const { - if (nullable_column) { + bool has_nullable_column) const { + if (has_nullable_column) { auto* nullable_column = assert_cast(column->assume_mutable().get()); auto* data_column = assert_cast(nullable_column->get_nested_column_ptr().get()) diff --git a/be/src/vec/exprs/vectorized_fn_call.cpp b/be/src/vec/exprs/vectorized_fn_call.cpp index 6df4e44370..c3e0f2ea46 100644 --- a/be/src/vec/exprs/vectorized_fn_call.cpp +++ b/be/src/vec/exprs/vectorized_fn_call.cpp @@ -22,7 +22,6 @@ #include #include -#include #include #include "common/config.h" @@ -30,11 +29,8 @@ #include "common/status.h" #include "runtime/runtime_state.h" #include "udf/udf.h" -#include "vec/aggregate_functions/aggregate_function_simple_factory.h" #include "vec/columns/column.h" #include "vec/core/block.h" -#include "vec/core/column_with_type_and_name.h" -#include "vec/core/columns_with_type_and_name.h" #include "vec/data_types/data_type.h" #include "vec/data_types/data_type_agg_state.h" #include "vec/exprs/vexpr_context.h" @@ -111,7 +107,6 @@ Status VectorizedFnCall::prepare(RuntimeState* state, const RowDescriptor& desc, } VExpr::register_function_context(state, context); _function_name = _fn.name.function_name; - _can_fast_execute = can_fast_execute(); _prepare_finished = true; return Status::OK(); } @@ -135,13 +130,38 @@ void VectorizedFnCall::close(VExprContext* context, FunctionContext::FunctionSta VExpr::close(context, scope); } +Status VectorizedFnCall::evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) { + DCHECK_GE(get_num_children(), 1); + return _evaluate_inverted_index(context, _function, segment_num_rows); +} + Status VectorizedFnCall::_do_execute(doris::vectorized::VExprContext* context, doris::vectorized::Block* block, int* result_column_id, std::vector& args) { - if (is_const_and_have_executed()) { // const have execute in open function + if (is_const_and_have_executed()) { // const have executed in open function return get_result_from_const(block, _expr_name, result_column_id); } + if (_can_fast_execute && fast_execute(context, block, result_column_id)) { + return Status::OK(); + } + DBUG_EXECUTE_IF("VectorizedFnCall.must_in_slow_path", { + if (get_child(0)->is_slot_ref()) { + auto debug_col_name = DebugPoints::instance()->get_debug_param_or_default( + "VectorizedFnCall.must_in_slow_path", "column_name", ""); + std::vector column_names; + boost::split(column_names, debug_col_name, boost::algorithm::is_any_of(",")); + + auto* column_slot_ref = assert_cast(get_child(0).get()); + std::string column_name = column_slot_ref->expr_name(); + auto it = std::find(column_names.begin(), column_names.end(), column_name); + if (it == column_names.end()) { + return Status::Error( + "column {} should in slow path while VectorizedFnCall::execute.", + column_name); + } + } + }) DCHECK(_open_finished || _getting_const_col) << debug_string(); // TODO: not execute const expr again, but use the const column in function context args.resize(_children.size()); @@ -156,14 +176,6 @@ Status VectorizedFnCall::_do_execute(doris::vectorized::VExprContext* context, size_t num_columns_without_result = block->columns(); // prepare a column to save result block->insert({nullptr, _data_type, _expr_name}); - if (_can_fast_execute) { - auto can_fast_execute = fast_execute(*block, args, num_columns_without_result, - block->rows(), _function->get_name()); - if (can_fast_execute) { - *result_column_id = num_columns_without_result; - return Status::OK(); - } - } RETURN_IF_ERROR(_function->execute(context->fn_context(_fn_context_index), *block, args, num_columns_without_result, block->rows(), false)); *result_column_id = num_columns_without_result; @@ -218,22 +230,6 @@ bool VectorizedFnCall::can_push_down_to_index() const { return _function->can_push_down_to_index(); } -bool VectorizedFnCall::can_fast_execute() const { - auto function_name = _function->get_name(); - if (function_name == "eq" || function_name == "ne" || function_name == "lt" || - function_name == "gt" || function_name == "le" || function_name == "ge") { - if (_children.size() == 2 && _children[0]->is_slot_ref() && _children[1]->is_literal()) { - return true; - } - } - return _function->can_push_down_to_index(); -} - -Status VectorizedFnCall::eval_inverted_index(segment_v2::FuncExprParams& params, - std::shared_ptr& result) { - return _function->eval_inverted_index(this, params, result); -} - bool VectorizedFnCall::equals(const VExpr& other) { const auto* other_ptr = dynamic_cast(&other); if (!other_ptr) { diff --git a/be/src/vec/exprs/vectorized_fn_call.h b/be/src/vec/exprs/vectorized_fn_call.h index dea5e90f9b..bae996136d 100644 --- a/be/src/vec/exprs/vectorized_fn_call.h +++ b/be/src/vec/exprs/vectorized_fn_call.h @@ -27,6 +27,8 @@ #include "udf/udf.h" #include "vec/core/column_numbers.h" #include "vec/exprs/vexpr.h" +#include "vec/exprs/vliteral.h" +#include "vec/exprs/vslot_ref.h" #include "vec/functions/function.h" namespace doris { @@ -50,6 +52,7 @@ public: Status execute_runtime_fitler(doris::vectorized::VExprContext* context, doris::vectorized::Block* block, int* result_column_id, std::vector& args) override; + Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override; Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; Status open(RuntimeState* state, VExprContext* context, FunctionContext::FunctionStateScope scope) override; @@ -67,9 +70,6 @@ public: static std::string debug_string(const std::vector& exprs); bool can_push_down_to_index() const override; - bool can_fast_execute() const override; - Status eval_inverted_index(segment_v2::FuncExprParams& params, - std::shared_ptr& result) override; bool equals(const VExpr& other) override; protected: diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp index 5cb0607411..4617d31664 100644 --- a/be/src/vec/exprs/vexpr.cpp +++ b/be/src/vec/exprs/vexpr.cpp @@ -33,6 +33,7 @@ #include "common/status.h" #include "vec/columns/column_vector.h" #include "vec/columns/columns_number.h" +#include "vec/data_types/data_type_array.h" #include "vec/data_types/data_type_factory.hpp" #include "vec/data_types/data_type_nullable.h" #include "vec/data_types/data_type_number.h" @@ -43,7 +44,6 @@ #include "vec/exprs/vcompound_pred.h" #include "vec/exprs/vectorized_fn_call.h" #include "vec/exprs/vexpr_context.h" -#include "vec/exprs/vexpr_fwd.h" #include "vec/exprs/vin_predicate.h" #include "vec/exprs/vinfo_func.h" #include "vec/exprs/vlambda_function_call_expr.h" @@ -602,80 +602,134 @@ Status VExpr::get_result_from_const(vectorized::Block* block, const std::string& return Status::OK(); } -bool VExpr::fast_execute(Block& block, const ColumnNumbers& arguments, size_t result, - size_t input_rows_count, const std::string& function_name) { - if (!_enable_inverted_index_query) { - return false; - } - - std::string result_column_name = gen_predicate_result_sign(block, arguments, function_name); - if (!block.has(result_column_name)) { - DBUG_EXECUTE_IF("segment_iterator.fast_execute", { - auto debug_col_name = DebugPoints::instance()->get_debug_param_or_default( - "segment_iterator._read_columns_by_index", "column_name", ""); - - std::vector column_names; - boost::split(column_names, debug_col_name, boost::algorithm::is_any_of(",")); - - std::string column_name = block.get_by_position(arguments[0]).name; - auto it = std::find(column_names.begin(), column_names.end(), column_name); - if (it == column_names.end()) { - return Status::Error("fast_execute failed: {}", - result_column_name); - } - }) - return false; - } - - auto result_column = - block.get_by_name(result_column_name).column->convert_to_full_column_if_const(); - auto& result_info = block.get_by_position(result); - if (result_info.type->is_nullable()) { - block.replace_by_position(result, - ColumnNullable::create(std::move(result_column), - ColumnUInt8::create(input_rows_count, 0))); - } else { - block.replace_by_position(result, std::move(result_column)); - } - - return true; -} - -std::string VExpr::gen_predicate_result_sign(Block& block, const ColumnNumbers& arguments, - const std::string& function_name) const { - std::string pred_result_sign; - if (this->fn().name.function_name == "multi_match") { - pred_result_sign = - BeConsts::BLOCK_TEMP_COLUMN_PREFIX + std::to_string(this->index_unique_id()); - } else { - std::string column_name = block.get_by_position(arguments[0]).name; - pred_result_sign += - BeConsts::BLOCK_TEMP_COLUMN_PREFIX + column_name + "_" + function_name + "_"; - if (function_name == "in" || function_name == "not_in") { - if (arguments.size() - 1 > _in_list_value_count_threshold) { - return pred_result_sign; - } - // Generating 'result_sign' from 'inlist' requires sorting the values. - std::set values; - for (size_t i = 1; i < arguments.size(); i++) { - const auto& entry = block.get_by_position(arguments[i]); - if (!is_column_const(*entry.column)) { - return pred_result_sign; +Status VExpr::_evaluate_inverted_index(VExprContext* context, const FunctionBasePtr& function, + uint32_t segment_num_rows) { + std::vector iterators; + std::vector data_type_with_names; + std::vector column_ids; + vectorized::ColumnsWithTypeAndName arguments; + VExprSPtrs children_exprs; + for (auto child : children()) { + // if child is cast expr, we need to ensure target data type is the same with storage data type. + // or they are all string type + // and if data type is array, we need to get the nested data type to ensure that. + if (child->node_type() == TExprNodeType::CAST_EXPR) { + auto* cast_expr = assert_cast(child.get()); + DCHECK_EQ(cast_expr->children().size(), 1); + if (cast_expr->get_child(0)->is_slot_ref()) { + auto* column_slot_ref = assert_cast(cast_expr->get_child(0).get()); + auto column_id = column_slot_ref->column_id(); + const auto* storage_name_type = + context->get_inverted_index_context() + ->get_storage_name_and_type_by_column_id(column_id); + auto storage_type = remove_nullable(storage_name_type->second); + auto target_type = cast_expr->get_target_type(); + auto origin_primitive_type = storage_type->get_type_as_type_descriptor().type; + auto target_primitive_type = target_type->get_type_as_type_descriptor().type; + if (is_complex_type(storage_type)) { + if (is_array(storage_type) && is_array(target_type)) { + auto nested_storage_type = + (assert_cast(storage_type.get())) + ->get_nested_type(); + origin_primitive_type = + nested_storage_type->get_type_as_type_descriptor().type; + auto nested_target_type = + (assert_cast(target_type.get())) + ->get_nested_type(); + target_primitive_type = + nested_target_type->get_type_as_type_descriptor().type; + } else { + continue; + } + } + if (origin_primitive_type != TYPE_VARIANT && + (origin_primitive_type == target_primitive_type || + (is_string_type(target_primitive_type) && + is_string_type(origin_primitive_type)))) { + children_exprs.emplace_back(expr_without_cast(child)); } - values.insert(entry.type->to_string(*entry.column, 0)); } - pred_result_sign += boost::join(values, ","); - } else if (function_name == "collection_in" || function_name == "collection_not_in") { - return pred_result_sign; } else { - const auto& entry = block.get_by_position(arguments[1]); - if (!is_column_const(*entry.column)) { - return pred_result_sign; - } - pred_result_sign += entry.type->to_string(*entry.column, 0); + children_exprs.emplace_back(child); } } - return pred_result_sign; + + for (auto child : children_exprs) { + if (child->is_slot_ref()) { + auto* column_slot_ref = assert_cast(child.get()); + auto column_id = column_slot_ref->column_id(); + auto* iter = + context->get_inverted_index_context()->get_inverted_index_iterator_by_column_id( + column_id); + //column does not have inverted index + if (iter == nullptr) { + continue; + } + const auto* storage_name_type = + context->get_inverted_index_context()->get_storage_name_and_type_by_column_id( + column_id); + if (storage_name_type == nullptr) { + auto err_msg = fmt::format( + "storage_name_type cannot be found for column {} while in {} " + "evaluate_inverted_index", + column_id, expr_name()); + LOG(ERROR) << err_msg; + return Status::InternalError(err_msg); + } + iterators.emplace_back(iter); + data_type_with_names.emplace_back(*storage_name_type); + column_ids.emplace_back(column_id); + } else if (child->is_literal()) { + auto* column_literal = assert_cast(child.get()); + arguments.emplace_back(column_literal->get_column_ptr(), + column_literal->get_data_type(), column_literal->expr_name()); + } + } + auto result_bitmap = segment_v2::InvertedIndexResultBitmap(); + if (iterators.empty()) { + return Status::OK(); + } + // If arguments are empty, it means the left value in the expression is not a literal. + if (arguments.empty()) { + return Status::OK(); + } + auto res = function->evaluate_inverted_index(arguments, data_type_with_names, iterators, + segment_num_rows, result_bitmap); + if (!res.ok()) { + return res; + } + if (!result_bitmap.is_empty()) { + context->get_inverted_index_context()->set_inverted_index_result_for_expr(this, + result_bitmap); + for (auto column_id : column_ids) { + context->get_inverted_index_context()->set_true_for_inverted_index_status(this, + column_id); + } + // set fast_execute when expr evaluated by inverted index correctly + _can_fast_execute = true; + } + return Status::OK(); +} + +bool VExpr::fast_execute(doris::vectorized::VExprContext* context, doris::vectorized::Block* block, + int* result_column_id) { + if (context->get_inverted_index_context() && + context->get_inverted_index_context()->get_inverted_index_result_column().contains(this)) { + size_t num_columns_without_result = block->columns(); + // prepare a column to save result + auto result_column = + context->get_inverted_index_context()->get_inverted_index_result_column()[this]; + if (_data_type->is_nullable()) { + block->insert( + {ColumnNullable::create(result_column, ColumnUInt8::create(block->rows(), 0)), + _data_type, expr_name()}); + } else { + block->insert({result_column, _data_type, expr_name()}); + } + *result_column_id = num_columns_without_result; + return true; + } + return false; } bool VExpr::equals(const VExpr& other) { diff --git a/be/src/vec/exprs/vexpr.h b/be/src/vec/exprs/vexpr.h index 88b18c6787..3bc92850fe 100644 --- a/be/src/vec/exprs/vexpr.h +++ b/be/src/vec/exprs/vexpr.h @@ -115,6 +115,14 @@ public: virtual Status execute(VExprContext* context, Block* block, int* result_column_id) = 0; + // execute current expr with inverted index to filter block. Given a roaring bitmap of match rows + virtual Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) { + return Status::NotSupported("Not supported execute_with_inverted_index"); + } + + Status _evaluate_inverted_index(VExprContext* context, const FunctionBasePtr& function, + uint32_t segment_num_rows); + // Only the 4th parameter is used in the runtime filter. In and MinMax need overwrite the // interface virtual Status execute_runtime_fitler(VExprContext* context, Block* block, @@ -222,18 +230,10 @@ public: } // fast_execute can direct copy expr filter result which build by apply index in segment_iterator - bool fast_execute(Block& block, const ColumnNumbers& arguments, size_t result, - size_t input_rows_count, const std::string& function_name); - - std::string gen_predicate_result_sign(Block& block, const ColumnNumbers& arguments, - const std::string& function_name) const; + bool fast_execute(doris::vectorized::VExprContext* context, doris::vectorized::Block* block, + int* result_column_id); virtual bool can_push_down_to_index() const { return false; } - virtual bool can_fast_execute() const { return false; } - virtual Status eval_inverted_index(segment_v2::FuncExprParams& params, - std::shared_ptr& result) { - return Status::NotSupported("Not supported execute_with_inverted_index"); - } virtual bool equals(const VExpr& other); void set_index_unique_id(uint32_t index_unique_id) { _index_unique_id = index_unique_id; } uint32_t index_unique_id() const { return _index_unique_id; } diff --git a/be/src/vec/exprs/vexpr_context.cpp b/be/src/vec/exprs/vexpr_context.cpp index cdbf22a720..b493353341 100644 --- a/be/src/vec/exprs/vexpr_context.cpp +++ b/be/src/vec/exprs/vexpr_context.cpp @@ -120,6 +120,16 @@ int VExprContext::register_function_context(RuntimeState* state, const TypeDescr return _fn_contexts.size() - 1; } +Status VExprContext::evaluate_inverted_index(uint32_t segment_num_rows) { + Status st; + RETURN_IF_CATCH_EXCEPTION({ st = _root->evaluate_inverted_index(this, segment_num_rows); }); + return st; +} + +bool VExprContext::all_expr_inverted_index_evaluated() { + return _inverted_index_context->has_inverted_index_result_for_expr(_root.get()); +} + Status VExprContext::filter_block(VExprContext* vexpr_ctx, Block* block, int column_to_keep) { if (vexpr_ctx == nullptr || block->rows() == 0) { return Status::OK(); diff --git a/be/src/vec/exprs/vexpr_context.h b/be/src/vec/exprs/vexpr_context.h index 423e1aac12..4864260909 100644 --- a/be/src/vec/exprs/vexpr_context.h +++ b/be/src/vec/exprs/vexpr_context.h @@ -25,6 +25,7 @@ #include "common/factory_creator.h" #include "common/status.h" +#include "olap/rowset/segment_v2/inverted_index_reader.h" #include "runtime/types.h" #include "udf/udf.h" #include "vec/core/block.h" @@ -37,6 +38,114 @@ class RuntimeState; namespace doris::vectorized { +class InvertedIndexContext { +public: + InvertedIndexContext( + const std::vector& col_ids, + const std::vector>& + inverted_index_iterators, + const std::vector& storage_name_and_type_vec, + std::unordered_map>& + common_expr_inverted_index_status) + : _col_ids(col_ids), + _inverted_index_iterators(inverted_index_iterators), + _storage_name_and_type(storage_name_and_type_vec), + _expr_inverted_index_status(common_expr_inverted_index_status) {} + + segment_v2::InvertedIndexIterator* get_inverted_index_iterator_by_column_id( + int column_index) const { + if (column_index < 0 || column_index >= _col_ids.size()) { + return nullptr; + } + const auto& column_id = _col_ids[column_index]; + if (column_id >= _inverted_index_iterators.size()) { + return nullptr; + } + if (!_inverted_index_iterators[column_id]) { + return nullptr; + } + return _inverted_index_iterators[column_id].get(); + } + + const vectorized::IndexFieldNameAndTypePair* get_storage_name_and_type_by_column_id( + int column_index) const { + if (column_index < 0 || column_index >= _col_ids.size()) { + return nullptr; + } + const auto& column_id = _col_ids[column_index]; + if (column_id >= _storage_name_and_type.size()) { + return nullptr; + } + return &_storage_name_and_type[column_id]; + } + + bool has_inverted_index_result_for_expr(const vectorized::VExpr* expr) const { + return _inverted_index_result_bitmap.contains(expr); + } + + void set_inverted_index_result_for_expr(const vectorized::VExpr* expr, + segment_v2::InvertedIndexResultBitmap bitmap) { + _inverted_index_result_bitmap[expr] = std::move(bitmap); + } + + std::unordered_map& + get_inverted_index_result_bitmap() { + return _inverted_index_result_bitmap; + } + + std::unordered_map& get_inverted_index_result_column() { + return _inverted_index_result_column; + } + + const segment_v2::InvertedIndexResultBitmap* get_inverted_index_result_for_expr( + const vectorized::VExpr* expr) { + auto iter = _inverted_index_result_bitmap.find(expr); + if (iter == _inverted_index_result_bitmap.end()) { + return nullptr; + } + return &iter->second; + } + + void set_inverted_index_result_column_for_expr(const vectorized::VExpr* expr, + ColumnPtr column) { + _inverted_index_result_column[expr] = std::move(column); + } + + void set_true_for_inverted_index_status(const vectorized::VExpr* expr, int column_index) { + if (column_index < 0 || column_index >= _col_ids.size()) { + return; + } + const auto& column_id = _col_ids[column_index]; + if (_expr_inverted_index_status.contains(column_id)) { + if (_expr_inverted_index_status[column_id].contains(expr)) { + _expr_inverted_index_status[column_id][expr] = true; + } + } + } + +private: + // A reference to a vector of column IDs for the current expression's output columns. + const std::vector& _col_ids; + + // A reference to a vector of unique pointers to inverted index iterators. + const std::vector>& + _inverted_index_iterators; + + // A reference to a vector of storage name and type pairs related to schema. + const std::vector& _storage_name_and_type; + + // A map of expressions to their corresponding inverted index result bitmaps. + std::unordered_map + _inverted_index_result_bitmap; + + // A map of expressions to their corresponding result columns. + std::unordered_map _inverted_index_result_column; + + // A reference to a map of common expressions to their inverted index evaluation status. + std::unordered_map>& + _expr_inverted_index_status; +}; + class VExprContext { ENABLE_FACTORY_CREATOR(VExprContext); @@ -50,6 +159,13 @@ public: VExprSPtr root() { return _root; } void set_root(const VExprSPtr& expr) { _root = expr; } + void set_inverted_index_context(std::shared_ptr inverted_index_context) { + _inverted_index_context = std::move(inverted_index_context); + } + + std::shared_ptr get_inverted_index_context() const { + return _inverted_index_context; + } /// Creates a FunctionContext, and returns the index that's passed to fn_context() to /// retrieve the created context. Exprs that need a FunctionContext should call this in @@ -69,6 +185,14 @@ public: return _fn_contexts[i].get(); } + // execute expr with inverted index which column a, b has inverted indexes + // but some situation although column b has indexes, but apply index is not useful, we should + // skip this expr, just do not apply index anymore. + + [[nodiscard]] Status evaluate_inverted_index(uint32_t segment_num_rows); + + bool all_expr_inverted_index_evaluated(); + [[nodiscard]] static Status filter_block(VExprContext* vexpr_ctx, Block* block, int column_to_keep); @@ -175,5 +299,7 @@ private: // This flag only works on VSlotRef. // Force to materialize even if the slot need_materialize is false, we just ignore need_materialize flag bool _force_materialize_slot = false; + + std::shared_ptr _inverted_index_context; }; } // namespace doris::vectorized diff --git a/be/src/vec/exprs/vin_predicate.cpp b/be/src/vec/exprs/vin_predicate.cpp index 4d518f9f92..361f55f038 100644 --- a/be/src/vec/exprs/vin_predicate.cpp +++ b/be/src/vec/exprs/vin_predicate.cpp @@ -34,6 +34,8 @@ #include "vec/core/column_with_type_and_name.h" #include "vec/core/columns_with_type_and_name.h" #include "vec/exprs/vexpr_context.h" +#include "vec/exprs/vliteral.h" +#include "vec/exprs/vslot_ref.h" #include "vec/functions/simple_function_factory.h" namespace doris { @@ -79,16 +81,14 @@ Status VInPredicate::prepare(RuntimeState* state, const RowDescriptor& desc, VExpr::register_function_context(state, context); _prepare_finished = true; - _can_fast_execute = can_fast_execute(); - _in_list_value_count_threshold = state->query_options().in_list_value_count_threshold; return Status::OK(); } Status VInPredicate::open(RuntimeState* state, VExprContext* context, FunctionContext::FunctionStateScope scope) { DCHECK(_prepare_finished); - for (int i = 0; i < _children.size(); ++i) { - RETURN_IF_ERROR(_children[i]->open(state, context, scope)); + for (auto& child : _children) { + RETURN_IF_ERROR(child->open(state, context, scope)); } RETURN_IF_ERROR(VExpr::init_function_context(context, scope, _function)); if (scope == FunctionContext::FRAGMENT_LOCAL) { @@ -103,10 +103,18 @@ void VInPredicate::close(VExprContext* context, FunctionContext::FunctionStateSc VExpr::close(context, scope); } +Status VInPredicate::evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) { + DCHECK_GE(get_num_children(), 2); + return _evaluate_inverted_index(context, _function, segment_num_rows); +} + Status VInPredicate::execute(VExprContext* context, Block* block, int* result_column_id) { if (is_const_and_have_executed()) { // const have execute in open function return get_result_from_const(block, _expr_name, result_column_id); } + if (_can_fast_execute && fast_execute(context, block, result_column_id)) { + return Status::OK(); + } DCHECK(_open_finished || _getting_const_col); // TODO: not execute const expr again, but use the const column in function context doris::vectorized::ColumnNumbers arguments(_children.size()); @@ -120,15 +128,6 @@ Status VInPredicate::execute(VExprContext* context, Block* block, int* result_co // prepare a column to save result block->insert({nullptr, _data_type, _expr_name}); - if (_can_fast_execute) { - auto can_fast_execute = fast_execute(*block, arguments, num_columns_without_result, - block->rows(), _function->get_name()); - if (can_fast_execute) { - *result_column_id = num_columns_without_result; - return Status::OK(); - } - } - RETURN_IF_ERROR(_function->execute(context->fn_context(_fn_context_index), *block, arguments, num_columns_without_result, block->rows(), false)); *result_column_id = num_columns_without_result; @@ -152,4 +151,4 @@ std::string VInPredicate::debug_string() const { return out.str(); } -} // namespace doris::vectorized \ No newline at end of file +} // namespace doris::vectorized diff --git a/be/src/vec/exprs/vin_predicate.h b/be/src/vec/exprs/vin_predicate.h index 5f197f996e..4d227510b9 100644 --- a/be/src/vec/exprs/vin_predicate.h +++ b/be/src/vec/exprs/vin_predicate.h @@ -54,7 +54,7 @@ public: const FunctionBasePtr function() { return _function; } bool is_not_in() const { return _is_not_in; }; - bool can_fast_execute() const override { return true; } + Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override; private: FunctionBasePtr _function; diff --git a/be/src/vec/exprs/vliteral.h b/be/src/vec/exprs/vliteral.h index b7a48145a8..131c717c95 100644 --- a/be/src/vec/exprs/vliteral.h +++ b/be/src/vec/exprs/vliteral.h @@ -54,6 +54,7 @@ public: std::string value() const; const ColumnPtr& get_column_ptr() const { return _column_ptr; } + const DataTypePtr& get_data_type() const { return _data_type; } bool is_literal() const override { return true; } diff --git a/be/src/vec/exprs/vmatch_predicate.cpp b/be/src/vec/exprs/vmatch_predicate.cpp index a9869bdc4e..9dea7ca870 100644 --- a/be/src/vec/exprs/vmatch_predicate.cpp +++ b/be/src/vec/exprs/vmatch_predicate.cpp @@ -43,6 +43,8 @@ #include "vec/core/column_with_type_and_name.h" #include "vec/core/columns_with_type_and_name.h" #include "vec/exprs/vexpr_context.h" +#include "vec/exprs/vliteral.h" +#include "vec/exprs/vslot_ref.h" #include "vec/functions/simple_function_factory.h" namespace doris { @@ -130,9 +132,35 @@ void VMatchPredicate::close(VExprContext* context, FunctionContext::FunctionStat VExpr::close(context, scope); } +Status VMatchPredicate::evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) { + DCHECK_EQ(get_num_children(), 2); + return _evaluate_inverted_index(context, _function, segment_num_rows); +} + Status VMatchPredicate::execute(VExprContext* context, Block* block, int* result_column_id) { DCHECK(_open_finished || _getting_const_col); - // TODO: not execute const expr again, but use the const column in function context + if (_can_fast_execute && fast_execute(context, block, result_column_id)) { + return Status::OK(); + } + DBUG_EXECUTE_IF("VMatchPredicate.execute", { + return Status::Error( + "{} not support slow path, hit debug point.", _expr_name); + }); + DBUG_EXECUTE_IF("VMatchPredicate.must_in_slow_path", { + auto debug_col_name = DebugPoints::instance()->get_debug_param_or_default( + "VMatchPredicate.must_in_slow_path", "column_name", ""); + + std::vector column_names; + boost::split(column_names, debug_col_name, boost::algorithm::is_any_of(",")); + + auto* column_slot_ref = assert_cast(get_child(0).get()); + std::string column_name = column_slot_ref->expr_name(); + auto it = std::find(column_names.begin(), column_names.end(), column_name); + if (it == column_names.end()) { + return Status::Error( + "column {} should in slow path while VMatchPredicate::execute.", column_name); + } + }) doris::vectorized::ColumnNumbers arguments(_children.size()); for (int i = 0; i < _children.size(); ++i) { int column_id = -1; diff --git a/be/src/vec/exprs/vmatch_predicate.h b/be/src/vec/exprs/vmatch_predicate.h index af364b4fd5..aaa28658e8 100644 --- a/be/src/vec/exprs/vmatch_predicate.h +++ b/be/src/vec/exprs/vmatch_predicate.h @@ -54,12 +54,13 @@ public: Status open(RuntimeState* state, VExprContext* context, FunctionContext::FunctionStateScope scope) override; void close(VExprContext* context, FunctionContext::FunctionStateScope scope) override; + Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override; const std::string& expr_name() const override; const std::string& function_name() const; std::string debug_string() const override; - const FunctionBasePtr function() { return _function; } + FunctionBasePtr function() { return _function; } private: FunctionBasePtr _function; diff --git a/be/src/vec/functions/array/function_array_index.h b/be/src/vec/functions/array/function_array_index.h index d4a8fa3296..24c5edcdce 100644 --- a/be/src/vec/functions/array/function_array_index.h +++ b/be/src/vec/functions/array/function_array_index.h @@ -25,6 +25,7 @@ #include #include "common/status.h" +#include "olap/rowset/segment_v2/inverted_index_reader.h" #include "vec/columns/column.h" #include "vec/columns/column_array.h" #include "vec/columns/column_nullable.h" @@ -87,6 +88,91 @@ public: bool use_default_implementation_for_nulls() const override { return false; } + Status evaluate_inverted_index( + const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const override { + DCHECK(arguments.size() == 1); + DCHECK(data_type_with_names.size() == 1); + DCHECK(iterators.size() == 1); + auto* iter = iterators[0]; + auto data_type_with_name = data_type_with_names[0]; + if (iter == nullptr) { + return Status::OK(); + } + if (iter->get_inverted_index_reader_type() == + segment_v2::InvertedIndexReaderType::FULLTEXT) { + // parser is not none we can not make sure the result is correct in expr combination + // for example, filter: !array_index(array, 'tall:120cm, weight: 35kg') + // here we have rows [tall:120cm, weight: 35kg, hobbies: reading book] which be tokenized + // but query is also tokenized, and FULLTEXT reader will catch this row as matched, + // so array_index(array, 'tall:120cm, weight: 35kg') return this rowid, + // but we expect it to be filtered, because we want row is equal to 'tall:120cm, weight: 35kg' + return Status::OK(); + } + Field param_value; + arguments[0].column->get(0, param_value); + auto param_type = arguments[0].type->get_type_as_type_descriptor().type; + // The current implementation for the inverted index of arrays cannot handle cases where the array contains null values, + // meaning an item in the array is null. + if (param_value.is_null()) { + return Status::OK(); + } + + std::shared_ptr roaring = std::make_shared(); + std::shared_ptr null_bitmap = std::make_shared(); + if (iter->has_null()) { + segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + RETURN_IF_ERROR(iter->read_null_bitmap(&null_bitmap_cache_handle)); + null_bitmap = null_bitmap_cache_handle.get_bitmap(); + } + std::unique_ptr query_param = nullptr; + RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value( + param_type, ¶m_value, query_param)); + if (is_string_type(param_type)) { + Status st = iter->read_from_inverted_index( + data_type_with_name.first, query_param->get_value(), + segment_v2::InvertedIndexQueryType::EQUAL_QUERY, num_rows, roaring); + if (st.code() == ErrorCode::INVERTED_INDEX_NO_TERMS) { + // if analyzed param with no term, we do not filter any rows + // return all rows with OK status + roaring->addRange(0, num_rows); + } else if (st != Status::OK()) { + return st; + } + } else { + RETURN_IF_ERROR(iter->read_from_inverted_index( + data_type_with_name.first, query_param->get_value(), + segment_v2::InvertedIndexQueryType::EQUAL_QUERY, num_rows, roaring)); + } + // here debug for check array_contains function really filter rows by inverted index correctly + DBUG_EXECUTE_IF("array_func.array_contains", { + auto result_bitmap = DebugPoints::instance()->get_debug_param_or_default( + "array_func.array_contains", "result_bitmap", 0); + if (result_bitmap < 0) { + return Status::Error( + "result_bitmap count cannot be negative"); + } + if (roaring->cardinality() != result_bitmap) { + return Status::Error( + "array_contains really filtered {} by inverted index not equal to expected " + "{}", + roaring->cardinality(), result_bitmap); + } + }) + if (iter->has_null()) { + segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + RETURN_IF_ERROR(iter->read_null_bitmap(&null_bitmap_cache_handle)); + null_bitmap = null_bitmap_cache_handle.get_bitmap(); + } + segment_v2::InvertedIndexResultBitmap result(roaring, null_bitmap); + bitmap_result = result; + bitmap_result.mask_out_null(); + + return Status::OK(); + } + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { if constexpr (OldVersion) { return make_nullable(std::make_shared>()); diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h index 780eb16064..c479815e96 100644 --- a/be/src/vec/functions/function.h +++ b/be/src/vec/functions/function.h @@ -185,6 +185,15 @@ public: ->execute(context, block, arguments, result, input_rows_count, dry_run); } + virtual Status evaluate_inverted_index( + const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const { + return Status::NotSupported("evaluate_inverted_index is not supported in function: ", + get_name()); + } + /// Do cleaning work when function is finished, i.e., release state variables in the /// `FunctionContext` which are registered in `prepare` phase. virtual Status close(FunctionContext* context, FunctionContext::FunctionStateScope scope) { @@ -220,12 +229,6 @@ public: } virtual bool can_push_down_to_index() const { return false; } - - virtual Status eval_inverted_index(VExpr* context, segment_v2::FuncExprParams& params, - std::shared_ptr& result) { - return Status::NotSupported("eval_inverted_index is not supported in function: ", - get_name()); - } }; using FunctionBasePtr = std::shared_ptr; @@ -441,6 +444,16 @@ protected: size_t result, size_t input_rows_count) const final { return function->execute_impl(context, block, arguments, result, input_rows_count); } + + Status evaluate_inverted_index( + const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const { + return function->evaluate_inverted_index(arguments, data_type_with_names, iterators, + num_rows, bitmap_result); + } + Status execute_impl_dry_run(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) const final { @@ -498,6 +511,15 @@ public: return function->close(context, scope); } + Status evaluate_inverted_index( + const ColumnsWithTypeAndName& args, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const override { + return function->evaluate_inverted_index(args, data_type_with_names, iterators, num_rows, + bitmap_result); + } + IFunctionBase::Monotonicity get_monotonicity_for_range(const IDataType& type, const Field& left, const Field& right) const override { return function->get_monotonicity_for_range(type, left, right); @@ -509,11 +531,6 @@ public: bool can_push_down_to_index() const override { return function->can_push_down_to_index(); } - Status eval_inverted_index(VExpr* expr, segment_v2::FuncExprParams& params, - std::shared_ptr& result) override { - return function->eval_inverted_index(expr, params, result); - } - private: std::shared_ptr function; DataTypes arguments; diff --git a/be/src/vec/functions/function_multi_match.cpp b/be/src/vec/functions/function_multi_match.cpp index fa36453105..e2c8a4a4a8 100644 --- a/be/src/vec/functions/function_multi_match.cpp +++ b/be/src/vec/functions/function_multi_match.cpp @@ -17,27 +17,17 @@ #include "vec/functions/function_multi_match.h" -#include #include -#include -#include -#include -#include #include #include #include #include #include "io/fs/file_reader.h" -#include "olap/olap_common.h" #include "olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query.h" #include "olap/rowset/segment_v2/segment_iterator.h" -#include "runtime/primitive_type.h" #include "vec/columns/column.h" -#include "vec/data_types/data_type.h" -#include "vec/exprs/varray_literal.h" -#include "vec/exprs/vexpr.h" #include "vec/exprs/vslot_ref.h" #include "vec/functions/simple_function_factory.h" @@ -56,97 +46,42 @@ InvertedIndexQueryType get_query_type(const std::string& query_type) { return InvertedIndexQueryType::UNKNOWN_QUERY; } -Status FunctionMultiMatch::eval_inverted_index(VExpr* expr, segment_v2::FuncExprParams& params, - std::shared_ptr& result) { - // fields - std::vector query_fileds; - size_t i = 0; - for (; i < expr->get_num_children(); i++) { - auto child_expr = expr->get_child(i); - if (child_expr->node_type() == TExprNodeType::type::SLOT_REF) { - query_fileds.emplace_back(child_expr->expr_name()); - } else { - break; - } - } - if (i != expr->get_num_children() - 2) { - return Status::RuntimeError("parameter type incorrect: slot = {}", i); - } - +Status FunctionMultiMatch::evaluate_inverted_index( + const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const { + DCHECK(arguments.size() == 2); + std::shared_ptr roaring = std::make_shared(); + std::shared_ptr null_bitmap = std::make_shared(); // type - std::string param1 = std::static_pointer_cast(expr->get_child(i))->value(); - auto query_type = get_query_type(param1); + auto query_type_value = arguments[0].column->get_data_at(0); + auto query_type = get_query_type(query_type_value.to_string()); if (query_type == InvertedIndexQueryType::UNKNOWN_QUERY) { - return Status::RuntimeError("parameter query type incorrect: query_type = {}", query_type); + return Status::RuntimeError( + "parameter query type incorrect for function multi_match: query_type = {}", + query_type); } // query - std::string query_str = std::static_pointer_cast(expr->get_child(i + 1))->value(); - - auto& segment_iterator = params._segment_iterator; - auto& segment = segment_iterator->segment(); - auto& opts = segment_iterator->storage_read_options(); - auto& tablet_schema = opts.tablet_schema; - auto& idx_iterators = segment_iterator->inverted_index_iterators(); - - // check - std::vector columns_ids; - for (const auto& column_name : query_fileds) { - auto cid = tablet_schema->field_index(column_name); - if (cid < 0) { - return Status::RuntimeError("column name is incorrect: {}", column_name); - } - if (idx_iterators[cid] == nullptr) { - return Status::RuntimeError("column idx is incorrect: {}", column_name); - } - columns_ids.emplace_back(cid); + auto query_str = arguments[1].column->get_data_at(0); + auto param_type = arguments[1].type->get_type_as_type_descriptor().type; + if (!is_string_type(param_type)) { + return Status::Error( + "arguments for multi_match must be string"); } - - // cache key - roaring::Roaring cids_str; - cids_str.addMany(columns_ids.size(), columns_ids.data()); - cids_str.runOptimize(); - std::string column_name_binary(cids_str.getSizeInBytes(), 0); - cids_str.write(column_name_binary.data()); - - InvertedIndexQueryCache::CacheKey cache_key; - io::Path index_path = segment.file_reader()->path(); - cache_key.index_path = index_path.parent_path() / index_path.stem(); - cache_key.column_name = column_name_binary; - cache_key.query_type = query_type; - cache_key.value = query_str; - - // query cache - auto* cache = InvertedIndexQueryCache::instance(); - InvertedIndexQueryCacheHandle cache_handler; - if (cache->lookup(cache_key, &cache_handler)) { - result = cache_handler.get_bitmap(); - return Status::OK(); - } - // search - for (const auto& column_name : query_fileds) { - auto cid = tablet_schema->field_index(column_name); - const auto& column = tablet_schema->column(column_name); - const auto& index_reader = idx_iterators[cid]->reader(); - + for (int i = 0; i < data_type_with_names.size(); i++) { + auto column_name = data_type_with_names[i].first; + auto* iter = iterators[i]; auto single_result = std::make_shared(); - StringRef query_value(query_str.data()); - auto index_version = tablet_schema->get_inverted_index_storage_format(); - if (index_version == InvertedIndexStorageFormatPB::V1) { - RETURN_IF_ERROR(index_reader->query(opts.stats, opts.runtime_state, column_name, - &query_value, query_type, single_result)); - } else if (index_version == InvertedIndexStorageFormatPB::V2) { - RETURN_IF_ERROR(index_reader->query(opts.stats, opts.runtime_state, - std::to_string(column.unique_id()), &query_value, - query_type, single_result)); - } - (*result) |= (*single_result); + std::shared_ptr index = std::make_shared(); + RETURN_IF_ERROR(iter->read_from_inverted_index(column_name, &query_str, query_type, + num_rows, index)); + *roaring |= *index; } - - result->runOptimize(); - cache->insert(cache_key, result, &cache_handler); - + segment_v2::InvertedIndexResultBitmap result(roaring, null_bitmap); + bitmap_result = result; return Status::OK(); } diff --git a/be/src/vec/functions/function_multi_match.h b/be/src/vec/functions/function_multi_match.h index 8e30b6e9f2..4aca75acc0 100644 --- a/be/src/vec/functions/function_multi_match.h +++ b/be/src/vec/functions/function_multi_match.h @@ -58,8 +58,11 @@ public: bool can_push_down_to_index() const override { return true; } - Status eval_inverted_index(VExpr* expr, segment_v2::FuncExprParams& params, - std::shared_ptr& result) override; + Status evaluate_inverted_index( + const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const override; }; } // namespace doris::vectorized diff --git a/be/src/vec/functions/functions_comparison.h b/be/src/vec/functions/functions_comparison.h index 09597336c2..410700b594 100644 --- a/be/src/vec/functions/functions_comparison.h +++ b/be/src/vec/functions/functions_comparison.h @@ -39,6 +39,8 @@ #include "vec/functions/function_helpers.h" #include "vec/functions/functions_logical.h" #include "vec/runtime/vdatetime_value.h" +//#include "olap/rowset/segment_v2/inverted_index_reader.h" + namespace doris::vectorized { /** Comparison functions: ==, !=, <, >, <=, >=. @@ -524,6 +526,72 @@ public: return std::make_shared(); } + Status evaluate_inverted_index( + const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const override { + DCHECK(arguments.size() == 1); + DCHECK(data_type_with_names.size() == 1); + DCHECK(iterators.size() == 1); + auto* iter = iterators[0]; + auto data_type_with_name = data_type_with_names[0]; + if (iter == nullptr) { + return Status::OK(); + } + if (iter->get_inverted_index_reader_type() == + segment_v2::InvertedIndexReaderType::FULLTEXT) { + //NOT support comparison predicate when parser is FULLTEXT for expr inverted index evaluate. + return Status::OK(); + } + std::string column_name = data_type_with_name.first; + Field param_value; + arguments[0].column->get(0, param_value); + auto param_type = arguments[0].type->get_type_as_type_descriptor().type; + + std::unique_ptr query_param = nullptr; + RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value( + param_type, ¶m_value, query_param)); + segment_v2::InvertedIndexQueryType query_type; + std::string_view name_view(name); + if (name_view == NameEquals::name || name_view == NameNotEquals::name) { + query_type = segment_v2::InvertedIndexQueryType::EQUAL_QUERY; + } else if (name_view == NameLess::name) { + query_type = segment_v2::InvertedIndexQueryType::LESS_THAN_QUERY; + } else if (name_view == NameLessOrEquals::name) { + query_type = segment_v2::InvertedIndexQueryType::LESS_EQUAL_QUERY; + } else if (name_view == NameGreater::name) { + query_type = segment_v2::InvertedIndexQueryType::GREATER_THAN_QUERY; + } else if (name_view == NameGreaterOrEquals::name) { + query_type = segment_v2::InvertedIndexQueryType::GREATER_EQUAL_QUERY; + } else { + return Status::InvalidArgument("invalid comparison op type {}", Name::name); + } + + std::shared_ptr roaring = std::make_shared(); + RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value( + param_type, ¶m_value, query_param)); + RETURN_IF_ERROR(iter->read_from_inverted_index(column_name, query_param->get_value(), + query_type, num_rows, roaring)); + std::shared_ptr null_bitmap = std::make_shared(); + if (iter->has_null()) { + segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + RETURN_IF_ERROR(iter->read_null_bitmap(&null_bitmap_cache_handle)); + null_bitmap = null_bitmap_cache_handle.get_bitmap(); + } + segment_v2::InvertedIndexResultBitmap result(roaring, null_bitmap); + bitmap_result = result; + bitmap_result.mask_out_null(); + + if (name == "ne") { + roaring::Roaring full_result; + full_result.addRange(0, num_rows); + bitmap_result.op_not(&full_result); + } + + return Status::OK(); + } + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) const override { const auto& col_with_type_and_name_left = block.get_by_position(arguments[0]); diff --git a/be/src/vec/functions/in.h b/be/src/vec/functions/in.h index 9b5c5bb023..7ab18c2618 100644 --- a/be/src/vec/functions/in.h +++ b/be/src/vec/functions/in.h @@ -29,6 +29,7 @@ #include "common/status.h" #include "exprs/create_predicate_function.h" #include "exprs/hybrid_set.h" +#include "olap/rowset/segment_v2/inverted_index_reader.h" #include "runtime/define_primitive_type.h" #include "runtime/types.h" #include "udf/udf.h" @@ -135,6 +136,64 @@ public: return Status::OK(); } + Status evaluate_inverted_index( + const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const override { + DCHECK(data_type_with_names.size() == 1); + DCHECK(iterators.size() == 1); + auto* iter = iterators[0]; + auto data_type_with_name = data_type_with_names[0]; + std::shared_ptr roaring = std::make_shared(); + std::shared_ptr null_bitmap = std::make_shared(); + + if (iter == nullptr) { + return Status::OK(); + } + if (iter->get_inverted_index_reader_type() == + segment_v2::InvertedIndexReaderType::FULLTEXT) { + //NOT support in list when parser is FULLTEXT for expr inverted index evaluate. + return Status::OK(); + } + if (iter->has_null()) { + segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + RETURN_IF_ERROR(iter->read_null_bitmap(&null_bitmap_cache_handle)); + null_bitmap = null_bitmap_cache_handle.get_bitmap(); + } + std::string column_name = data_type_with_name.first; + for (const auto& arg : arguments) { + Field param_value; + arg.column->get(0, param_value); + auto param_type = arg.type->get_type_as_type_descriptor().type; + if (param_value.is_null()) { + // predicate like column NOT IN (NULL, '') should not push down to index. + if (negative) { + return Status::OK(); + } + *roaring |= *null_bitmap; + continue; + } + std::unique_ptr query_param = nullptr; + RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value( + param_type, ¶m_value, query_param)); + InvertedIndexQueryType query_type = InvertedIndexQueryType::EQUAL_QUERY; + std::shared_ptr index = std::make_shared(); + RETURN_IF_ERROR(iter->read_from_inverted_index(column_name, query_param->get_value(), + query_type, num_rows, index)); + *roaring |= *index; + } + segment_v2::InvertedIndexResultBitmap result(roaring, null_bitmap); + bitmap_result = result; + bitmap_result.mask_out_null(); + if constexpr (negative) { + roaring::Roaring full_result; + full_result.addRange(0, num_rows); + bitmap_result.op_not(&full_result); + } + return Status::OK(); + } + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) const override { auto* in_state = reinterpret_cast( diff --git a/be/src/vec/functions/match.cpp b/be/src/vec/functions/match.cpp index 5d87ed8efd..b70a605e96 100644 --- a/be/src/vec/functions/match.cpp +++ b/be/src/vec/functions/match.cpp @@ -24,7 +24,63 @@ #include "util/debug_points.h" namespace doris::vectorized { +Status FunctionMatchBase::evaluate_inverted_index( + const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const { + DCHECK(arguments.size() == 1); + DCHECK(data_type_with_names.size() == 1); + DCHECK(iterators.size() == 1); + auto* iter = iterators[0]; + auto data_type_with_name = data_type_with_names[0]; + if (iter == nullptr) { + return Status::OK(); + } + const std::string& function_name = get_name(); + if (function_name == MATCH_PHRASE_FUNCTION || function_name == MATCH_PHRASE_PREFIX_FUNCTION || + function_name == MATCH_PHRASE_EDGE_FUNCTION) { + if (iter->get_inverted_index_reader_type() == InvertedIndexReaderType::FULLTEXT && + get_parser_phrase_support_string_from_properties(iter->get_index_properties()) == + INVERTED_INDEX_PARSER_PHRASE_SUPPORT_NO) { + return Status::Error( + "phrase queries require setting support_phrase = true"); + } + } + std::shared_ptr roaring = std::make_shared(); + Field param_value; + arguments[0].column->get(0, param_value); + auto param_type = arguments[0].type->get_type_as_type_descriptor().type; + if (!is_string_type(param_type)) { + return Status::Error( + "arguments for match must be string"); + } + std::unique_ptr query_param = nullptr; + RETURN_IF_ERROR(segment_v2::InvertedIndexQueryParamFactory::create_query_value( + param_type, ¶m_value, query_param)); + if (is_string_type(param_type)) { + auto inverted_index_query_type = get_query_type_from_fn_name(); + RETURN_IF_ERROR( + iter->read_from_inverted_index(data_type_with_name.first, query_param->get_value(), + inverted_index_query_type, num_rows, roaring)); + } else { + return Status::Error( + "invalid params type for FunctionMatchBase::evaluate_inverted_index {}", + param_type); + } + std::shared_ptr null_bitmap = std::make_shared(); + if (iter->has_null()) { + segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + RETURN_IF_ERROR(iter->read_null_bitmap(&null_bitmap_cache_handle)); + null_bitmap = null_bitmap_cache_handle.get_bitmap(); + } + segment_v2::InvertedIndexResultBitmap result(roaring, null_bitmap); + bitmap_result = result; + bitmap_result.mask_out_null(); + + return Status::OK(); +} Status FunctionMatchBase::execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) const { @@ -32,78 +88,70 @@ Status FunctionMatchBase::execute_impl(FunctionContext* context, Block& block, DataTypePtr& type_ptr = block.get_by_position(arguments[1]).type; auto match_query_str = type_ptr->to_string(*column_ptr, 0); std::string column_name = block.get_by_position(arguments[0]).name; - auto match_pred_column_name = - BeConsts::BLOCK_TEMP_COLUMN_PREFIX + column_name + "_match_" + match_query_str; - if (!block.has(match_pred_column_name)) { - VLOG_DEBUG << "begin to execute match directly, column_name=" << column_name - << ", match_query_str=" << match_query_str; - InvertedIndexCtx* inverted_index_ctx = reinterpret_cast( - context->get_function_state(FunctionContext::THREAD_LOCAL)); - if (inverted_index_ctx == nullptr) { - inverted_index_ctx = reinterpret_cast( - context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); - } - - const ColumnPtr source_col = - block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); - const auto* values = check_and_get_column(source_col.get()); - const ColumnArray* array_col = nullptr; - if (source_col->is_column_array()) { - if (source_col->is_nullable()) { - auto* nullable = check_and_get_column(source_col.get()); - array_col = check_and_get_column(*nullable->get_nested_column_ptr()); - } else { - array_col = check_and_get_column(source_col.get()); - } - if (array_col && !array_col->get_data().is_column_string()) { - return Status::NotSupported( - fmt::format("unsupported nested array of type {} for function {}", - is_column_nullable(array_col->get_data()) - ? array_col->get_data().get_name() - : array_col->get_data().get_family_name(), - get_name())); - } - - if (is_column_nullable(array_col->get_data())) { - const auto& array_nested_null_column = - reinterpret_cast(array_col->get_data()); - values = check_and_get_column( - *(array_nested_null_column.get_nested_column_ptr())); - } else { - // array column element is always set Nullable for now. - values = check_and_get_column(*(array_col->get_data_ptr())); - } - } else if (auto* nullable = check_and_get_column(source_col.get())) { - // match null - if (type_ptr->is_nullable()) { - if (column_ptr->only_null()) { - block.get_by_position(result).column = nullable->get_null_map_column_ptr(); - return Status::OK(); - } - } else { - values = check_and_get_column(*nullable->get_nested_column_ptr()); - } - } - - if (!values) { - LOG(WARNING) << "Illegal column " << source_col->get_name(); - return Status::InternalError("Not supported input column types"); - } - // result column - auto res = ColumnUInt8::create(); - ColumnUInt8::Container& vec_res = res->get_data(); - // set default value to 0, and match functions only need to set 1/true - vec_res.resize_fill(input_rows_count); - RETURN_IF_ERROR(execute_match( - context, column_name, match_query_str, input_rows_count, values, inverted_index_ctx, - (array_col ? &(array_col->get_offsets()) : nullptr), vec_res)); - block.replace_by_position(result, std::move(res)); - } else { - auto match_pred_column = - block.get_by_name(match_pred_column_name).column->convert_to_full_column_if_const(); - block.replace_by_position(result, std::move(match_pred_column)); + VLOG_DEBUG << "begin to execute match directly, column_name=" << column_name + << ", match_query_str=" << match_query_str; + InvertedIndexCtx* inverted_index_ctx = reinterpret_cast( + context->get_function_state(FunctionContext::THREAD_LOCAL)); + if (inverted_index_ctx == nullptr) { + inverted_index_ctx = reinterpret_cast( + context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); } + const ColumnPtr source_col = + block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); + const auto* values = check_and_get_column(source_col.get()); + const ColumnArray* array_col = nullptr; + if (source_col->is_column_array()) { + if (source_col->is_nullable()) { + auto* nullable = check_and_get_column(source_col.get()); + array_col = check_and_get_column(*nullable->get_nested_column_ptr()); + } else { + array_col = check_and_get_column(source_col.get()); + } + if (array_col && !array_col->get_data().is_column_string()) { + return Status::NotSupported( + fmt::format("unsupported nested array of type {} for function {}", + is_column_nullable(array_col->get_data()) + ? array_col->get_data().get_name() + : array_col->get_data().get_family_name(), + get_name())); + } + + if (is_column_nullable(array_col->get_data())) { + const auto& array_nested_null_column = + reinterpret_cast(array_col->get_data()); + values = check_and_get_column( + *(array_nested_null_column.get_nested_column_ptr())); + } else { + // array column element is always set Nullable for now. + values = check_and_get_column(*(array_col->get_data_ptr())); + } + } else if (auto* nullable = check_and_get_column(source_col.get())) { + // match null + if (type_ptr->is_nullable()) { + if (column_ptr->only_null()) { + block.get_by_position(result).column = nullable->get_null_map_column_ptr(); + return Status::OK(); + } + } else { + values = check_and_get_column(*nullable->get_nested_column_ptr()); + } + } + + if (!values) { + LOG(WARNING) << "Illegal column " << source_col->get_name(); + return Status::InternalError("Not supported input column types"); + } + // result column + auto res = ColumnUInt8::create(); + ColumnUInt8::Container& vec_res = res->get_data(); + // set default value to 0, and match functions only need to set 1/true + vec_res.resize_fill(input_rows_count); + RETURN_IF_ERROR(execute_match(context, column_name, match_query_str, input_rows_count, values, + inverted_index_ctx, + (array_col ? &(array_col->get_offsets()) : nullptr), vec_res)); + block.replace_by_position(result, std::move(res)); + return Status::OK(); } diff --git a/be/src/vec/functions/match.h b/be/src/vec/functions/match.h index 40a2fb7c1a..3026e4a06c 100644 --- a/be/src/vec/functions/match.h +++ b/be/src/vec/functions/match.h @@ -95,6 +95,11 @@ public: int32_t& current_src_array_offset) const; Status check(FunctionContext* context, const std::string& function_name) const; + Status evaluate_inverted_index( + const ColumnsWithTypeAndName& arguments, + const std::vector& data_type_with_names, + std::vector iterators, uint32_t num_rows, + segment_v2::InvertedIndexResultBitmap& bitmap_result) const override; }; class FunctionMatchAny : public FunctionMatchBase { diff --git a/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix.out b/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix.out index 11af32e55e..d5a81c28fe 100644 --- a/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix.out +++ b/regression-test/data/inverted_index_p0/test_index_match_phrase_prefix.out @@ -2,6 +2,18 @@ -- !sql -- 863 +-- !sql -- +235 + +-- !sql -- +166 + +-- !sql -- +56 + +-- !sql -- +7 + -- !sql -- 863 @@ -14,21 +26,12 @@ -- !sql -- 235 --- !sql -- -235 - -- !sql -- 166 -- !sql -- 166 --- !sql -- -166 - --- !sql -- -56 - -- !sql -- 56 @@ -41,6 +44,3 @@ -- !sql -- 7 --- !sql -- -7 - diff --git a/regression-test/suites/fault_injection_p0/test_all_index_hit_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_all_index_hit_fault_injection.groovy index d1a8e7c764..891b600076 100644 --- a/regression-test/suites/fault_injection_p0/test_all_index_hit_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_all_index_hit_fault_injection.groovy @@ -96,10 +96,10 @@ suite("test_all_index_hit_fault_injection", "nonConcurrent") { load_httplogs_data.call(indexTbName2, 'test_all_index_hit_fault_injection_2', 'true', 'json', 'documents-1000.json') sql "sync" - + sql """ set enable_common_expr_pushdown = true """ try { GetDebugPoint().enableDebugPointForAllBEs("segment_iterator._read_columns_by_index", [column_name: "clientip,request"]) - GetDebugPoint().enableDebugPointForAllBEs("segment_iterator.fast_execute", [column_name: "status,size"]) + GetDebugPoint().enableDebugPointForAllBEs("VectorizedFnCall.must_in_slow_path", [column_name: "status,size"]) qt_sql """ select count() from ${indexTbName1} where (request match_phrase 'hm'); """ @@ -124,7 +124,7 @@ suite("test_all_index_hit_fault_injection", "nonConcurrent") { } finally { GetDebugPoint().disableDebugPointForAllBEs("segment_iterator._read_columns_by_index") - GetDebugPoint().disableDebugPointForAllBEs("segment_iterator.fast_execute") + GetDebugPoint().disableDebugPointForAllBEs("VectorizedFnCall.must_in_slow_path") } } finally { } diff --git a/regression-test/suites/fault_injection_p0/test_index_inlist_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_index_inlist_fault_injection.groovy index a9b3d51352..5485239460 100644 --- a/regression-test/suites/fault_injection_p0/test_index_inlist_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_index_inlist_fault_injection.groovy @@ -76,8 +76,6 @@ suite("test_index_inlist_fault_injection", "nonConcurrent") { sql "sync" try { - GetDebugPoint().enableDebugPointForAllBEs("segment_iterator._rowid_result_for_index") - sql """ set enable_common_expr_pushdown = true; """ qt_sql """ select /*+ SET_VAR(inverted_index_skip_threshold = 0) */ count() from ${indexTbName} where clientip in ('40.135.0.0', '232.0.0.0', '26.1.0.0'); """ @@ -88,7 +86,6 @@ suite("test_index_inlist_fault_injection", "nonConcurrent") { qt_sql """ select /*+ SET_VAR(inverted_index_skip_threshold = 0) */ count() from ${indexTbName} where (request match 'hm' and status in (1, 304, 200)); """ } finally { - GetDebugPoint().disableDebugPointForAllBEs("segment_iterator._rowid_result_for_index") } try { diff --git a/regression-test/suites/fault_injection_p0/test_index_lowercase_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_index_lowercase_fault_injection.groovy index e18060a7d6..9c3f9b9deb 100644 --- a/regression-test/suites/fault_injection_p0/test_index_lowercase_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_index_lowercase_fault_injection.groovy @@ -59,6 +59,7 @@ suite("test_index_lowercase_fault_injection", "nonConcurrent") { sql """ INSERT INTO ${testTable} VALUES (893964653, '232.0.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 3781); """ sql 'sync' + sql """ set enable_common_expr_pushdown = true """ qt_sql """ select count() from ${testTable} where (request match 'HTTP'); """ qt_sql """ select count() from ${testTable} where (request match 'http'); """ diff --git a/regression-test/suites/fault_injection_p0/test_index_mow_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_index_mow_fault_injection.groovy index a1aa9a36f9..a0559119d2 100644 --- a/regression-test/suites/fault_injection_p0/test_index_mow_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_index_mow_fault_injection.groovy @@ -47,6 +47,7 @@ suite("test_index_mow_fault_injection", "nonConcurrent") { try { sql "DROP TABLE IF EXISTS ${testTable_unique}" create_httplogs_unique_table.call(testTable_unique) + sql """ set enable_common_expr_pushdown = true """ sql """ INSERT INTO ${testTable_unique} VALUES (893964617, '40.135.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 24736); """ sql """ INSERT INTO ${testTable_unique} VALUES (893964653, '232.0.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 3781); """ diff --git a/regression-test/suites/fault_injection_p0/test_need_read_data_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_need_read_data_fault_injection.groovy index 22e9e5eda3..4e0b633ddb 100644 --- a/regression-test/suites/fault_injection_p0/test_need_read_data_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_need_read_data_fault_injection.groovy @@ -73,6 +73,7 @@ suite("test_need_read_data_fault_injection", "nonConcurrent") { load_httplogs_data.call(indexTbName, 'test_need_read_data_fault_injection', 'true', 'json', 'documents-1000.json') sql "sync" + sql """ set enable_common_expr_pushdown = true """ try { GetDebugPoint().enableDebugPointForAllBEs("segment_iterator._read_columns_by_index") diff --git a/regression-test/suites/fault_injection_p0/test_topn_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_topn_fault_injection.groovy index 37315a4952..7e6c97c847 100644 --- a/regression-test/suites/fault_injection_p0/test_topn_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_topn_fault_injection.groovy @@ -96,6 +96,7 @@ suite("test_topn_fault_injection", "nonConcurrent") { load_httplogs_data.call(indexTbName2, 'test_topn_fault_injection2', 'true', 'json', 'documents-1000.json') sql "sync" + sql """ set enable_common_expr_pushdown = true """ try { GetDebugPoint().enableDebugPointForAllBEs("segment_iterator.topn_opt_1") diff --git a/regression-test/suites/inverted_index_p0/char_filter/test_char_replace.groovy b/regression-test/suites/inverted_index_p0/char_filter/test_char_replace.groovy index 556a91c5df..192c123aea 100644 --- a/regression-test/suites/inverted_index_p0/char_filter/test_char_replace.groovy +++ b/regression-test/suites/inverted_index_p0/char_filter/test_char_replace.groovy @@ -45,7 +45,7 @@ suite("test_char_replace") { "replication_allocation" = "tag.location.default: 1" ); """ - + sql """ set enable_common_expr_pushdown = true """ def var_result = sql "show variables" logger.info("show variales result: " + var_result ) diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy index 90281c7f53..5d93c7d857 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy @@ -155,6 +155,7 @@ suite("test_index_compaction_dup_keys", "nonConcurrent") { sql """ INSERT INTO ${tableName} VALUES (2, "bason", "bason hate pear", 99); """ sql """ INSERT INTO ${tableName} VALUES (3, "andy", "andy love apple", 100); """ sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate pear", 99); """ + sql """ set enable_common_expr_pushdown = true """ qt_sql """ select * from ${tableName} order by id, name, hobbies, score """ qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """ diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy index b014f10142..629f789b22 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy @@ -295,6 +295,7 @@ suite("test_index_compaction_null", "nonConcurrent") { "inverted_index_storage_format" = "V1" ) """ + sql """ set enable_common_expr_pushdown = true """ tablets = sql_return_maparray """ show tablets from ${tableName}; """ run_test.call(tablets) diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy index 3c30638a52..c7cb3d54fa 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy @@ -153,6 +153,7 @@ suite("test_index_compaction_unique_keys", "nonConcurrent") { "inverted_index_storage_format" = "V1" ); """ + sql """ set enable_common_expr_pushdown = true """ sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """ sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate pear", 99); """ diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy index 51408ef98c..47ce08cb6f 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy @@ -135,6 +135,7 @@ suite("test_index_compaction_with_multi_index_segments", "nonConcurrent") { // check config check_config.call("inverted_index_compaction_enable", "true") check_config.call("inverted_index_max_buffered_docs", "5") + sql """ set enable_common_expr_pushdown = true """ /** * test duplicated tables diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy index c1f9ac2d7c..cfe2f8004d 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy @@ -155,6 +155,7 @@ suite("test_cumulative_compaction_with_format_v2", "inverted_index_format_v2") { """ sql """ sync """ + sql """ set enable_common_expr_pushdown = true """ qt_select_default """ SELECT * FROM ${tableName} t WHERE city MATCH 'Beijing' ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """ diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy index 1a75137e1e..2de9c7fcbc 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_column_with_format_v2.groovy @@ -95,6 +95,7 @@ suite("test_drop_column_with_format_v2", "inverted_index_format_v2"){ sql """ INSERT INTO ${tableName} VALUES (2, "bason", 99); """ sql """ INSERT INTO ${tableName} VALUES (3, "andy", 100); """ sql """ INSERT INTO ${tableName} VALUES (3, "bason", 99); """ + sql """ set enable_common_expr_pushdown = true """ qt_sql "SELECT * FROM $tableName WHERE name match 'andy' order by id, name, score;" diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy index 12877099f3..e572535109 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_drop_index_with_format_v2.groovy @@ -74,6 +74,7 @@ suite("test_drop_index_with_format_v2", "inverted_index_format_v2"){ sql """ INSERT INTO ${tableName} VALUES (2, "bason", 99); """ sql """ INSERT INTO ${tableName} VALUES (3, "andy", 100); """ sql """ INSERT INTO ${tableName} VALUES (3, "bason", 99); """ + sql """ set enable_common_expr_pushdown = true """ qt_sql "SELECT * FROM $tableName WHERE name match 'andy' order by id, name, score;" diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy index 639f73349c..50eacbea7e 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_mor_table_with_format_v2.groovy @@ -155,6 +155,7 @@ suite("test_mor_table_with_format_v2", "inverted_index_format_v2") { """ sql """ sync """ + sql """ set enable_common_expr_pushdown = true """ sql """ DELETE FROM ${tableName} WHERE user_id = 3 """ diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy index fcb1a15f3d..717a094795 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy @@ -155,6 +155,7 @@ suite("test_mow_table_with_format_v2", "inverted_index_format_v2") { """ sql """ sync """ + sql """ set enable_common_expr_pushdown = true """ qt_select_default """ SELECT * FROM ${tableName} t WHERE city MATCH 'Beijing' ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """ diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy index 38b03510a0..baaff07852 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_rename_column_with_format_v2.groovy @@ -67,6 +67,7 @@ suite("test_rename_column_with_format_v2", "inverted_index_format_v2"){ sql """ INSERT INTO ${tableName} VALUES (2, "bason", 99); """ sql """ INSERT INTO ${tableName} VALUES (3, "andy", 100); """ sql """ INSERT INTO ${tableName} VALUES (3, "bason", 99); """ + sql """ set enable_common_expr_pushdown = true """ qt_sql "SELECT * FROM $tableName WHERE name match 'andy' order by id, name, score;" diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy index fce14b8796..931074c352 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_single_replica_compaction_with_format_v2.groovy @@ -159,6 +159,7 @@ suite("test_single_replica_compaction_with_format_v2", "inverted_index_format_v2 """ sql """ sync """ + sql """ set enable_common_expr_pushdown = true """ qt_select_default """ SELECT * FROM ${tableName} t WHERE city MATCH 'Beijing' ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """ diff --git a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_v1.groovy b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_v1.groovy index e30dddf54c..02ebc1b5b1 100644 --- a/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_v1.groovy +++ b/regression-test/suites/inverted_index_p0/storage_format/test_storage_format_v1.groovy @@ -98,6 +98,7 @@ suite("test_storage_format_v1", "p0") { create_httplogs_dup_table.call(testTable_dup) load_httplogs_data.call(testTable_dup, 'test_httplogs_load_count_on_index', 'true', 'json', 'documents-1000.json') sql "sync" + sql """ set enable_common_expr_pushdown = true """ qt_sql(" select COUNT(*) from ${testTable_dup} where request match 'images' ") diff --git a/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy index 1364d3da82..3a81e8cead 100644 --- a/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy @@ -63,6 +63,7 @@ suite("test_array_contains_with_inverted_index"){ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', '0974e7a82e30d1af83205e474fadd0a2', '[\"w\"]'); """ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', '26823b3995ee38bd145ddd910b2f6300', '[\"x\"]'); """ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', 'ee27ee1da291e46403c408e220bed6e1', '[\"y\"]'); """ + sql """ set enable_common_expr_pushdown = true """ qt_sql """ select count() from ${indexTblName}""" order_qt_sql """ select * from tai where array_contains(inventors, 's') order by id; """ diff --git a/regression-test/suites/inverted_index_p0/test_array_index.groovy b/regression-test/suites/inverted_index_p0/test_array_index.groovy index 38eee6d405..d8879bf0f8 100644 --- a/regression-test/suites/inverted_index_p0/test_array_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_array_index.groovy @@ -61,6 +61,7 @@ suite("test_array_index1"){ "enable_single_replica_compaction" = "false" ); """ + sql """ set enable_common_expr_pushdown = true """ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '6afef581285b6608bf80d5a4e46cf839', '[\"a\", \"b\", \"c\"]'); """ sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', 'd93d942d985a8fb7547c72dada8d332d', '[\"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\"]'); """ diff --git a/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy b/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy index 81665829e8..75e38dc7a6 100644 --- a/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy +++ b/regression-test/suites/inverted_index_p0/test_array_with_inverted_index_all_type.groovy @@ -279,6 +279,7 @@ suite("test_array_with_inverted_index_all_type"){ } // query test + sql """ set enable_common_expr_pushdown = true """ for (int i = 0; i < 6; i+=1) { def indexTblName = indexTblNames[i] diff --git a/regression-test/suites/inverted_index_p0/test_chinese_analyzer.groovy b/regression-test/suites/inverted_index_p0/test_chinese_analyzer.groovy index 8278170cd4..417c829ca8 100644 --- a/regression-test/suites/inverted_index_p0/test_chinese_analyzer.groovy +++ b/regression-test/suites/inverted_index_p0/test_chinese_analyzer.groovy @@ -98,6 +98,7 @@ suite("test_chinese_analyzer"){ "replication_allocation" = "tag.location.default: 1" ); """ + sql """ set enable_common_expr_pushdown = true """ sql "INSERT INTO $indexTblName3 VALUES (1, '我来到北京清华大学'), (2, '我爱你中国'), (3, '人民可以得到更多实惠'), (4, '陕西省西安市高新区创业大厦A座,我的手机号码是12345678901,邮箱是12345678@qq.com,,ip是1.1.1.1,this information is created automatically.');" qt_sql "SELECT * FROM $indexTblName3 WHERE c MATCH_PHRASE '我爱你' ORDER BY id;" diff --git a/regression-test/suites/inverted_index_p0/test_compound.groovy b/regression-test/suites/inverted_index_p0/test_compound.groovy index 9b1b8c7dde..e1f8034114 100644 --- a/regression-test/suites/inverted_index_p0/test_compound.groovy +++ b/regression-test/suites/inverted_index_p0/test_compound.groovy @@ -57,6 +57,7 @@ suite("test_compound", "p0"){ (9, '9', '9', '9'), (10, '10', '10', '10'); """ + sql """ set enable_common_expr_pushdown = true """ qt_sql "SELECT count() FROM $indexTblName WHERE (id >= 2 AND id < 9) and (a match '2' or b match '5' and c match '5');" qt_sql "SELECT count() FROM $indexTblName WHERE (id >= 2 AND id < 9) and (a match '2' or b match '5' or c match '6');" diff --git a/regression-test/suites/inverted_index_p0/test_compound_1.groovy b/regression-test/suites/inverted_index_p0/test_compound_1.groovy index 92dd2f78f0..a8aca4b526 100644 --- a/regression-test/suites/inverted_index_p0/test_compound_1.groovy +++ b/regression-test/suites/inverted_index_p0/test_compound_1.groovy @@ -80,6 +80,7 @@ suite("test_compound_1", "p0"){ load_httplogs_data.call(indexTbName, 'test_compound_1', 'true', 'json', 'documents-1000.json') sql "sync" + sql """ set enable_common_expr_pushdown = true """ qt_sql """ select count() from ${indexTbName} where (request match_phrase 'english' and clientip match_phrase '4' or request match_phrase 'images'); """ qt_sql """ select count() from ${indexTbName} where (request match_phrase 'hm' and clientip match_phrase '3' or request match_phrase 'gif'); """ diff --git a/regression-test/suites/inverted_index_p0/test_compound_inlist.groovy b/regression-test/suites/inverted_index_p0/test_compound_inlist.groovy index 855e2c0eec..d49b1df03c 100644 --- a/regression-test/suites/inverted_index_p0/test_compound_inlist.groovy +++ b/regression-test/suites/inverted_index_p0/test_compound_inlist.groovy @@ -103,6 +103,7 @@ suite("test_compound_inlist", "nonConcurrent"){ load_httplogs_data.call(indexTbName2, 'test_compound_list_2', 'true', 'json', 'documents-1000.json') sql "sync" + sql """ set enable_common_expr_pushdown = true """ qt_sql """ select /*+ SET_VAR(inverted_index_skip_threshold = 0) */ count() from ${indexTbName1} where (((request match_phrase 'images' and clientip match_phrase '3') or (request match_phrase 'english' and clientip match_phrase '4')) and status in (1, 2, 304)); """ qt_sql """ select /*+ SET_VAR(inverted_index_skip_threshold = 0) */ count() from ${indexTbName2} where (((request match_phrase 'images' and clientip match_phrase '3') or (request match_phrase 'english' and clientip match_phrase '4')) and status in (1, 2, 304)); """ diff --git a/regression-test/suites/inverted_index_p0/test_count_on_index.groovy b/regression-test/suites/inverted_index_p0/test_count_on_index.groovy index 490e998ed5..95feccb8a1 100644 --- a/regression-test/suites/inverted_index_p0/test_count_on_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_count_on_index.groovy @@ -141,6 +141,7 @@ suite("test_count_on_index_httplogs", "p0") { stream_load_data.call(testTable_unique, 'test_httplogs_load_count_on_index', 'true', 'json', 'documents-1000.json') sql "sync" + sql """ set enable_common_expr_pushdown = true """ sql """set experimental_enable_nereids_planner=true;""" sql """set enable_fallback_to_original_planner=false;""" // case1: test duplicate table diff --git a/regression-test/suites/inverted_index_p0/test_count_on_index_2.groovy b/regression-test/suites/inverted_index_p0/test_count_on_index_2.groovy index de94067e2f..77e5de052b 100644 --- a/regression-test/suites/inverted_index_p0/test_count_on_index_2.groovy +++ b/regression-test/suites/inverted_index_p0/test_count_on_index_2.groovy @@ -153,6 +153,7 @@ suite("test_count_on_index_2", "p0"){ load_httplogs_data.call(indexTbName2, indexTbName2, 'true', 'json', 'documents-1000.json') sql "sync" + sql """ set enable_common_expr_pushdown = true """ qt_sql """ select count() from ${indexTbName1} where `@timestamp` >= 893964736 and `@timestamp` <= 893966453; """ qt_sql """ select count() from ${indexTbName2} where `@timestamp` >= 893964736 and `@timestamp` <= 893966453; """ diff --git a/regression-test/suites/inverted_index_p0/test_delete.groovy b/regression-test/suites/inverted_index_p0/test_delete.groovy index 3d6100ca4f..06cd51cc87 100644 --- a/regression-test/suites/inverted_index_p0/test_delete.groovy +++ b/regression-test/suites/inverted_index_p0/test_delete.groovy @@ -47,6 +47,7 @@ suite("test_delete"){ "enable_single_replica_compaction" = "false" ); """ + sql """ set enable_common_expr_pushdown = true """ sql """ INSERT INTO `${indexTblName}`(`a`, `b`, `c`) VALUES ('1', '6afef581285b6608bf80d5a4e46cf839', 'aaa'), ('2', '48a33ec3453a28bce84b8f96fe161956', 'bbb'), ('3', '021603e7dcfe65d44af0efd0e5aee154', 'ccc'), ('4', 'ee27ee1da291e46403c408e220bed6e1', 'ddd'), diff --git a/regression-test/suites/inverted_index_p0/test_equal_on_fulltext.groovy b/regression-test/suites/inverted_index_p0/test_equal_on_fulltext.groovy index 6d3a1bd06c..debb5b07e3 100644 --- a/regression-test/suites/inverted_index_p0/test_equal_on_fulltext.groovy +++ b/regression-test/suites/inverted_index_p0/test_equal_on_fulltext.groovy @@ -45,6 +45,7 @@ suite("test_equal_on_fulltext", "p0"){ def var_result = sql "show variables" logger.info("show variales result: " + var_result ) + sql """ set enable_common_expr_pushdown = true """ sql "INSERT INTO $indexTblName VALUES (1, 'I am the person'), (2, 'I am a person'), (3, 'I am your person');" qt_sql "SELECT * FROM $indexTblName WHERE c = 'I am a person' ORDER BY id;" diff --git a/regression-test/suites/inverted_index_p0/test_index_chinese_column.groovy b/regression-test/suites/inverted_index_p0/test_index_chinese_column.groovy index 21a94e1ffe..e0221fb67a 100644 --- a/regression-test/suites/inverted_index_p0/test_index_chinese_column.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_chinese_column.groovy @@ -38,6 +38,7 @@ suite("test_index_chinese_column", "inverted_index_select"){ def table_name_v2 = "test_index_chinese_column_v2" sql "set enable_unicode_name_support=true" + sql """ set enable_common_expr_pushdown = true """ createAndInsertData(table_name_v1, "V1") createAndInsertData(table_name_v2, "V2") diff --git a/regression-test/suites/inverted_index_p0/test_index_complex_match.groovy b/regression-test/suites/inverted_index_p0/test_index_complex_match.groovy index 191e147685..0b77c4aec5 100644 --- a/regression-test/suites/inverted_index_p0/test_index_complex_match.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_complex_match.groovy @@ -101,6 +101,7 @@ suite("test_index_complex_match", "p0"){ load_httplogs_data.call(indexTbName2, indexTbName2, 'true', 'json', 'documents-1000.json') sql "sync" + sql """ set enable_common_expr_pushdown = true """ qt_sql """ select count() from ${indexTbName1} where clientip match_phrase '247.37.0.0'; """ qt_sql """ select count() from ${indexTbName1} where clientip match_phrase_prefix '247'; """ diff --git a/regression-test/suites/inverted_index_p0/test_index_delete.groovy b/regression-test/suites/inverted_index_p0/test_index_delete.groovy index 51d9dd4430..8b9474d937 100644 --- a/regression-test/suites/inverted_index_p0/test_index_delete.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_delete.groovy @@ -49,6 +49,7 @@ suite("test_index_delete", "p0") { try { sql "sync" + sql """ set enable_common_expr_pushdown = true """ sql """ delete from ${indexTbName1} where a >= 9; """ sql "sync" diff --git a/regression-test/suites/inverted_index_p0/test_index_empty_string.groovy b/regression-test/suites/inverted_index_p0/test_index_empty_string.groovy index 2cf1d844d2..05232ec6bc 100644 --- a/regression-test/suites/inverted_index_p0/test_index_empty_string.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_empty_string.groovy @@ -47,7 +47,7 @@ suite("test_index_empty_string", "p0"){ (1, '', '1'), (2, '2', ''); """ - + sql """ set enable_common_expr_pushdown = true """ qt_sql "SELECT count() FROM $indexTblName WHERE a match '';" qt_sql "SELECT count() FROM $indexTblName WHERE b match '';" } diff --git a/regression-test/suites/inverted_index_p0/test_index_equal_select.groovy b/regression-test/suites/inverted_index_p0/test_index_equal_select.groovy index 108e5615e4..de8991e72e 100644 --- a/regression-test/suites/inverted_index_p0/test_index_equal_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_equal_select.groovy @@ -54,6 +54,7 @@ suite("test_index_equal_select", "inverted_index_select"){ ("san zhang", 10, "grade 5", "2017-10-01", "tall:100cm, weight: 30kg, hobbies:", "", "", "", "", ""), ("li sisi", 11, "grade 6", "2016-10-01", "tall:150cm, weight: 40kg, hobbies: sing, dancing, running", "good at handiwork and beaty", "", "li ba", "li liuliu", "") """ + sql """ set enable_common_expr_pushdown = true """ // case1: test equal // case1.0: test index coulume equal ‘’ diff --git a/regression-test/suites/inverted_index_p0/test_index_key_match_select.groovy b/regression-test/suites/inverted_index_p0/test_index_key_match_select.groovy index dc04ba8786..b3b758aa23 100644 --- a/regression-test/suites/inverted_index_p0/test_index_key_match_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_key_match_select.groovy @@ -44,6 +44,7 @@ suite("test_index_key_match_select", "inverted_index_select"){ ("u3", ["u1"]), ("u4", ["u3"]) """ + sql """ set enable_common_expr_pushdown = true """ qt_sql "SELECT * FROM ${indexTbName1} WHERE user MATCH_ANY 'u1, u2' ORDER BY user LIMIT 10;" qt_sql "SELECT * FROM ${indexTbName1} WHERE user MATCH_ANY 'u1, u2, u3' ORDER BY user LIMIT 10;" } \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_index_match_phrase.groovy b/regression-test/suites/inverted_index_p0/test_index_match_phrase.groovy index d15ab10f9c..0d89b5b841 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_phrase.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_phrase.groovy @@ -153,6 +153,7 @@ suite("test_index_match_phrase_select", "inverted_index_select"){ sql """ build index ${text_colume1}_idx on ${indexTbName1} """ wait_for_build_index_on_partition_finish(indexTbName1, timeout) } + sql """ set enable_common_expr_pushdown = true; """ // case1: test match_phrase "" try { diff --git a/regression-test/suites/inverted_index_p0/test_index_match_phrase_edge.groovy b/regression-test/suites/inverted_index_p0/test_index_match_phrase_edge.groovy index 8d4ab3d232..b7fe566455 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_phrase_edge.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_phrase_edge.groovy @@ -16,7 +16,7 @@ // under the License. -suite("test_index_match_phrase_edge", "p0"){ +suite("test_index_match_phrase_edge", "nonConcurrent"){ def indexTbName1 = "test_index_match_phrase_edge" sql "DROP TABLE IF EXISTS ${indexTbName1}" @@ -56,6 +56,8 @@ suite("test_index_match_phrase_edge", "p0"){ try { sql "sync" + sql """ set enable_common_expr_pushdown = true; """ + GetDebugPoint().enableDebugPointForAllBEs("VMatchPredicate.execute") qt_sql """ select * from ${indexTbName1} where b match_phrase_edge 'x.h'; """ qt_sql """ select * from ${indexTbName1} where b match_phrase_edge 'v_i'; """ @@ -75,6 +77,6 @@ suite("test_index_match_phrase_edge", "p0"){ qt_sql """ select count() from ${indexTbName1} where c match_phrase_edge 'b'; """ } finally { - //try_sql("DROP TABLE IF EXISTS ${testTable}") + GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") } } \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_index_match_phrase_ordered.groovy b/regression-test/suites/inverted_index_p0/test_index_match_phrase_ordered.groovy index 137bab70f0..a65811d4f6 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_phrase_ordered.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_phrase_ordered.groovy @@ -16,7 +16,7 @@ // under the License. -suite("test_index_match_phrase_ordered", "p0"){ +suite("test_index_match_phrase_ordered", "nonConcurrent"){ def indexTbName1 = "test_index_match_phrase_ordered" sql "DROP TABLE IF EXISTS ${indexTbName1}" @@ -49,6 +49,8 @@ suite("test_index_match_phrase_ordered", "p0"){ try { sql "sync" + sql """ set enable_common_expr_pushdown = true; """ + GetDebugPoint().enableDebugPointForAllBEs("VMatchPredicate.execute") qt_sql """ select count() from ${indexTbName1} where b match_phrase 'the lazy'; """ qt_sql """ select count() from ${indexTbName1} where b match_phrase 'the lazy ~1'; """ @@ -82,6 +84,6 @@ suite("test_index_match_phrase_ordered", "p0"){ qt_sql """ select count() from ${indexTbName1} where b match_phrase 'the quick ~6'; """ qt_sql """ select count() from ${indexTbName1} where b match_phrase 'the quick ~6+'; """ } finally { - //try_sql("DROP TABLE IF EXISTS ${testTable}") + GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") } } \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix.groovy b/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix.groovy index 1f83d48a97..3906065e8d 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix.groovy @@ -16,7 +16,7 @@ // under the License. -suite("test_index_match_phrase_prefix", "p0"){ +suite("test_index_match_phrase_prefix", "nonConcurrent"){ def indexTbName1 = "test_index_match_phrase_prefix" def indexTbName2 = "test_index_match_phrase_prefix2" @@ -98,28 +98,32 @@ suite("test_index_match_phrase_prefix", "p0"){ load_httplogs_data.call(indexTbName2, indexTbName2, 'true', 'json', 'documents-1000.json') sql "sync" - + sql """ set enable_common_expr_pushdown = true; """ + GetDebugPoint().enableDebugPointForAllBEs("VMatchPredicate.execute") qt_sql """ select count() from ${indexTbName1} where request match_phrase_prefix 'ima'; """ + qt_sql """ select count() from ${indexTbName1} where request match_phrase_prefix 'images/h'; """ + qt_sql """ select count() from ${indexTbName1} where request match_phrase_prefix 'images/hm'; """ + qt_sql """ select count() from ${indexTbName1} where request match_phrase_prefix '/french/images/n'; """ + qt_sql """ select count() from ${indexTbName1} where request match_phrase_prefix '/french/tickets/images/ti'; """ + GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") + qt_sql """ select count() from ${indexTbName2} where request match_phrase_prefix 'ima'; """ qt_sql """ select count() from ${indexTbName1} where request like '%ima%'; """ - qt_sql """ select count() from ${indexTbName1} where request match_phrase_prefix 'images/h'; """ qt_sql """ select count() from ${indexTbName2} where request match_phrase_prefix 'images/h'; """ qt_sql """ select count() from ${indexTbName1} where request like '%images/h%'; """ - qt_sql """ select count() from ${indexTbName1} where request match_phrase_prefix 'images/hm'; """ qt_sql """ select count() from ${indexTbName2} where request match_phrase_prefix 'images/hm'; """ qt_sql """ select count() from ${indexTbName1} where request like '%images/hm%'; """ - qt_sql """ select count() from ${indexTbName1} where request match_phrase_prefix '/french/images/n'; """ + qt_sql """ select count() from ${indexTbName2} where request match_phrase_prefix '/french/images/n'; """ qt_sql """ select count() from ${indexTbName1} where request like '%/french/images/n%'; """ - qt_sql """ select count() from ${indexTbName1} where request match_phrase_prefix '/french/tickets/images/ti'; """ qt_sql """ select count() from ${indexTbName2} where request match_phrase_prefix '/french/tickets/images/ti'; """ qt_sql """ select count() from ${indexTbName1} where request like '%/french/tickets/images/ti%'; """ } finally { - //try_sql("DROP TABLE IF EXISTS ${testTable}") + GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") } } \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy b/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy index 1e2ac0d790..f42462f12a 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix_1.groovy @@ -16,7 +16,7 @@ // under the License. -suite("test_index_match_phrase_prefix_1", "p0"){ +suite("test_index_match_phrase_prefix_1", "nonConcurrent"){ def indexTbName1 = "test_index_match_phrase_prefix_1" sql "DROP TABLE IF EXISTS ${indexTbName1}" @@ -49,6 +49,8 @@ suite("test_index_match_phrase_prefix_1", "p0"){ try { sql "sync" + sql """ set enable_common_expr_pushdown = true; """ + GetDebugPoint().enableDebugPointForAllBEs("VMatchPredicate.execute") qt_sql """ select count() from ${indexTbName1} where c match_phrase_prefix 'O1704361998540E2Cemx9S'; """ qt_sql """ select count() from ${indexTbName1} where d match_phrase_prefix 'O1704361998540E2Cemx9S'; """ @@ -57,6 +59,6 @@ suite("test_index_match_phrase_prefix_1", "p0"){ qt_sql """ select count() from ${indexTbName1} where d match_phrase_prefix 'O1704361998540E2Cemx9S=123456789'; """ } finally { - //try_sql("DROP TABLE IF EXISTS ${testTable}") + GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") } } \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_index_match_phrase_slop.groovy b/regression-test/suites/inverted_index_p0/test_index_match_phrase_slop.groovy index a8454878e6..5f772e0e7a 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_phrase_slop.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_phrase_slop.groovy @@ -16,7 +16,7 @@ // under the License. -suite("test_index_match_phrase_slop", "p0"){ +suite("test_index_match_phrase_slop", "nonConcurrent"){ def indexTbName1 = "test_index_match_phrase_slop" sql "DROP TABLE IF EXISTS ${indexTbName1}" @@ -82,6 +82,8 @@ suite("test_index_match_phrase_slop", "p0"){ sql """ INSERT INTO ${indexTbName1} VALUES (1, "127.0.0.1", "I'm glad I kept my fingers crossed ~4", 1, 1); """ sql "sync" + sql """ set enable_common_expr_pushdown = true; """ + GetDebugPoint().enableDebugPointForAllBEs("VMatchPredicate.execute") qt_sql """ select count() from ${indexTbName1} where request match_phrase 'get jpg'; """ qt_sql """ select count() from ${indexTbName1} where request match_phrase 'get jpg ~2'; """ @@ -117,6 +119,6 @@ suite("test_index_match_phrase_slop", "p0"){ qt_sql """ select * from ${indexTbName1} where request match_phrase 'glad crossed \\~4'; """ } finally { - //try_sql("DROP TABLE IF EXISTS ${testTable}") + GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") } } \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy b/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy index 3186343270..49f0f56398 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy @@ -16,7 +16,7 @@ // under the License. -suite("test_index_match_regexp", "p0"){ +suite("test_index_match_regexp", "nonConcurrent"){ def indexTbName1 = "test_index_match_regexp" sql "DROP TABLE IF EXISTS ${indexTbName1}" @@ -79,6 +79,8 @@ suite("test_index_match_regexp", "p0"){ load_httplogs_data.call(indexTbName1, 'test_index_match_regexp', 'true', 'json', 'documents-1000.json') sql "sync" + sql """ set enable_common_expr_pushdown = true; """ + GetDebugPoint().enableDebugPointForAllBEs("VMatchPredicate.execute") qt_sql """ select count() from test_index_match_regexp where request match_regexp ''; """ qt_sql """ select count() from test_index_match_regexp where request match_regexp '^h'; """ @@ -89,6 +91,6 @@ suite("test_index_match_regexp", "p0"){ qt_sql """ select count() from test_index_match_regexp where request match_regexp 'nonexistence'; """ } finally { - //try_sql("DROP TABLE IF EXISTS ${testTable}") + GetDebugPoint().disableDebugPointForAllBEs("VMatchPredicate.execute") } } \ No newline at end of file diff --git a/regression-test/suites/inverted_index_p0/test_index_match_select.groovy b/regression-test/suites/inverted_index_p0/test_index_match_select.groovy index 21e17a1be7..82451cd09c 100644 --- a/regression-test/suites/inverted_index_p0/test_index_match_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_match_select.groovy @@ -152,6 +152,7 @@ suite("test_index_match_select", "inverted_index_select"){ sql """ build index ${text_colume1}_idx on ${indexTbName1} """ wait_for_build_index_on_partition_finish(indexTbName1, timeout) } + sql """ set enable_common_expr_pushdown = true; """ // case1: match term // case1.0 test match "" diff --git a/regression-test/suites/inverted_index_p0/test_index_multi_match.groovy b/regression-test/suites/inverted_index_p0/test_index_multi_match.groovy index c926bc4d51..884fbaffa6 100644 --- a/regression-test/suites/inverted_index_p0/test_index_multi_match.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_multi_match.groovy @@ -126,4 +126,4 @@ suite("test_index_multi_match", "p0"){ } finally { //try_sql("DROP TABLE IF EXISTS ${testTable}") } -} \ No newline at end of file +} diff --git a/regression-test/suites/inverted_index_p0/test_index_no_need_read_data.groovy b/regression-test/suites/inverted_index_p0/test_index_no_need_read_data.groovy index fd871b0a00..924fffe54d 100644 --- a/regression-test/suites/inverted_index_p0/test_index_no_need_read_data.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_no_need_read_data.groovy @@ -54,6 +54,7 @@ suite("test_index_no_need_read_data", "inverted_index_select"){ // case1: enable nereids planner sql "set enable_nereids_planner = true" + sql """ set enable_common_expr_pushdown = true; """ qt_select_nereids_0 "SELECT * FROM ${table1} ORDER BY id" qt_select_nereids_1 "SELECT count() FROM ${table1} WHERE n > 100" diff --git a/regression-test/suites/inverted_index_p0/test_index_null.groovy b/regression-test/suites/inverted_index_p0/test_index_null.groovy index ef61149661..90ef0d9a06 100644 --- a/regression-test/suites/inverted_index_p0/test_index_null.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_null.groovy @@ -42,7 +42,7 @@ suite("test__null_index", "inverted_index"){ "replication_allocation" = "tag.location.default: 1" ); """ - + sql """ set enable_common_expr_pushdown = true; """ sql "INSERT INTO $indexTblName VALUES (1, []), (2, []), (3, []);" qt_sql "SELECT * FROM $indexTblName WHERE value match_all 'a';" } diff --git a/regression-test/suites/inverted_index_p0/test_index_range_between_select.groovy b/regression-test/suites/inverted_index_p0/test_index_range_between_select.groovy index c93191973c..5401c9e410 100644 --- a/regression-test/suites/inverted_index_p0/test_index_range_between_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_range_between_select.groovy @@ -117,6 +117,7 @@ suite("test_index_range_between_select", "inverted_index_select"){ """ wait_for_latest_op_on_table_finish(indexTbName1, timeout) } + sql """ set enable_common_expr_pushdown = true; """ // case1: test simple between case // case1.0: test data index colume select in specific between condition diff --git a/regression-test/suites/inverted_index_p0/test_index_range_bigger_and_equal_select.groovy b/regression-test/suites/inverted_index_p0/test_index_range_bigger_and_equal_select.groovy index e0e3ee949d..517caaea65 100644 --- a/regression-test/suites/inverted_index_p0/test_index_range_bigger_and_equal_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_range_bigger_and_equal_select.groovy @@ -54,7 +54,7 @@ suite("test_index_range_bigger_and_equal_select", "inverted_index_select"){ ("san zhang", 10, "grade 5", "2017-10-01", "tall:100cm, weight: 30kg, hobbies:", "", "", "", "", ""), ("li sisi", 11, "grade 6", "2016-10-01", "tall:150cm, weight: 40kg, hobbies: sing, dancing, running", "good at handiwork and beaty", "", "li ba", "li liuliu", "") """ - + sql """ set enable_common_expr_pushdown = true; """ // case1. test >= // case1.0: test only >= qt_sql "select * from ${indexTbName1} where name>='' order by name " diff --git a/regression-test/suites/inverted_index_p0/test_index_range_bigger_select.groovy b/regression-test/suites/inverted_index_p0/test_index_range_bigger_select.groovy index d1d29d2bd7..937d24ba2d 100644 --- a/regression-test/suites/inverted_index_p0/test_index_range_bigger_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_range_bigger_select.groovy @@ -54,7 +54,7 @@ suite("test_index_range_bigger_select", "inverted_index_select"){ ("san zhang", 10, "grade 5", "2017-10-01", "tall:100cm, weight: 30kg, hobbies:", "", "", "", "", ""), ("li sisi", 11, "grade 6", "2016-10-01", "tall:150cm, weight: 40kg, hobbies: sing, dancing, running", "good at handiwork and beaty", "", "li ba", "li liuliu", "") """ - + sql """ set enable_common_expr_pushdown = true; """ // case1. test > // case1.0: test only > qt_sql "select * from ${indexTbName1} where name>'' order by name " diff --git a/regression-test/suites/inverted_index_p0/test_index_range_in_select.groovy b/regression-test/suites/inverted_index_p0/test_index_range_in_select.groovy index 637ce50eea..136d35cac0 100644 --- a/regression-test/suites/inverted_index_p0/test_index_range_in_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_range_in_select.groovy @@ -151,6 +151,7 @@ suite("test_index_range_in_select", "inverted_index_select"){ sql """ build index ${text_colume1}_idx on ${indexTbName1} """ wait_for_build_index_on_partition_finish(indexTbName1, timeout) } + sql """ set enable_common_expr_pushdown = true; """ // case1: select in // case1.0: select in specific condition diff --git a/regression-test/suites/inverted_index_p0/test_index_range_not_in_select.groovy b/regression-test/suites/inverted_index_p0/test_index_range_not_in_select.groovy index 5cb79845a4..54fc77e0f8 100644 --- a/regression-test/suites/inverted_index_p0/test_index_range_not_in_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_range_not_in_select.groovy @@ -111,6 +111,7 @@ suite("test_index_range_not_in_select", "inverted_index_select"){ } assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") } + sql """ set enable_common_expr_pushdown = true; """ for (int i = 0; i < 2; i++) { logger.info("select table with index times " + i) diff --git a/regression-test/suites/inverted_index_p0/test_index_range_smaller_and_equal_select.groovy b/regression-test/suites/inverted_index_p0/test_index_range_smaller_and_equal_select.groovy index 0c5cd61ee6..4cc3ca1ab6 100644 --- a/regression-test/suites/inverted_index_p0/test_index_range_smaller_and_equal_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_range_smaller_and_equal_select.groovy @@ -54,7 +54,7 @@ suite("test_index_range_smaller_and_equal_select", "inverted_index_select"){ ("san zhang", 10, "grade 5", "2017-10-01", "tall:100cm, weight: 30kg, hobbies:", "", "", "", "", ""), ("li sisi", 11, "grade 6", "2016-10-01", "tall:150cm, weight: 40kg, hobbies: sing, dancing, running", "good at handiwork and beaty", "", "li ba", "li liuliu", "") """ - + sql """ set enable_common_expr_pushdown = true; """ // case1. test <= // case1.0: test only <= qt_sql "select * from ${indexTbName1} where name<='' order by name" diff --git a/regression-test/suites/inverted_index_p0/test_index_range_smaller_select.groovy b/regression-test/suites/inverted_index_p0/test_index_range_smaller_select.groovy index 83d2220c05..cf03e7c8ca 100644 --- a/regression-test/suites/inverted_index_p0/test_index_range_smaller_select.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_range_smaller_select.groovy @@ -54,7 +54,7 @@ suite("test_index_range_smaller_select", "inverted_index_select"){ ("san zhang", 10, "grade 5", "2017-10-01", "tall:100cm, weight: 30kg, hobbies:", "", "", "", "", ""), ("li sisi", 11, "grade 6", "2016-10-01", "tall:150cm, weight: 40kg, hobbies: sing, dancing, running", "good at handiwork and beaty", "", "li ba", "li liuliu", "") """ - + sql """ set enable_common_expr_pushdown = true; """ // case1. test < // case1.0: test only < sql "select * from ${indexTbName1} where name<'' order by name " diff --git a/regression-test/suites/inverted_index_p0/test_index_rqg_bug.groovy b/regression-test/suites/inverted_index_p0/test_index_rqg_bug.groovy index cc56178266..02762e5798 100644 --- a/regression-test/suites/inverted_index_p0/test_index_rqg_bug.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug.groovy @@ -142,7 +142,7 @@ suite("test_index_rqg_bug", "test_index_rqg_bug"){ (48, -10, 7, 3, 4, -1290467110130692882, NULL, -5421887030808227301, 2147894047624029750, '2023-12-20', '2026-02-18', '2023-12-10', '2024-02-18', 'v', 'f', 'u', 'z', 'w', 'l', 'i', 'b'), (49, 4, -10, -10, -4, 7177870619817484302, 2010854013707344984, 515636226818986547, -4617727694631456148, '2023-12-14', '2024-01-09', '2023-12-11', '2024-01-08', 'k', 'o', 'r', 'h', 'x', 'v', 'm', 'r'); """ - + sql """ set enable_common_expr_pushdown = true; """ qt_select_bug_1 """ SELECT MIN(DISTINCT table1.col_date_undef_signed_not_null) AS field1, diff --git a/regression-test/suites/inverted_index_p0/test_index_rqg_bug2.groovy b/regression-test/suites/inverted_index_p0/test_index_rqg_bug2.groovy index faede0bc3c..e1411a7043 100644 --- a/regression-test/suites/inverted_index_p0/test_index_rqg_bug2.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug2.groovy @@ -82,7 +82,7 @@ suite("test_index_rqg_bug2", "test_index_rqg_bug"){ sql """ INSERT INTO ${table1}(pk,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_undef_signed_not_null_index_inverted,col_bigint_undef_signed,col_bigint_undef_signed_index_inverted,col_bigint_undef_signed_not_null,col_bigint_undef_signed_not_null_index_inverted,col_date_undef_signed,col_date_undef_signed_index_inverted,col_date_undef_signed_not_null,col_date_undef_signed_not_null_index_inverted,col_varchar_10__undef_signed,col_varchar_10__undef_signed_index_inverted,col_varchar_10__undef_signed_not_null,col_varchar_10__undef_signed_not_null_index_inverted,col_varchar_1024__undef_signed,col_varchar_1024__undef_signed_index_inverted,col_varchar_1024__undef_signed_not_null,col_varchar_1024__undef_signed_not_null_index_inverted,col_array_bigint__undef_signed,col_array_bigint__undef_signed_index_inverted,col_array_bigint__undef_signed_not_null,col_array_bigint__undef_signed_not_null_index_inverted,col_array_varchar_64___undef_signed,col_array_varchar_64___undef_signed_index_inverted,col_array_varchar_64___undef_signed_not_null,col_array_varchar_64___undef_signed_not_null_index_inverted,col_array_date__undef_signed,col_array_date__undef_signed_index_inverted,col_array_date__undef_signed_not_null,col_array_date__undef_signed_not_null_index_inverted) values (0,0,3,9,0,4421173,-8231206,-7461,2743,'2023-12-19','2023-12-15','2024-01-08','2025-06-18','e','h','x','p',null,null,'s','g',[1005281, 5740, 15054, null],[-5954, -19854, null, null, 3273993, -743, 337700, null, -1752691, 23217, 3506706],[null, null, -5265, null, 860851, -29957, null, -9710, 10115, -3146277, 11735, null, null, 18080, 7566486, null, -6484648, null, -4857664, 15816, 9875, -2686449, -941314, 8627, null, -7359, -4659, 28539, -16240, null, 17323, -2496932, 2292142, null, 11991, -8504, -9360, -8096795, 14727, null, null, 3306678, 14025, null, -6519359, 5625825, 6130567, null, null, null, null, null, 17792, null, -7991400, null, null, 22846, -28743],[-12355, -4008, 7917948, -2452821, null, 16207, 28195],['i', 'x', 'd', 't', 'e'],['l', null, 'l'],['m', 'w', null, 'd', 'y', 'f', 'c', 'q', 't', 'q'],[null, 'v', null, 'b', 'a', 'c', 'v', 'o', 'w', 'r', 'k'],["2025-02-18"],["2024-02-18", null, "2023-12-11", "2025-06-18", "2023-12-15", null, "2026-02-18", "2026-02-18", "2023-12-17"],["2024-01-17", "2023-12-10", "2023-12-09", "2024-01-09", "2026-02-18"],["2023-12-13", null, "2023-12-13"]),(1,2,2,3,3,-8059,null,5394,-1559484,'2023-12-15','2024-01-08','2025-06-18','2023-12-10','p','i','n','a','c','j','n','r',[null, 3283380, 3627449, -30527, -2736105, -10860, null],[null, 2459969, null, null, -3092713, 4381894, null, 1817602, null, 1706544, 16985],[null, 3991, -3206489],[-3022450, 1639, -5855269, null, null, -3561, -2618, 21678, 22004, 6803800, -357540, -2566523, -19982, 7530447, -1828804, 14471, null, null, -613238, 8073, -27918, null, null, 6211343, 18433, null, 30493, null, 8284, 1765944, 2897396, 15644, -3006094, null, 19552, null, 4895858, 15398, null, null, -8101195, 5945, null, -2924303],['z', null, 'q', 'd', null, null],['m', 'w', 'i', 'p', null, 'y', 'h', null, 'e', 'l', 'c', 'y', null, null, null, 's', 'i', 'g', 'l', 'b', 'v', 'c', null, 'd', 'a', 'j', 'f', 'h', 't', 'f', 'l'],['m', 'm', 'c', null, 's', 'z', null],['o', 'g', 'v', 'o', null, 'p', 'a', null, 's', null, 'k', null, 'i', 'b', 't', null, 's', 'm', 't', 'k', null, 'v', 'v', 'l', 'p', 'w', 'h', 'z'],["2024-01-17", "2025-06-18", "2024-01-19", "2024-01-09", "2023-12-18", "2024-01-08", "2023-12-18", "2025-02-18", "2025-06-18", "2025-06-18", "2026-01-18", "2024-01-31", "2024-01-08", "2024-02-18", "2024-02-18", "2023-12-09", "2023-12-16", null, null, "2024-02-18", "2024-01-17", "2027-01-16", "2024-01-09", null, null, "2025-06-18", "2024-02-18", "2023-12-20", "2025-06-18", "2026-02-18", "2024-01-19", "2027-01-16", "2024-01-17", "2024-02-18", "2025-06-18", null, "2027-01-09", "2024-02-18", "2023-12-13", "2025-02-17", "2024-01-09", "2023-12-18"],["2023-12-10", "2027-01-16", "2023-12-18", "2025-06-18", "2023-12-15", "2024-02-18", null, "2023-12-17", "2023-12-12", "2024-02-18", "2023-12-12", "2026-02-18", "2025-06-18", "2027-01-16"],["2023-12-19", null, "2023-12-14", "2023-12-10", "2023-12-11", "2024-01-17", "2024-01-17", "2023-12-12"],["2026-02-18", "2023-12-20", "2025-02-18", "2023-12-09", "2023-12-11", "2024-01-31", "2024-01-31", "2023-12-13", "2027-01-09", "2025-06-18", "2023-12-12", "2024-01-08", "2027-01-16", "2024-01-19", "2024-02-18", "2024-02-18", "2023-12-16", "2023-12-13", "2023-12-15", "2027-01-16", "2024-02-18", "2024-02-18", "2023-12-10", null, "2023-12-12", "2023-12-16", null, "2023-12-18", "2025-06-18", null, "2023-12-20", "2023-12-17", "2025-02-18"]),(2,-4,4,-4,-4,null,-8299718,-7790681,19649,'2023-12-20','2023-12-13','2023-12-09','2024-02-18','z','q','n','i','p','h','k','n',[29956, null, 2708152, 4180766, null, -16646, null, -960, -26572, null, 4418, null, null, null, null, -27193, -29347, -8615, -7836, -16066, -3784267, 2303156, 963691, 32368, null, -22973, -5475540, null, -8794, 5610750, 17826, null, null, null, 2642936, -29332, 21828, -24485, 23310, -21928],[null, null, null, null, null, 1690605, null, -4551, -24909, null, -8141713, 1664203, -6145254],[5432838, -28532, 9084],[6438479, null, 3698119, null, -21146, 30309, 23574, 19919, 8448, -8596, 6998816, -4624542, -22538, 19632, -30444, null, 7453453, -2480542, 16136, -356373, -4239, null, -1557467, -397756, -756, null, 9332, null, null, -321, -4952849, null, 65629, 17251, -3555105, -30177, -230, 18043, null, -13366, -7390776, null, 6654835],[null, 'g'],['a', 't', 'n', 'e', 'z', null, 'x'],[null, 'd', 'r', 'u', 'i', 'j', 'h', 'a', 'b', 'h'],[],["2023-12-17"],["2025-06-18", "2024-01-09", "2024-01-31"],["2025-02-18", "2023-12-15", null, "2024-01-17", "2024-02-18"],["2024-01-19", "2023-12-12", "2025-02-17", "2027-01-16", "2023-12-18", "2026-02-18", "2023-12-12", "2023-12-11", "2025-02-18", "2026-02-18", "2023-12-18", "2023-12-20", "2023-12-13", "2025-06-18", "2024-01-31", "2025-06-18", "2023-12-09", "2024-02-18", null, "2024-02-18", "2023-12-17", "2023-12-18", "2025-02-18", "2023-12-12", null, "2023-12-20", "2024-02-18", "2023-12-12", "2024-02-18", null, "2026-01-18", "2024-01-09", null, "2023-12-20", "2023-12-16", "2024-01-08", null, "2026-02-18", null, "2027-01-16", "2023-12-12", "2024-01-08"]),(3,9,-10,3,5,-5152627,-5145135,6986504,-20604,'2023-12-15','2023-12-13','2023-12-14','2023-12-15','v','f','u','o','n','b','h','y',[3443],[6391740, 5422888, -492, 8245552, 2636, -2780809, -29978],[16088, -7247349, -7458590, 2740940, null, null, null, -21826, -1848935, null, null, null, 4418535],[1142157, -3273, -15958, 2881572, -27206, -26818, null, -28001, 2584151, null, null, 11184, -31822, null, null, null, -5407054, null, -5605359, null, -11412],['e', 'g', 'b', 'o', 'w'],['g', 'g', 'k', 'i', 'w', 'n', 'o', 'v', 'f', 'd', 'z', 'q', null, 'w', 'f', 'w', null, null, null, 's', 'h', 'v', 's', 'u', null, 'n'],['d', 'n', 'g'],['e', 'm', 'k', 'm', 'h', 'u', 'z', 'l', 'n', 'q', 'x'],["2023-12-20", "2023-12-14", "2025-02-17", "2023-12-15", null, "2024-02-18", "2023-12-15", null, "2023-12-16"],["2025-02-18", "2023-12-14", "2025-06-18"],["2023-12-13", "2025-06-18", "2027-01-09", null, "2024-02-18", "2023-12-19", "2026-02-18", "2026-01-18"],["2023-12-18", "2024-01-17", "2024-02-18", "2025-06-18"]),(4,3,-4,3,-4,null,-1584250,-332882,-4905447,'2025-02-17',null,'2027-01-09','2023-12-16',null,'y','e','o','u','v','m','c',[-15763, -5359106, -24434],[-3779239, null, null, null, null, 5060, null, null, -31955, -6906924, 7177028],[-6845, -602, null, 21649, -17902, -25075, 13328, 5810518, 22416, -53406, 16876, 6328, null, null, null, -23941],[11186, null],['u', 'y', 'n', 'h', 't', 'k', null, null, 'l'],[null, 's', 't', 'q', 'p', 't', 'e', 'c', 'y', 'm', 'x', 'w', 'm', 'b', 'g', 'w', 'z', null, 'r', 'd', null, 'n', null, 'm', 'e', 'u', null, 'q', 'm', 'p', null, 'z', 't', 'o', 's'],[null, 'd'],[null, 'b', null, 'r', 'h', 's', 'q', 'h', 'm'],[null, null, "2024-02-18", "2024-01-17", "2023-12-13", "2024-01-17"],["2027-01-16", "2025-06-18", null],["2024-01-09"],["2024-01-31", "2025-02-18", "2027-01-16"]),(5,3,null,6,8,2854778,9666,-4700976,-7871284,'2025-02-18',null,'2024-01-09','2024-01-17','v','j','l','v',null,'b','n','g',[-9161, 806576, null, null, 2539356, 3477002, null, 19621, null],[7250884, null, 8219730, null, 6493797, -31014, -2669, 3599799, -735224, 25173, 13308, 5906342, 8070499, 4572, -6979430, 27920],[],[-6159748, null, null, 9043, null, null, 20090, null, null, -22439, null],['x', null, 'd', 'h'],[null, 'i', 'w', 't', 'f', null],['d', 'e', 'x', 'p', 's', 'i', 'f', 'd'],['a', 'z'],["2024-02-18", "2023-12-20", "2025-02-18", "2024-02-18", "2024-02-18", "2023-12-13", "2024-02-18", "2025-06-18", "2023-12-11", null, "2027-01-09", "2023-12-19"],["2025-02-17", "2026-01-18", "2027-01-16", "2023-12-14", "2025-02-18", "2024-01-19", "2025-06-18", "2023-12-16", "2027-01-09", "2023-12-10", "2025-02-18", null, "2025-06-18"],["2023-12-14", "2023-12-19", "2024-01-31", "2025-02-17"],["2023-12-11", "2024-01-08", "2025-02-17", null, "2024-01-08", "2023-12-19", "2024-02-18", "2023-12-14", "2024-01-09", "2026-02-18", "2023-12-12", "2023-12-20", "2024-02-18", "2025-02-17", "2023-12-16", "2025-02-17", "2023-12-18", "2023-12-13", "2027-01-16", "2023-12-15", null, "2026-02-18", "2023-12-15", "2024-01-08", "2023-12-12", "2023-12-09", "2023-12-18", "2024-01-31", "2023-12-11", "2023-12-15", "2025-02-17", null, "2024-02-18", "2023-12-10", "2023-12-16", null, "2024-01-08", null, "2024-01-31", "2023-12-09", "2023-12-10", null, "2023-12-11", "2024-02-18", "2023-12-15", null, "2025-02-17", "2024-01-31", "2023-12-09", "2023-12-14", null, "2026-01-18", "2023-12-20", "2024-01-31", "2024-02-18", "2023-12-14", "2024-01-09", "2024-01-17", "2024-02-18", "2024-02-18", "2026-01-18", "2024-02-18", "2024-01-08", "2023-12-17", "2023-12-16", "2023-12-15", "2023-12-09", "2023-12-20", "2024-01-31", null, "2023-12-14", "2023-12-13"]),(6,-10,9,-4,4,17064,null,-3444527,6831972,'2027-01-16',null,'2025-02-17','2023-12-20','a','v','e','x','q','t','v','c',[-12248, 5706253],[8265758, 1891518, null, 343319, null, 6145286, -7277684, -4439, 18409, -27099, -4940368, -1878, -6233, 1380723],[17182, 3109441, 7300860, -7329372, null, null, null, 6012060],[371441, null, 30764, 4967, -11962, 60226, null, null, null, null, null, -5034956, 6903140, -15094, -29581, 11835, -2859160, 17616, 10084, -1767810, 27580],['w', 'u', 'j', null, 'm', 'u', 'w', 'c'],['u', 'o', 'e', 'x', 'e', 'f', 'c', 'd', 'z', null, 'o'],['v', null, 'z', 'f', 'u', 'r', 'l', 'k', null, 'k', 'k', null, null, 'n', 'v', 'y', 'k', 'd', 'i', 'b', 'e', 'p', null, 'z', 'y', 'b', 'f', 'i', 'b', 'r', 'd', 'e', null, 'n', null, 'l', 'c', 'g', null, 'j', null, 'd', 'n', 'e', 's', 'r', null, 's', 'j', 'v', 'p', 'u'],['y', 'u', 'u'],["2025-06-18", "2025-02-18", null, "2024-01-17", "2024-02-18", "2024-02-18", null, "2024-02-18", null, "2023-12-18", "2027-01-16", "2023-12-13", null, "2023-12-15", "2027-01-16", "2023-12-15", "2025-02-17", "2026-02-18", "2024-01-17", "2023-12-18", "2023-12-09"],[null, "2023-12-20", "2024-02-18", "2024-01-09", null, null, "2025-02-17", "2023-12-12", "2023-12-18", "2025-02-18", "2023-12-10", "2026-02-18", "2024-02-18", "2025-06-18", "2023-12-18", "2023-12-17", "2027-01-09", "2024-01-19", "2023-12-17", "2024-01-17", "2027-01-09", "2024-01-09", null, "2024-01-31", "2023-12-11", "2024-02-18", "2026-01-18"],["2025-06-18", "2023-12-11", "2024-01-08", "2026-01-18", "2025-06-18", "2027-01-09", "2027-01-09", null],["2023-12-18", "2024-02-18", "2023-12-19", "2025-02-18", "2025-06-18", "2024-02-18", "2024-01-19", "2025-06-18", "2024-02-18", "2023-12-12", null, "2027-01-09", "2024-01-17", "2023-12-12", "2023-12-20", "2025-06-18", "2026-01-18", "2024-01-08", "2025-06-18", null, null, "2024-01-31", "2023-12-11", null, "2024-01-17", null, "2023-12-17", "2023-12-14", "2026-02-18", "2023-12-15", "2025-06-18", "2023-12-12", null, "2024-02-18", "2025-06-18", "2023-12-16", null, "2023-12-11", "2025-06-18", "2023-12-10", "2027-01-09"]),(7,6,-4,-10,1,-8383139,488676,25942,-6844644,null,'2024-01-08','2023-12-20','2027-01-09','k','f','m','r','l','t','f','x',[null, 31370, null, 11344, null],[1289638, 7378090, -4076, -3654288, 20933, 3795642, null, 7046449, -25007],[2771190, 20162, 877099, 2413, -4529674, 2233135, 14455, -3560001, -29631, 2047976, 5856889, -27930, -4774575, -7977519, null, 30017, null, -954908, null, 4583480, null, null],[4552647, 18706, 21794, -8005, -2240647, null, -2388, null, 732884, 3046184, null, null, 7337, -4615980, -6795924, -21823, -5841, 2815120, null, null, -8481, 32709, -4908792, 14179, -4490016, null, -2945, null, null, null, null, -23025, 24814, -2769112, 24921, null, null, 7970959, -24730, 1030977, -7668, 16500, 7548094, 16112, null, null, null, null],['q', 'o', 'k', 'c', 's', null, 'p', null, 'g', 'z', 'd', 'r', 'd', 'a', null, 'w', null, 'l', 'm', 'v'],[null, 'z', 'm', 'u', null, null, 'e', 'x', 'p', null],[null, 'h', null, 't', null, null],['f', null, 'v', null, 'l', 'k', 'a', null, 'l', null, null, 'i', null, 'h', 'j', null, 'm', 'z', 'b', null, 'h', 'c', 'z', 't', 'v', 'e', 'i', 't', 't', 'x', 'a', 'y', 'v', 'd', 'e', 'h', 's', 'u', null, 'b', 'g', 'h', null, 'e', null, 't', 'f', 'j', 'c', 'd', 'a', 'v', 'r', null, 'o', 'h', 'n', null, null, null, 'n', 'c', 'p', null, null, 'n', null, 's', 'y', null, 'k', 'w', 'i', 'a', null, 'x', 'p', 'y', 't', null, null, 'u', null],[null, "2023-12-18", "2024-02-18"],["2023-12-12", "2024-02-18", "2023-12-19", "2023-12-12", "2026-01-18", "2023-12-18"],[null, "2024-02-18", "2023-12-14", "2023-12-13", "2025-02-17", "2024-01-09", "2025-06-18", "2025-02-18"],[null, "2023-12-20", "2025-06-18", "2023-12-14"]),(8,-4,1,7,-10,-7132,1008329,5183204,-25057,'2024-01-09','2023-12-10','2024-01-17','2025-06-18',null,null,'w','n','v','n','m','w',[15252, 27759, -6459, null, -5040142, null],[null, null],[null, null, -8485, 3758840, null],[28819, -4400, null, -708, null, null, -4493405, -5017, null, null, 5215210, null, -213, -4337490, null, -2456643, 26206, 26048, -9226, -1046, null, -24230, null, 7979008, -3717682, -3668055, 9611, 21102, null, -2425835, 5395423, -8103945, null],[null, 'y', 'y', 'p', 't', 'v', null, 'v', 'g', 'e', 'f', 'v', 'q', 'k', 'i', 'w', 'h', 'j', 'y', 't', 'm', 'j', null, 'q', 'e', 'g', null, 'p', 'e', 'u', 'z', 'a', 'g', null, 'n', 'p', 'n'],[],[],['t', 'h', null, 'o', 'p', 'z', 'c', 'n', null, null, 'b', 'c', 'f', 'f', null, 's', 'm', 'u', 'i', null, 'i', 'a', 'u', 'h', 'f', 'y', 'v', 'b', 'p', 'f', 'a', 'p', 't', 'u', 'c', 'a'],["2025-02-17", null, "2024-02-18", null, "2026-01-18", "2024-02-18"],["2023-12-18", "2023-12-09"],[],["2024-01-09", "2023-12-13", "2024-01-17", "2023-12-16", "2026-02-18", "2026-01-18", "2023-12-10", "2023-12-16"]),(9,0,null,5,-10,-12101,null,-294874,6623565,'2023-12-14','2025-02-18','2023-12-15','2023-12-17','y','y','y','w','n','y','r','e',[-7239304],[-1460490, 5930961, 2339440, 1278440, 4116181, 6436894],[-12742, -4825983, null, 5439401, -5250, -27217],[null, 6417146, 4081139],['p', 'j', 'w', 'x', 'o', 'k', null, 't', null, 'b', 'e', 'x', 'o', 's', 'l', 'q', 'l', null, null, 'j', 'y', 'n', 'e', 'g', null, 'a', 's', 'v', 'q', 'u', 'j', 't', 'a', 'z', 'm', 'x', 'd', null, 'y', 'f', 'm', null, 'b', 'e', null, 'i', 'x', 'u', null],['h', 'f', null, null, 'g', 'p', 'e', 'f', 'l', 'm', 'v', 'm', 'g', 'p', 'g', 'm', 'e', null, 'k', 'h', 'a', 'd', 'y', 'j', 'j', 'm', 'g', 'r', 'c', 'u', null, 'd', 'i', 'r', 'l', 'j', null, null],['x', 'm', 'q', 'a', null, 't', 'r', 'n', 't', 'z', 'q', 'c', null, 'd', 'k'],['f', null, 'z', 'o'],["2023-12-16", null, "2024-01-09", "2024-01-19"],[],["2023-12-12", "2024-01-08", "2024-01-09", "2023-12-11", "2023-12-16", "2023-12-19", "2025-02-17", "2023-12-12", "2023-12-15", "2023-12-12", "2024-02-18"],["2025-06-18", "2027-01-09", "2023-12-09", "2024-01-17", "2024-01-19", "2023-12-10", "2024-01-09"]),(10,-4,6,-4,8,7668480,12311,-2302965,24187,'2024-02-18','2023-12-14','2023-12-19','2023-12-18','w','e','q','c','r','j','p','m',[4435282, 695137, 5167063, -7144199, 4491, 622539, 328263],[null, 7263600, null, 2962764, null, null, 4443],[null, null, -315785, -19966, -12331, 2297033, -2155440, null, -11150, -17178, 4148249, 10145, -2822, 572786, null, null, null, null, null, -5736527, null, null, null, 6880525, 6270008, 1824, -32226, null, null, null, -7586, null, 2928908, -6873675, 4374133, 30907, -6856016, -7325917, null, null, -23290, null, 12121, -56130, 1723237, null, -13447, 2900221],[-3887447, 29513, -27218, 7731280, 8020208, -7153276, -6191389],['z', null, null, 'h', null, 'j'],['w', 'd', 't', 'n', 'p', 'u', 'f'],['o', 'x', 'l', 'p', 'd', 'q', 'p', 'n', 'w'],['n', 'r', 'g', null, 'h', 'e', 'y', 'r', 'o'],["2027-01-16", "2024-01-09", "2025-06-18", "2025-02-17", "2023-12-13", "2023-12-10"],["2027-01-09", "2023-12-09", "2024-02-18", "2026-01-18", "2025-06-18"],["2023-12-13", "2025-02-17", "2023-12-18", "2023-12-17", "2024-02-18", "2023-12-16", null, null, "2026-01-18"],[null, "2025-06-18", "2023-12-12", "2023-12-16", "2023-12-18", null, "2025-02-18", "2027-01-16", "2024-02-18", "2023-12-18"]),(11,7,3,0,-10,27729,-1597057,8437,10443,'2023-12-12','2023-12-09','2026-01-18','2023-12-20','j','g','i','r','l','g','g','t',[],[-392, -21067],[5163, -7859152, 30104, null, 963298, -21071, 8298766],[-3715006, null, -28561, -6846775, 1179, -4711549, -26316, -2904644, null, -4577223, 5725555, 30687, null, 3598210, null, 1875361, -4470182, 2060, -5637840],['r', null],[null, 'l', 'u', null, 't', null, 'o', null, 'w', 'l', 'l', 'k'],['p', null, 'x', 'f', 'e', 'q', 'z', 'd', 'e', 'j', 'g', null, 'c', 'v', 'y', 's', 'l', null, 'g', null, 'p', 'x'],['u', 'x', 'h', 'x', 'c', 'z', 's', null, 'c', 'p', 'j', 'k', 'd', 't', null, 'o', null, 'm', 't', 'k', 'j', 'h', null, 'n', 'i', 'c', 'q', null, 'v', null, null, 'o', 'p', 'b', 't', 'b', 'q', 'r', 'b', 't', 'e', null, 'r'],["2026-01-18", "2025-02-17", "2024-01-31"],["2024-02-18", "2023-12-18", "2024-01-31", "2023-12-12", "2023-12-13", "2023-12-17", "2023-12-19", "2024-02-18", "2023-12-17", "2023-12-14", "2024-01-08", "2024-02-18", "2025-06-18", "2024-01-31", "2024-02-18", "2023-12-11", null, "2024-01-17", null, "2023-12-19", null, "2023-12-19", "2023-12-13", "2026-02-18", "2023-12-19", null, "2024-02-18", "2024-01-08", null, "2024-01-17", "2023-12-20", "2023-12-18", "2023-12-10", "2023-12-13", null, "2025-06-18", "2023-12-17", null, null, null, "2023-12-17", "2024-01-19", "2023-12-14", "2025-02-17", "2023-12-12", "2026-02-18", "2025-02-17"],[null, "2023-12-19", "2023-12-15", "2023-12-18", "2025-02-18", "2023-12-20", "2023-12-18", "2024-02-18", "2024-01-17"],[]),(12,-10,-4,0,-10,null,-27963,-2818488,1425862,'2023-12-10','2023-12-14','2023-12-09','2025-06-18','w','a','n','v',null,'v','f','c',[null, 1305092, -2438814, -6947222, -2614382, -28347, 32717, null, -23097, null, 4288469, 1963103, null, 1001797, 6249472, 623992, 758210],[7055189, -32253, -2337710, null, null, 17178, null, null, 1646866, -438270, null, 5981487, -17043, -7135891, -23360, 4857165, 24183, -1995678, 636071, 783, -805, 10852, -5138163, 24798, 1858131, 15237, -3565, null, -23376],[951977, 2311680, 31740, null, -26586, -5684614, null, -5229659, 27866, -10603, -2936, null, -12738, null, -388178, -19354, null, 8377388],[null, 16024, -468293, -6161072],[null, 'a', 'v', null],['b', 'k', 't', null, 'u', 'o', 'q', 'a', null, 'j', 'b', 'r', 'u', 'x', 'h', 'h', 'k', null, 'm', 'o', 'w', 'l', 'h', null, 'y', 'p', 'c', null, 'm', 'o', 'y', 'i', 'm', 'q', 'x', 'm', 't', 'f', 'x', 't'],['s', 's', 'e', 'b', 'p', 'y', 'f'],['u', 'c', 'p', 'h', 'o'],["2023-12-12", "2023-12-15", "2023-12-13"],["2025-06-18", "2023-12-11", "2023-12-12", "2027-01-16", "2024-02-18", "2023-12-11", "2027-01-16", "2024-02-18", "2023-12-16", "2027-01-09", "2023-12-12", "2024-01-09", "2025-02-17", "2024-01-19", "2023-12-20", "2024-01-08", "2023-12-11", "2023-12-20", "2024-02-18", "2023-12-19", "2024-01-08", "2023-12-12", "2025-06-18", null, "2025-06-18", null, "2023-12-09", "2023-12-18", null],["2023-12-16", "2024-02-18", null, "2024-02-18", "2026-01-18", "2023-12-14", "2023-12-12", "2025-06-18", "2026-02-18", "2027-01-16"],["2025-06-18", "2024-01-08", "2024-01-09", "2027-01-09", null, "2023-12-16", "2027-01-16"]),(13,5,null,8,-4,null,3558665,3390,10801,'2024-01-17','2024-01-08','2024-01-08','2025-02-18','s','i','s','u','d','t','u','o',[-5186684, 3411, null, 5556519, null, 6257, null, 30818, 30628, -24827, -1912, null],[3939, null, null, 19998, -27853, null, -6305, -24322, null, -6037403, null, -30153, 3345863, -394168, -3201436, 27663, 25009, null, 29942, 24805, null, 1808732, -4244398, -3569708, 7456786, null, 310339, null, null, -19688, null, -1303433, 5384133, -543381, -329, 9036, 3698961, null, -5843128, null, null, -741, null, 26031],[-3462562, 6447984, null, 7605875, null, 1538616, 449, -3868637, null, -1787, -4423897, -11485, null, -1150262, 6422841, 2106955, 4331427, -181, null],[null, -8448, -16310, -20983, 1397438],[null, 's', 'c', 'w', 'b', 'y', null, 'k'],[],['q', 'y', 'c', 'q', 'z', 'u', 'q', 'k'],['u', null, 'x', null, 'z', 'w', 't', 't', 'r', 'b', 's', 'o', 'c', 'j', 'v', 's', 'p', 'x', 'e', 'o', 'm', 'm', 'j', null, null, 'v', 'm', 'g', 'x', 'f', null, 'c', 'u', 'p', null, null, 'n', null, 'l'],["2024-01-08", "2023-12-11", "2024-02-18", "2023-12-16", "2025-02-18", null, null, "2023-12-14", "2023-12-15", "2023-12-12", "2024-02-18", null, "2023-12-13", "2023-12-19", null, "2024-02-18", "2023-12-11", "2024-01-17", "2027-01-09", "2023-12-09", "2024-02-18", "2023-12-14", "2023-12-17", "2027-01-16", "2023-12-13", "2023-12-14", "2023-12-13", null, "2027-01-09", "2024-01-08", "2024-01-19", "2027-01-16", "2024-02-18", "2024-01-19", "2024-01-31", "2024-02-18", "2023-12-17", "2023-12-11", "2023-12-11", "2023-12-09", "2024-01-17", "2023-12-20", "2026-01-18", "2025-02-18", "2024-02-18", "2023-12-12", "2023-12-13", "2024-01-09", "2023-12-16", "2023-12-11", "2025-02-18", "2025-06-18", null, "2027-01-16", "2024-02-18", "2024-01-17", null, "2023-12-19", "2025-06-18", "2024-01-19", "2023-12-15", "2024-02-18", "2023-12-20", "2023-12-12", "2024-01-19", "2023-12-18", "2023-12-10", "2026-02-18", "2023-12-09", "2023-12-09", "2025-02-18", "2023-12-11", "2026-01-18", "2023-12-19", "2024-01-19", "2025-02-17", "2023-12-13", "2024-01-08", "2025-06-18", "2027-01-16", "2025-02-17", "2023-12-17", "2023-12-20", "2023-12-17", "2024-01-31", "2023-12-18", "2023-12-19", "2027-01-09", "2023-12-17", "2027-01-16", "2023-12-18", "2024-01-31", "2025-02-17", "2026-02-18", "2025-02-18", "2024-02-18", "2023-12-19"],["2023-12-19", "2023-12-17", "2026-01-18", "2023-12-09", "2023-12-13", "2025-06-18", "2023-12-11", "2023-12-09", "2023-12-09", "2026-01-18", "2024-01-19", "2024-01-09", "2025-06-18", "2023-12-09", "2023-12-12", "2023-12-18", "2024-01-08", "2024-02-18", "2026-01-18", "2027-01-16", null, "2024-01-31", "2027-01-16", "2025-06-18", "2025-06-18", null, "2025-06-18", null, "2024-01-19", "2027-01-09", "2025-06-18", "2025-06-18", "2025-06-18", null, "2024-01-19", "2024-02-18", "2025-06-18", "2025-06-18", "2023-12-09", "2025-02-17", "2023-12-11", "2026-02-18", "2024-01-17", "2026-02-18", "2025-06-18", "2023-12-18", "2026-01-18", "2023-12-16", "2023-12-15", "2025-06-18", "2024-02-18", "2024-01-09", "2023-12-14", "2025-06-18", "2026-02-18", "2023-12-19", "2023-12-12", "2023-12-12", "2024-02-18", "2026-01-18", "2027-01-16", "2024-01-09", "2023-12-16", "2024-02-18", "2024-01-08", "2027-01-09", "2024-02-18", "2024-01-31", null, "2024-01-17", "2024-02-18", "2023-12-12", "2023-12-09", "2024-01-08", "2025-02-17", "2024-02-18", "2024-01-09", "2026-02-18", "2025-02-18", "2026-02-18", "2026-02-18", "2023-12-15", "2023-12-16", "2023-12-11", "2023-12-17", "2024-02-18", "2027-01-16", "2023-12-10"],["2024-01-31", "2025-02-18"],[null, "2023-12-17", "2023-12-18", "2023-12-14", null, "2023-12-17", "2025-06-18", "2025-06-18", null, null, "2024-02-18", null, "2025-06-18", "2023-12-09", "2024-01-19", "2023-12-11", "2027-01-09", "2023-12-20", "2023-12-17", "2023-12-11", "2025-02-17", "2023-12-10", "2025-06-18", null, null, "2026-01-18"]),(14,5,4,9,-4,null,-6919782,14792,5528709,'2024-02-18','2026-01-18','2026-02-18','2025-06-18','d','b','m','o','d','a','s','r',[3181050, -23959, null, 6563755, 6061],[-17423, null, null, null],[null, null, 25045, -19501, null, -15807, null, -8336621, 5072, -19448, -472845, null, 4409],[-23925],['h', 'r', 'x'],[],['m', null, null],[null, 's', null, null, null, 'q', 'j', 'h'],["2027-01-16"],["2023-12-10", "2024-01-09"],["2023-12-20", "2024-01-31", null, "2024-01-31", "2023-12-20", null, "2025-02-18", "2023-12-16", "2023-12-09", null, "2024-01-31", "2024-01-19", "2024-02-18"],["2024-01-08", "2024-01-09"]),(15,4,-4,5,8,-28910,-8118655,5617,7465079,'2025-02-17','2023-12-19','2027-01-09','2027-01-09','u','e','y','f','a','l','b','c',[17894, null, null, null, 3463, -30173, -6635474],[null, -29403, -2945200, -10643, -3074444, 2596382, null, -31507, null, null, null, -1859636, 5348331, 4720562, 14109, 19714, 28650, 28246, 2045787, null, null, null, 10957, -6899012, null, null, null],[-837730, 986528, 6323561, 7993, -8234195, null, null, -2027585, null, null, null, 10504, -4749, 31158, 25319, null, null, -5326842, -5365541, 2200382, null, -6135103, 2870363, null, null, -14882, null, 17664, 734, 19430, 17512, null, null, -22941, -19808, null, -356592, 7205186, null, null, 7268014, -7876274, -6850378, 3418],[239344, -7900359, -4679874, -6340806, -924637, null, null, -2088500, null, -5296280, null, -5950761, null, -2743293, null, 23006, -5690930, -7113168, 24992, -1228822, null, 30513],[null, 'x', null, 'g', 's', 'm', 'e', 'z', 'k', 'p', 'p', 'n', 'q', null, 'z', null, 'j', null, 'o', 'y', 'y', null, null, 'j', 'f', null, 'h', 'l', 'u'],['f', 'f', 'b', 'g', 'z', 's', 'v', 'v', 'z', 'd', 'v', 'f', 'd', 'i'],['n', 'f', null, null],['l', 'w', 'u', null, 'a', null, null, 'm', 'y', 'l', 'v', 'm', 's', 'c', 'n', 'h', 'u', 'z'],[null, "2024-01-09", "2025-02-17", "2023-12-13", "2026-02-18", "2023-12-10", "2024-01-31", "2024-01-31", "2024-01-19", "2023-12-13", "2024-01-19", "2025-06-18", "2025-02-17", "2027-01-09", "2024-01-17", "2024-02-18", "2024-01-17", "2023-12-15", "2026-01-18", "2025-06-18", "2025-06-18", "2023-12-19", "2023-12-09", "2023-12-14", "2026-01-18", "2023-12-19", "2027-01-16", "2026-02-18", null, "2025-06-18", "2024-02-18", "2024-02-18", "2024-01-09", null, "2023-12-19", null, "2023-12-18", "2024-01-19", "2023-12-18", "2025-06-18", "2023-12-13", null, "2023-12-11"],["2024-02-18", "2023-12-18"],[null, "2023-12-11", "2023-12-14", "2025-06-18", "2025-02-18", "2024-01-31", "2024-02-18", "2023-12-17", "2023-12-09", "2023-12-10", "2025-06-18", "2023-12-11", "2025-02-18", "2024-01-17", "2023-12-11", "2023-12-13", null, "2023-12-14", "2026-02-18", "2024-01-09", "2024-02-18", "2023-12-11", "2023-12-10", "2026-02-18", "2025-02-17", "2023-12-20", "2023-12-10", "2024-01-17", "2024-02-18", "2024-02-18", "2024-01-08", "2023-12-18", "2027-01-16", "2023-12-18", "2023-12-13", "2026-01-18", "2024-01-17", "2024-01-09", "2024-01-31", "2023-12-09", "2023-12-13", "2026-02-18", "2023-12-19", "2023-12-12", null, "2024-01-09", "2023-12-11", "2025-02-18", "2023-12-14", "2024-02-18", "2024-01-09", "2025-06-18", "2027-01-16", "2026-01-18", "2024-01-31", null, "2025-02-17", null, "2024-02-18", "2025-06-18", "2025-02-17", "2024-01-19", "2023-12-09", "2024-01-17", "2025-06-18", null, "2024-01-31", "2026-02-18", null, "2024-01-17", "2027-01-09", "2023-12-15", null, "2025-02-17", "2023-12-11", "2026-02-18", "2023-12-14", "2024-01-31", "2023-12-19", null],["2025-02-18", "2025-02-18", "2027-01-09", "2024-01-09", "2026-01-18", null, "2025-06-18", "2023-12-17", "2024-02-18", "2026-02-18", null, "2025-06-18", "2023-12-09", "2026-02-18", "2024-01-09", "2024-01-17", "2025-02-17", "2023-12-20", null, "2024-02-18", "2023-12-14", "2026-02-18", "2025-06-18", "2025-02-18", "2025-06-18", "2023-12-09", "2023-12-14", "2027-01-16", null, "2024-02-18", "2025-06-18", "2023-12-19", "2024-01-17", "2025-02-17", "2024-01-09"]),(16,8,6,-4,-4,30089,682261,3437,-13954,'2023-12-11',null,'2023-12-18','2023-12-16','j','q','g','m',null,null,'a','b',[5589761, null, -6141, -829684, null, -2464531, null],[-5777733, -8175496, null, null, -6160945, 1884632, -6941554, null, null, 7548929, 18625, 26231, -7353138, 4833709, -5089, 26486, -5840395, 28505, null, 17482, 7233510, -1629007, 1500069, -5038307, 21571, 31219, -1234923, -4758654, -24295, -1294544, -4644, -372141, -3536794, -5029586, -726654, 6911890, -2819474, 866330, -7881, -7890785, -3375667, -19409, null, 30313, -21264, 3274253, null, -3911042, 6247194, 13814],[1553786, null, null, -6903982, 22297, 2621822, null, null, 2092183, 508828, null, 1963331, -4981507, null, null, -8349, null, 331040, null, -17039, null, -8342455, 3978260, -4805305, -1010827, -4163171, -31921, 1994228, null, null, -6510208, -13536, 99231, 29612, -29057, null, null, -30540, null, null, -28386, 4776056, null, -5832862, -3467200, 11076, 458022, -16212, 1761, null, -20271, null, -1796947, -1901630, null, 2123642, -16292, -3228029, null, -7859248, null, -488, 61494, -7763832, -6262170, null, 1114541, null, null, -7147, null, null, null, null, -16138, 13011, null, null, -7710594, 7254450, 1806497, null, null, null],[7067452, null, -30604, 1449060, null, 2409450, 9369, -8147794, -27009, -3197541, 4534865, -1160, 2114, null, -407, null, 4703307, -9201, -4995270],['w', null, null, 'f', 'h', 'd'],['c', null],['y', 'n', null, null, 'c', 'w', 'm', null, 'e', 'y', 'c', 'l', null, 'w', null, null, 'a'],['w', 'f', null, 'a', 'c'],["2024-01-19", "2023-12-15", "2024-01-08", "2024-01-09", "2024-01-31"],["2024-02-18", "2025-02-17", "2023-12-13", null, "2023-12-14", "2023-12-14", "2025-02-18", "2024-01-09"],["2023-12-11", "2023-12-20", null, "2024-02-18", "2025-06-18", "2023-12-17", "2023-12-15"],["2024-01-19", "2023-12-18", "2024-01-08", "2024-01-19", null]),(17,-10,2,0,0,-3963084,-4213,4815,-1179164,'2023-12-10',null,'2023-12-17','2023-12-10','i','r','y','q','b','r','x','u',[4307633, -16947, null, -7862, -4232563, null],[-5496764, 3255696, 8195495, null, null, -5719, null, null, null, null, -7172947, null, 7157],[-5317, null],[-6848, -2821134, 1324574, 7026121, null, null, 1825837, -7975568, 29170, -788852, 6471854, null, -1420090, 15726, -27909, -1605263, -7104, -12889, -20339, -75192, null, -264319, -21528, null, 843, 6821, null, null, -5197035, null, 303660, null, -13114, -1545788, 7648700, null, 20672, -1063066, null, null, 3120252, 10972, null, null, null, 1011, 7256375, 22013, -18143],['f', 'm', null, 'v', 'v', 'e', null],['a', 'q', 'm', 'b'],['t', null, 'm', 'w', 't', 'p', 'k', 'r', null, null, 'f', null, null, 'v', 'm', 'w', 'z', 'c', 's', 'o', null, 'r', 'j', 'r', 'j', 'k', 'y', null, 'x', null, 'i', null, null, 't', 'o', null, null, 'i', 'n', 'j', 'r', 'h', 'p', null, 'i', 'o', 'n', 'a', 'w', 'o', 'f', 'g', 'b', 'm', 'm', 'a', null, 'v', null, 'f', 'u', 'x', 'k', null, 'm', null, 'y', 'o', 'p'],['s', 'z', 'k', 'z', 'u', 's', 'w', null],["2024-01-17", "2026-01-18", "2025-06-18", "2023-12-18", "2023-12-14", "2027-01-09", "2025-06-18", "2024-02-18", "2024-01-17", null, "2023-12-20", "2023-12-18", "2026-01-18", null, "2026-02-18", null, "2023-12-20", "2025-02-17", "2024-02-18", "2023-12-18"],["2024-01-31", "2024-02-18", "2023-12-18"],["2023-12-13", "2024-02-18", "2023-12-18", "2024-02-18"],["2026-02-18", "2023-12-10", "2024-02-18", "2023-12-17", "2025-02-17", "2023-12-19", "2025-02-17", "2023-12-15", "2023-12-18", null, null, null, "2024-01-17", "2023-12-17", "2024-02-18"]),(18,1,null,8,2,null,15665,-4275671,6028673,'2023-12-13',null,'2024-01-08','2023-12-11','e',null,'l','u','h','m','l','c',[-10318, -1263585, -6254186, 2654133, null, -6057511, null, null],[6098826, -8202533, 807153, 7776466, null, null, -17404, null, 5176, 19635, 1659787],[-611181, -3239393, 32010, 4591, 24397, -2884938],[null, -19540],[],['h', 'a', 'd', 't', 't', 'v', 'u', null, 'z', 's', 'k', 'g', 'w', null, 'p', 'l', 'q', null],[null],[null, 'm', 'j', 'w', 'j', 'h', 'c', 'a'],["2023-12-20", "2024-01-17", "2023-12-11", "2023-12-11", null, "2024-01-08", null],["2023-12-19", "2023-12-18", "2023-12-16", "2023-12-13", "2023-12-12", "2026-01-18", null, "2025-02-18", "2025-02-17", "2024-01-09", "2024-01-19", "2025-06-18", "2027-01-16", "2024-01-19", "2023-12-20", "2023-12-12", "2023-12-13", "2023-12-15", null, "2023-12-11", null, "2023-12-12", "2024-02-18", "2023-12-14", "2024-02-18", "2023-12-17", "2024-01-17", "2023-12-16", "2023-12-13", "2023-12-13", "2024-02-18", null, "2024-01-31"],["2025-02-17", "2023-12-15", "2023-12-17", "2024-01-19", null, null],["2023-12-13", "2026-01-18", "2023-12-14", "2024-01-09", "2027-01-16"]),(19,9,null,1,7,6154640,-7229,-8184918,-21039,'2023-12-20','2024-02-18','2023-12-20','2023-12-18','s','o','a','q','k','o','v','k',[-5182, 3195842, -5351075, -1231302, null, 844501, -5263, 20861, 19067, null, -20869, null, 4319584, null, null, null, -5756228, -30379, null, -528148, null, -31188, 8084, 7850, null, 7848296, null, -6726665, 2320612, 30713, -7784, -17238, 17215],[6314665, -473, null, 5473545, null, 22163],[822733, 18378, -10390],[-4771049, null, 16393, 2563103],['h', 'u', 'm', 'h', 'j', null, null, null, null, 'h', 'e', 'm', 'y', 'v', 'a', 'f', 'l', null, 'r', null, 'q', 'g', 'o', 'd', 'y', 'l', 'd', 'b', 'a', null, 'n', 'z', 'm', 'h', 'u', 'c', 'i', 'm', 'e', 'd', 'f', 'q', null, 'u'],['g', null, 'l'],['i', 'g', 'h', 'v', 'j', 'q', 'y', null, 'q', null, 'n', 'n', 'w', 't', 'i'],[null, 'j', null, 'l', 'v', 'i', 'n', 'l', 'e', null],["2024-01-17", "2023-12-09", "2025-02-17", "2025-06-18", "2023-12-16", null],["2023-12-17", null, "2025-06-18", "2024-01-17"],[null, "2023-12-19", null, null, null, "2023-12-14", "2023-12-11", null, "2023-12-14", "2023-12-18", "2025-06-18", "2023-12-19", "2025-02-18", "2024-01-31"],["2023-12-19", "2023-12-16", "2025-02-18", "2023-12-11", "2024-01-09", "2024-01-09", "2027-01-16"]),(20,8,null,-4,5,null,-7011009,4903342,2698351,'2024-02-18','2023-12-14','2023-12-19','2023-12-15','a',null,'z','m','d','c','e','g',[null, null, null],[32511, null, -3407, null, 1470715, -6492],[-1502, null, -6175, -32320, -14444, -7645258, null, null, -2461007],[],['d', 'o', 'b', 'p', 'u', 'g', 'i', 'h', 'q'],[null, 'x', 't', 'c', 'f', 'j'],['m', 'g', 'r', null],['i', null, 'a', 'y', 'e', 'x', 'g', null, 'j', 't', 'd', 'z'],["2024-01-09", "2025-02-17", "2025-06-18", "2025-06-18", "2026-01-18", "2023-12-18"],["2026-02-18"],[null, "2024-01-31", "2023-12-15", null, "2023-12-18", "2025-02-18", "2026-02-18", "2024-02-18", null, "2024-01-19", "2023-12-11", "2026-01-18", "2026-02-18", "2023-12-13", "2025-02-18", "2024-01-08", "2024-02-18", "2023-12-10", "2025-06-18", "2024-02-18", "2023-12-18", null, "2026-01-18", null, "2024-01-17", "2024-02-18", "2023-12-15", "2024-01-09", "2023-12-16", "2023-12-11", "2024-01-08", "2026-02-18", "2024-02-18", null, "2025-06-18", null, "2023-12-18", "2023-12-09", "2025-02-17", "2026-01-18", "2025-06-18", null, "2023-12-10", "2026-02-18", "2027-01-16", "2023-12-19", "2023-12-11", null, null, "2024-02-18", "2023-12-18", "2025-02-17", "2024-01-17", "2023-12-20"],["2023-12-10", "2023-12-12", "2023-12-19", "2023-12-12", "2026-01-18", "2023-12-12", "2023-12-13", "2023-12-09", "2024-01-31", "2024-01-17", "2027-01-09", "2024-02-18", null, "2025-02-18", "2026-01-18", "2026-01-18", "2025-06-18", "2023-12-09", "2027-01-09", "2026-01-18", "2023-12-17", "2023-12-17", "2023-12-19", "2024-02-18", "2023-12-14", "2024-01-19", "2024-01-08", "2023-12-11", "2023-12-19", "2026-01-18", "2023-12-17", null, "2026-01-18", "2027-01-09", "2023-12-20", "2024-01-09", "2023-12-16", "2023-12-20", null, null, "2026-01-18", "2023-12-15", null, null, "2023-12-11", "2023-12-20", "2026-01-18", "2024-02-18", "2023-12-17", null, "2025-06-18", "2024-02-18", "2023-12-12", "2025-06-18", null, "2023-12-19", "2023-12-14", "2025-06-18", "2024-01-19", "2023-12-14", "2024-01-31", "2023-12-16", "2023-12-14", "2023-12-13", "2025-06-18", "2026-02-18", "2023-12-17", "2023-12-09", "2025-06-18", "2027-01-16", "2024-02-18", "2024-02-18", "2027-01-09", "2023-12-20", "2026-01-18", "2024-01-17", "2023-12-19", "2024-01-17", "2025-02-17", "2025-02-17", "2024-01-17", "2023-12-15", "2024-01-17", null, "2023-12-18", "2024-01-08"]),(21,6,2,-4,8,-23565,-14066,17507,-14163,'2025-06-18','2025-02-17','2026-02-18','2024-02-18',null,'i','y','i','z','t','d','r',[-3792146, 23264, -5192962, null, -6847533, -30804, 3881259, 1707873, -4366, -2416413, null, null, 3425107, 1394876, 1385, 673, 2629308, 10899, -16112, -6722, 9906, 3936, -3956341, 7095173, 7670244, -1905, null, 27842, null, null, -4997295, -6041520, 29388, null, 7118891, -7387, 22531, null, null, -4325349, null, null, null, 6913556, -5885, null, -3199528, 210046],[24991, -3202548, null, 6355130, null, -3602045, 5821567, 13256, null, -19560, 26768, -5251, -2518385, null, -451737, -8351018, null, null, -3993, 6887, null, null, 5815493, 24050, -12003, -5364723, 4882394, 2077766, null, -725502, 3068508, null, 5143543, null, null, -4286258, -24989, null, 4166738, 6312, null, -511, -5141213, null, null, -7091378, null, null, null, null, -6840846, 210343, -7095563, -32068, null, -3629517, -5029, 25835, null, 991962, 5588154, -21357, -4061, -2894260, -19610, 5872764, 3129526, 7911490, null, 11556, -2086157, -25102, -7973928, -24334, 11499, -1762301, null, -1204046, null, 7908467, null, -77492, -8565, null],[2209760, null, 28627, -2346639, -5523462, -1791, -1880156],[],['i', null, 'f', 'd', 'x', 'k', 'h', 'x', 'e'],['b', 'o', 't', 'i', 'r', 'i', 't', 'i', 'm', 'p', 'e', 'y', null, 'z'],['t'],[],["2024-01-31", "2023-12-13", null, "2023-12-10", "2024-01-19", "2023-12-10", "2023-12-16"],["2024-02-18", "2026-01-18", "2023-12-15", null],[null, "2023-12-18", "2026-01-18", "2024-02-18", "2024-01-09"],["2023-12-12", "2024-02-18", "2023-12-19", "2023-12-16", null, "2024-01-08", "2023-12-15", null, "2023-12-13", "2024-02-18"]); """ - + sql """ set enable_common_expr_pushdown = true; """ qt_select_bug_2 """ SELECT table1.col_date_undef_signed_not_null_index_inverted, diff --git a/regression-test/suites/inverted_index_p0/test_index_rqg_bug3.groovy b/regression-test/suites/inverted_index_p0/test_index_rqg_bug3.groovy index eb49100eff..8fa4cdc4b8 100644 --- a/regression-test/suites/inverted_index_p0/test_index_rqg_bug3.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug3.groovy @@ -63,7 +63,7 @@ suite("test_index_rqg_bug3", "test_index_rqg_bug3"){ sql """ insert into ${table1}(pk,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_undef_signed_not_null_index_inverted,col_bigint_undef_signed,col_bigint_undef_signed_index_inverted,col_bigint_undef_signed_not_null,col_bigint_undef_signed_not_null_index_inverted,col_date_undef_signed,col_date_undef_signed_index_inverted,col_date_undef_signed_not_null,col_date_undef_signed_not_null_index_inverted,col_varchar_10__undef_signed,col_varchar_10__undef_signed_index_inverted,col_varchar_10__undef_signed_not_null,col_varchar_10__undef_signed_not_null_index_inverted,col_varchar_1024__undef_signed,col_varchar_1024__undef_signed_index_inverted,col_varchar_1024__undef_signed_not_null,col_varchar_1024__undef_signed_not_null_index_inverted) values (0,null,null,6,-10,686522353588051391,null,-4348126053228614825,-6032791270525051561,'2025-06-18','2023-12-20','2024-02-18','2024-02-18',null,'o','o','x',null,null,'q','o'),(1,-4,-4,2,-4,-2326501514208488669,-8144130583099259882,6094773265621719520,-4389392644235126205,'2024-01-09','2023-12-19','2024-01-31','2024-02-18',null,null,'c','n',null,'k','p','y'),(2,-10,-4,-4,-10,-5761419801766661970,null,1575077084160592390,-6748295140918895470,'2023-12-09','2024-01-31','2023-12-11','2024-01-08','j',null,'m','k','p','s','s','u'),(3,2,null,3,6,7058901979703960583,-8791856880716210018,9136811854525385821,-584135196107353733,'2024-02-18','2023-12-14','2024-01-19','2024-02-18','e','h','i','b','x','a','h','d'),(4,6,-4,4,0,2965210993977402598,null,1623398766611349339,-719530465607559880,'2024-02-18','2024-01-31','2023-12-12','2027-01-16','c','p','h','o','t','l','t','k'),(5,-4,-10,7,-4,null,-4312428052005262901,1664741259066823538,-6520957605791451399,'2024-02-18','2024-01-19','2027-01-09','2024-01-31','f','d','q','k',null,'u','n','x'),(6,-10,0,4,-4,-4719935591389099252,-8933690584562378263,1354434296669241202,2684538782485650790,'2023-12-16','2026-01-18','2024-01-08','2025-06-18',null,null,'f','x','x','v','i','m'),(7,0,-10,-10,-10,-5480618991908294867,null,5334008298577672775,7173424974650158839,'2024-01-09',null,'2023-12-09','2024-02-18','y','k','x','u',null,null,'a','b'),(8,6,-4,5,8,-7188890154699493125,-1925845279956226794,-5657889190097714482,1640041513228273840,'2027-01-16',null,'2025-02-18','2023-12-20','y','x','p','w','d','j','k','d'),(9,null,-4,1,3,-2080159247468648985,-1306911382131817506,1219720796746600015,-978348523441835274,'2024-02-18','2025-06-18','2025-06-18','2025-02-18',null,'i','y','s','c',null,'m','o'),(10,2,1,-10,0,-7569151440408756108,null,1393963672572074196,-3822268135049166532,'2024-01-08','2027-01-16','2023-12-13','2024-02-18','s',null,'q','z',null,'u','b','v'),(11,9,8,-10,7,-4419333711868488787,5670069993430301249,-5101280938951897282,7291919080934920934,'2027-01-09','2025-02-18','2024-01-17','2026-01-18','u','c','d','r',null,'m','r','p'),(12,9,8,7,9,-697217673128595873,-2415793798160514654,-1909943068043737865,5844073448689265407,'2024-01-17','2025-02-17','2023-12-17','2023-12-14','a','z','v','o','q','u','z','h'),(13,-10,6,-10,1,null,null,-6933219731543784592,-4745778751501231119,'2023-12-20',null,'2026-01-18','2026-01-18',null,'r','w','c','k',null,'t','e'),(14,6,4,-10,-10,null,377907223978572611,-7447442174599113505,4949011330273779695,'2023-12-17','2024-02-18','2026-01-18','2024-01-08','g','w','u','k',null,'m','g','d'),(15,null,-4,-10,3,-5441857898369120483,-2001300041828347883,4385022502994073333,6762545521805735020,'2024-01-17','2023-12-11','2023-12-15','2023-12-10','p','u','p','i','i','c','j','v'),(16,8,1,-10,3,7499177078109776887,8002215544264694167,-4914597203639379766,7611185654075676834,null,null,'2024-01-08','2023-12-17','e','h','q','t',null,null,'p','d'),(17,-4,-10,3,6,null,null,4596273190394276006,-3248366019937329149,'2024-01-09','2023-12-19','2023-12-20','2023-12-11',null,null,'i','f',null,'t','q','a'),(18,0,-4,-10,1,null,-2000849949571150330,7208571222986345744,2598345170237057509,'2024-01-09','2023-12-09','2024-02-18','2024-01-19','a','f','w','o','b','y','q','f'),(19,7,7,-4,0,null,5717592572856392823,-8128856226419623044,-7534868883394863810,'2023-12-20','2023-12-09','2023-12-10','2023-12-12','r','t','y','d','l','a','y','v'),(20,2,null,0,2,null,-6905288165492491017,1934258578152616096,-1388806210542225140,null,null,'2024-01-19','2026-01-18','p',null,'p','g','u','b','i','c'),(21,null,null,-10,-4,1698759627767041241,null,-6613269394014189122,1915677852069340594,'2023-12-18','2024-01-31','2025-02-18','2024-01-08','e','g','l','h',null,null,'v','n'),(22,-10,-10,-4,5,-3720952595350369266,1539673860923570193,5089313038468606351,262016952853919148,'2023-12-15','2025-02-18','2024-01-19','2024-01-08','d','g','d','e','l',null,'m','g'),(23,-10,1,-10,7,null,-4884323809040291936,-4428424779275301738,-3325468851678420401,'2023-12-19','2027-01-16','2026-02-18','2024-01-09','z','v','v','v','r','d','j','y'),(24,4,8,3,-10,1026316126533561197,-8966784351064986909,496857885215447340,-6148636280121789215,'2024-02-18','2025-06-18','2026-02-18','2025-02-18','f',null,'j','k',null,'s','i','q'),(25,5,5,4,6,8574091287090543865,null,-773937635554104337,6026917236758217609,'2026-02-18','2027-01-09','2023-12-12','2024-01-19','s','p','t','d','t','l','u','m'),(26,0,-10,-10,9,-2429694321063869458,null,8908961259233183763,6894623222255264210,'2024-01-17','2023-12-14','2023-12-11','2023-12-09','w','o','l','g','m','r','h','i'),(27,5,0,2,-10,7748161344545453064,null,3244053576839674045,-7948008233666340932,'2024-02-18','2023-12-20','2024-01-08','2023-12-14','n','a','r','q','c','y','q','u'),(28,4,-4,5,4,null,2204997326584988589,7997961660331289189,8763906081360257030,'2025-06-18','2023-12-18','2023-12-16','2023-12-16','k','i','d','t','y','c','o','a'),(29,0,5,-4,4,null,null,7562098503580472041,929168144988769048,'2026-01-18','2023-12-11','2023-12-10','2024-01-31','p','d','j','j','j','h','f','p'),(30,5,-10,-4,4,3945007524910600918,null,-8466778503120120353,-9169615505661358433,'2023-12-13','2024-01-19','2023-12-16','2023-12-10',null,'p','g','d','e','e','r','u'),(31,5,4,8,6,-7544567449016691208,-7026985677386241453,-2698203866546802012,-8383194363413620107,'2024-01-09','2027-01-09','2025-02-18','2025-02-18','f','t','g','n','r','i','p','o'),(32,-4,null,7,-10,-5468978947503379654,-5676001133436456624,-5328902013300281884,2338117992866157501,'2023-12-20','2023-12-15','2023-12-20','2024-01-08','f','z','y','t','j','c','e','x'),(33,-10,6,-10,-10,6715916167220457165,-3864032264700366706,7115861918737510196,-937991761308321600,'2025-02-18','2024-02-18','2023-12-19','2024-02-18','x',null,'t','x','h','o','p','v'),(34,-4,8,9,-10,-4718823602932239136,-3633212199616285968,-5190227402771860745,5545611345261203982,'2024-01-08','2026-01-18','2024-02-18','2023-12-16','f','t','n','h',null,'y','e','u'),(35,-4,3,-10,8,null,7722389449895645140,-4965839022248115530,6494405496923526511,'2023-12-10','2024-02-18','2026-02-18','2024-01-09','u','t','a','t','w',null,'h','w'),(36,5,6,-10,0,null,null,84960662585385706,2611830596442233539,null,'2026-01-18','2023-12-15','2026-01-18','b','t','p','b','g','g','z','k'),(37,-10,-10,3,-10,null,-5462312886211329186,-2793882411087244594,7564921654378090880,'2025-06-18','2027-01-09','2027-01-16','2023-12-09','n','k','l','z','y','i','o','c'),(38,4,null,-10,3,null,2065313248357950073,2398686393322288278,-5793325226082177083,'2023-12-14','2024-01-17','2023-12-12','2024-01-31','m',null,'n','c','g','f','r','m'),(39,5,1,9,0,-2901110266746457515,-7419676417711330947,5568223068212783910,-8443206843568552423,'2023-12-20','2023-12-15','2024-02-18','2024-01-17','j',null,'x','m','c','u','j','a'),(40,-4,5,-4,-4,3686987810771014559,4528672918224579415,-531153650185309112,-4795413900154192584,'2023-12-12','2024-01-19','2024-01-31','2024-01-19','m','o','k','p','v','s','f','c'),(41,null,6,-10,-10,1371451390800312645,-945321182848207621,-8418988114521301883,-8987180461079691062,'2024-01-09','2023-12-10','2023-12-19','2023-12-12','s','i','x','u','h','e','q','y'),(42,-10,null,2,1,null,-2863490765313461654,3110048825870954129,-2547950560699735251,'2025-06-18','2024-01-08','2023-12-10','2023-12-10','d','y','d','h','t','o','t','w'),(43,3,0,4,1,-7282895869404970499,5532011705197458854,-4502369753730677912,-3032934141238479600,'2023-12-18','2024-02-18','2023-12-19','2026-02-18','w',null,'m','n','g',null,'j','q'),(44,null,null,2,9,5430716729257430348,null,-8208477558683957238,-7953995265596299120,'2023-12-18','2023-12-18','2023-12-11','2025-06-18','w','w','a','u','k','k','j','q'),(45,6,-4,-10,1,-8903356732633014005,null,2532821444113211740,-2346292639048659545,'2024-01-08','2023-12-12','2025-02-18','2023-12-19','v','b','k','e','i','q','h','l'),(46,7,-10,-10,-10,null,-6646527298990960109,-7898216427196445987,-1558528416630681469,null,'2027-01-09','2024-02-18','2025-02-17','k','d','o','n','h','g','x','p'),(47,3,6,3,7,-6864291355591117572,4024432796468900765,-6272917113481022986,-1984131539617763529,'2024-01-17','2024-01-17','2025-06-18','2025-06-18','c','p','t','y','i','c','y','i'),(48,9,null,1,-4,null,-7244007199905240117,8657019614874868097,-492287318340969091,'2024-02-18','2023-12-11','2024-01-09','2027-01-09',null,null,'m','i','t','k','r','x'),(49,1,null,1,5,8407263822602373073,-3275760834800206047,-2117832965174816037,5807219087033669504,'2023-12-13','2024-01-19','2023-12-16','2024-02-18','r','n','n','o','r','g','j','q'); """ - + sql """ set enable_common_expr_pushdown = true; """ qt_select_bug_1 """ SELECT col_int_undef_signed_not_null, col_date_undef_signed_not_null FROM ${table1} AS table1 diff --git a/regression-test/suites/inverted_index_p0/test_index_rqg_bug4.groovy b/regression-test/suites/inverted_index_p0/test_index_rqg_bug4.groovy index 9353616f95..b29532c12c 100644 --- a/regression-test/suites/inverted_index_p0/test_index_rqg_bug4.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug4.groovy @@ -126,6 +126,7 @@ suite("test_index_rqg_bug4", "test_index_rqg_bug"){ try { sql "sync" + sql """ set enable_common_expr_pushdown = true; """ qt_sql """ select diff --git a/regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy b/regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy index 7021391093..cb3bbe65e2 100644 --- a/regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy +++ b/regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy @@ -21,7 +21,7 @@ suite("test_index_skip_read_data", "p0"){ def indexTbName2 = "test_index_skip_read_data_mow" def indexTbName3 = "test_index_skip_read_data_mor" - + sql """ set enable_common_expr_pushdown = true; """ // dup sql "DROP TABLE IF EXISTS ${indexTbName1}" diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index.groovy index 280782f75c..ff719eed03 100644 --- a/regression-test/suites/inverted_index_p0/test_inverted_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_inverted_index.groovy @@ -21,6 +21,7 @@ suite("test_inverted_index", "inverted_index") { def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ return jobStateResult[0][9] } + sql """ set enable_common_expr_pushdown = true; """ sql "DROP TABLE IF EXISTS ${tbName1}" sql """ CREATE TABLE IF NOT EXISTS ${tbName1} ( diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index_keyword.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index_keyword.groovy index 1663de8e7c..fbc584d7dd 100644 --- a/regression-test/suites/inverted_index_p0/test_inverted_index_keyword.groovy +++ b/regression-test/suites/inverted_index_p0/test_inverted_index_keyword.groovy @@ -68,7 +68,7 @@ suite("test_inverted_index_keyword"){ (3, '我在北京市'), (3, '我在西安市') """ - + sql """ set enable_common_expr_pushdown = true; """ qt_sql "SELECT * FROM ${indexTblName} where c match '330204195805121025'"; qt_sql "SELECT * FROM ${indexTblName} where c match '36'"; qt_sql "SELECT * FROM ${indexTblName} where c match '330225197806187713'"; diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy index 1905067d43..0f9ca2f31e 100644 --- a/regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy +++ b/regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy @@ -48,7 +48,7 @@ suite("test_inverted_index_mor", "p0"){ sql """ INSERT INTO $indexTblName VALUES (1, 2, 12, 1.2, '1 2'), (3, 4, 34, 3.4, '3 4'); """ sql """ INSERT INTO $indexTblName VALUES (11, 12, 1112, 11.12, '11 22'), (13, 14, 1314, 13.14, '13 14'); """ - + sql """ set enable_common_expr_pushdown = true; """ // original data qt_11 """ SELECT * FROM $indexTblName ORDER BY k1,k2 """ diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index_null.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index_null.groovy index 43cf22d572..6dac5d3e44 100644 --- a/regression-test/suites/inverted_index_p0/test_inverted_index_null.groovy +++ b/regression-test/suites/inverted_index_p0/test_inverted_index_null.groovy @@ -53,6 +53,7 @@ suite("test_inverted_index_null") { (7,'tengxun','qie','addr gg','lj',null), (8,'tengxun2','qie',null,'lj',800) """ + sql """ set enable_common_expr_pushdown = true """ // select all data qt_select_0 "SELECT * FROM ${table1} ORDER BY id" diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index_null_ram_dir.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index_null_ram_dir.groovy index 6059db2062..784c231641 100644 --- a/regression-test/suites/inverted_index_p0/test_inverted_index_null_ram_dir.groovy +++ b/regression-test/suites/inverted_index_p0/test_inverted_index_null_ram_dir.groovy @@ -105,6 +105,7 @@ suite("test_inverted_index_null_ram_dir") { (7,'tengxun','qie','addr gg','lj',null), (8,'tengxun2','qie',null,'lj',800) """ + sql """ set enable_common_expr_pushdown = true """ // select all data qt_select_0 "SELECT * FROM ${tableName} ORDER BY id" diff --git a/regression-test/suites/inverted_index_p0/test_lowercase.groovy b/regression-test/suites/inverted_index_p0/test_lowercase.groovy index 0670e05594..3dacec72f4 100644 --- a/regression-test/suites/inverted_index_p0/test_lowercase.groovy +++ b/regression-test/suites/inverted_index_p0/test_lowercase.groovy @@ -56,6 +56,7 @@ suite("test_lowercase"){ "replication_allocation" = "tag.location.default: 1" ); """ + sql """ set enable_common_expr_pushdown = true """ sql "INSERT INTO $indexTblName2 VALUES (1, 'hello 我来到北京清华大学'), (2, 'HELLO 我爱你中国'), (3, 'Hello 人民可以得到更多实惠');" qt_sql "SELECT * FROM $indexTblName2 WHERE c MATCH 'hello' ORDER BY id"; diff --git a/regression-test/suites/inverted_index_p0/test_match_query_without_index.groovy b/regression-test/suites/inverted_index_p0/test_match_query_without_index.groovy index 3d255f2762..61a564c51d 100644 --- a/regression-test/suites/inverted_index_p0/test_match_query_without_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_match_query_without_index.groovy @@ -59,7 +59,7 @@ suite("test_match_query_without_index", "inverted_index_select"){ ("san zhang", "grade 5", "", "", "", 10, "2017-10-01", "tall:100cm, weight: 30kg, hobbies:", "", ""), ("li sisi", "grade 6", "li ba", "li liuliu", "", 11, "2016-10-01", "tall:150cm, weight: 40kg, hobbies: sing, dancing, running", "good at handiwork and beaty", "") """ - + sql """ set enable_common_expr_pushdown = true """ // case1: match any try { sql """ select * from ${indexTbName1} where ${varchar_colume1} match_any "" order by name; """ diff --git a/regression-test/suites/inverted_index_p0/test_match_without_index.groovy b/regression-test/suites/inverted_index_p0/test_match_without_index.groovy index 94cee179ed..a008cf703d 100644 --- a/regression-test/suites/inverted_index_p0/test_match_without_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_match_without_index.groovy @@ -39,6 +39,7 @@ suite("test_match_without_index", "p0") { "replication_allocation" = "tag.location.default: 1" ); """ + sql """ set enable_common_expr_pushdown = true """ sql """ INSERT INTO ${testTable} VALUES (123, '17.0.0.0', 'HTTP GET', '200', 20); """ sql """ INSERT INTO ${testTable} VALUES (123, '17.0.0.0', 'Life is like a box of chocolates, you never know what you are going to get.', '200', 20); """ diff --git a/regression-test/suites/inverted_index_p0/test_need_read_data.groovy b/regression-test/suites/inverted_index_p0/test_need_read_data.groovy index 0e4f0bf9a4..133888b47d 100644 --- a/regression-test/suites/inverted_index_p0/test_need_read_data.groovy +++ b/regression-test/suites/inverted_index_p0/test_need_read_data.groovy @@ -73,6 +73,7 @@ suite("test_need_read_data", "p0"){ sql """ insert into ${indexTbName1}(pk,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_undef_signed_not_null_index_inverted,col_bigint_undef_signed,col_bigint_undef_signed_index_inverted,col_bigint_undef_signed_not_null,col_bigint_undef_signed_not_null_index_inverted,col_date_undef_signed,col_date_undef_signed_index_inverted,col_date_undef_signed_not_null,col_date_undef_signed_not_null_index_inverted,col_varchar_10__undef_signed,col_varchar_10__undef_signed_index_inverted,col_varchar_10__undef_signed_not_null,col_varchar_10__undef_signed_not_null_index_inverted,col_varchar_1024__undef_signed,col_varchar_1024__undef_signed_index_inverted,col_varchar_1024__undef_signed_not_null,col_varchar_1024__undef_signed_not_null_index_inverted) values (0,null,4,4,-10,-2806067520539703493,-6601893530499828925,-7112891289289392099,-4883682982770875774,'2024-02-18','2025-02-18','2026-02-18','2025-06-18','s',null,'p','h','m','v','p','y'),(1,null,null,-4,5,null,5288406618687277699,2255222069393083918,-7885696364844049115,null,'2023-12-12','2025-02-18','2025-02-17','r','u','f','q','n','x','q','b'),(2,1,null,3,-10,4359919727878435225,3857327739249958938,-1608704987438726460,3650073478443923028,'2025-06-18','2024-02-18','2023-12-11','2023-12-13','s','p','x','u','e','k','l','c'),(3,4,-4,-4,9,null,-2952515504097288835,-5248236183608833050,-959402810989602748,null,'2024-01-09','2023-12-12','2023-12-11','i','f','r','m','y','j','z','h'),(4,3,-10,-10,-10,7595845588678689951,1807745217759831665,-3269384278721380385,-2626336667443528939,'2024-01-31','2023-12-13','2024-02-18','2023-12-15','c','b','q','a','k',null,'c','q'),(5,2,3,-10,1,4845076071359725577,-2081520813928284051,7965382457542866166,-1263489703824567045,'2023-12-20','2025-02-18','2025-02-17','2023-12-12',null,'p','v','k','s',null,'c','y'),(6,-4,7,-4,1,null,-312662439708631344,5986577765852616616,4340850184105875353,'2023-12-16','2024-01-17','2024-01-19','2026-02-18','s','q','w','z','m','a','v','z'),(7,5,-10,5,4,2838843073924070168,1039278568099501661,-2970064881715473209,-3546456947974778751,'2023-12-15','2024-01-31','2023-12-18','2024-01-08','m','t','t','y','y',null,'l','l'),(8,null,1,7,4,8007492376294713910,994564705132157298,-4772736062894230873,7998340640624222727,'2023-12-13','2027-01-09','2024-02-18','2024-01-19','t',null,'p','n','j','l','g','c'),(9,8,5,2,0,-3365354435906005141,6104736481079857939,-9027968242311553543,8197347709168535456,'2023-12-20','2024-02-18','2023-12-10','2024-02-18','t',null,'p','h','b','e','w','a'),(10,4,5,9,3,null,-4270500778304447716,7975155833293678874,-5070901586921449703,'2024-01-19','2024-01-19','2027-01-09','2024-02-18','d','b','o','p','b','r','f','q'),(11,5,-4,3,-4,9029358127154882937,-6951089794937682659,2630099804275038168,-5951780849025883849,null,null,'2024-01-17','2023-12-18','k','b','s','t','z','w','r','y'),(12,3,-4,1,-4,309499821905665998,null,9039699521215326765,1922704068231658563,'2023-12-17','2024-02-18','2025-06-18','2027-01-09','m','z','l','l','c',null,'h','a'),(13,4,null,7,1,-8088552747885790045,null,4290463296755479391,-143636852574500943,'2023-12-15','2024-02-18','2023-12-11','2023-12-16',null,'o','j','r',null,'l','p','z'),(14,null,3,8,-4,-2114077344204837001,null,-2621506620475509311,3516417267200208461,'2023-12-20','2024-01-09','2023-12-18','2024-01-31','q','d','u','y','k','o','l','o'),(15,5,0,3,1,null,469795947256142816,-3769981987932056248,-5475689888751280017,'2025-06-18',null,'2024-02-18','2025-06-18','y','c','r','o','y','p','o','x'),(16,-4,5,6,2,-5508267849003968462,3862598879377665395,8797778903170766625,383525525986717298,'2024-02-18','2023-12-11','2024-01-17','2023-12-17','s','c','m','c',null,null,'s','o'),(17,-4,6,6,0,-1057012716454277307,-8435107454217223314,-120847043370486063,-7398709528414815558,'2027-01-16','2024-02-18','2025-02-17','2024-01-17',null,'f','u','d','l','a','h','r'),(18,2,6,1,-10,null,339922889780584305,8187645881248196657,9163411866066071700,'2023-12-14','2024-01-08','2024-01-31','2023-12-13','u',null,'y','a','v','v','u','z'),(19,8,-4,9,-10,null,null,-5043486900779665508,6743373354123723422,'2023-12-19','2023-12-14','2024-01-17','2023-12-14','u','t','z','o','b','n','y','t'),(20,-4,7,-4,0,-14388658393529797,null,4788467486005193064,545093836722695629,'2025-06-18','2024-02-18','2027-01-09','2023-12-09','u','b','u','r','e',null,'f','o'),(21,-4,-4,4,8,2245289792981551579,-4485687312935149586,-6812240256635951381,8434108874554436761,'2023-12-09','2023-12-16','2026-02-18','2023-12-09',null,'k','v','q','r',null,'z','e'),(22,-4,3,3,1,-1218928283971104430,null,4325982422669856960,-3538203895099667724,'2027-01-16',null,'2024-02-18','2023-12-19',null,null,'v','e','u','n','q','h'),(23,-4,0,7,7,null,null,-8073893843395705054,-5272967641252019267,'2024-01-17','2027-01-09','2024-01-19','2023-12-14','n','q','o','t','e',null,'w','u'),(24,-10,3,-10,5,null,-8586767268780407575,3995596016735621711,429810224315638782,'2025-06-18','2026-02-18','2023-12-18','2023-12-14','m','p','k','e',null,'l','m','j'),(25,9,null,8,9,-7406209357285295571,8508527424784069010,-6700207760307684345,-6778419266053660893,'2024-01-31','2023-12-15','2025-06-18','2023-12-20',null,null,'t','f','b','k','e','b'),(26,-4,7,-10,1,549615970930886740,-8537815265488791461,9105177114666776852,808842453982447397,'2023-12-14','2026-02-18','2025-06-18','2023-12-18','z','g','v','b','m','s','b','r'),(27,6,8,-4,0,2363364623910847992,2435555377853278383,-7059658388492780701,1808647950247830538,'2023-12-14','2023-12-14','2027-01-09','2025-06-18',null,null,'o','q','i','g','i','u'),(28,-4,-4,0,-10,2837109330712127287,3258333757532106070,-5486400602525569363,7454355268455905775,'2023-12-18','2024-01-09','2025-06-18','2024-01-17',null,'e','h','j',null,'u','a','y'),(29,7,-4,5,-4,-3944687198069417525,7676519542662933720,6921050061010821518,8686366976901132291,'2026-01-18','2024-01-17','2024-02-18','2024-01-08',null,'a','d','y','i','l','r','h'),(30,-4,null,4,7,null,3744030733185485721,-6094790559118439736,-5022434353993604500,'2023-12-09','2023-12-20','2025-06-18','2023-12-14','u','a','o','m','c','n','o','z'),(31,1,4,2,9,-6750407042384384034,-2505324657561681875,5301019286095013646,3139887746721886789,null,'2023-12-15','2024-01-08','2023-12-17','q','q','u','x','b','m','z','p'),(32,-4,null,9,9,null,-1729390457612771468,6708980039726745192,-7293963935049205901,'2024-02-18','2023-12-17','2023-12-13','2023-12-11','d',null,'v','m','e','f','w','y'),(33,4,5,-10,9,3535708249298387360,-8975616986595863107,-862904611806478134,-3482957162935231780,'2024-02-18','2023-12-20','2023-12-19','2025-02-18','b','h','c','v','s','b','e','e'),(34,-4,null,-4,0,null,-2891436930426307984,-1704292540467538099,-8564095007543756456,'2023-12-13','2027-01-09','2025-02-18','2026-02-18',null,null,'f','t','w','s','z','h'),(35,-10,5,8,8,-5214808208417488395,null,-7973404656651152899,-7504002864628497355,'2023-12-19','2023-12-10','2025-02-18','2023-12-12','v','x','f','t','q','x','t','n'),(36,5,5,6,-10,null,-5034216029789478132,7055943879035021136,3694646370699507306,'2027-01-09','2024-01-19','2025-06-18','2025-06-18',null,'s','t','d',null,'w','r','i'),(37,-4,-4,4,8,-8353588562907627230,-4962760661862938335,-6383251065296502747,-3956983090211228295,'2023-12-11','2024-01-08','2023-12-16','2024-02-18','u','j','w','g','w','r','c','f'),(38,-10,-10,-4,-4,null,null,247919645069562206,8288875655774835875,'2025-02-17','2026-01-18','2023-12-11','2026-02-18','u','m','y','a','k',null,'d','g'),(39,2,9,4,0,-202685904718522032,5896989546064098495,5695022708672839468,-7757738794882960533,'2027-01-09','2023-12-15','2027-01-16','2023-12-16','c','u','l','z',null,'d','x','h'),(40,null,null,7,7,4835497339758608794,-5034982165921306189,548692450913004773,-5189451515582531492,'2024-01-19','2024-01-19','2024-02-18','2023-12-11','w',null,'m','b','w','u','t','u'),(41,5,0,6,3,-4294233837078291532,null,-4951306529377186523,-6918657392711546234,'2023-12-15','2023-12-15','2023-12-09','2024-01-09',null,'p','c','b','a','i','j','n'),(42,null,-4,-10,9,-1890221339559069903,-9174324348086863583,-7768565663714277414,1583589820059033801,'2024-01-19','2024-01-31','2026-02-18','2024-02-18','i','x','z','z',null,'s','n','h'),(43,null,8,4,6,null,-3040322546432413315,-7308614762246284846,-7065674754425766028,'2023-12-20','2024-02-18','2025-02-17','2024-02-18','z','z','l','l','a','j','p','f'),(44,8,-10,8,7,3533075847821605226,-8953779906693032030,-4566442548311033242,-6944000050952871345,'2025-02-18','2023-12-19','2023-12-16','2024-01-31','i','h','n','u','j','y','r','g'),(45,8,-4,6,-4,null,null,-6687031592408843506,-2644229079223766810,'2023-12-12','2025-02-18','2023-12-12','2024-01-09',null,'y','j','b','p','l','c','c'),(46,4,-4,-10,-4,4827922355005130144,2170052469942214303,-1634163889160521502,2246460621190812016,'2023-12-15','2023-12-11','2024-01-08','2025-02-18',null,null,'t','x','n','i','e','l'),(47,null,4,8,7,null,-7852983538684806346,-8869292859054504183,-7739200759355598281,null,'2023-12-17','2025-06-18','2024-01-17','i','r','m','s',null,'m','j','r'),(48,-10,7,9,0,-4878175313227689694,6741975426332123319,-384904760880798675,3671673134248536674,null,'2023-12-15','2023-12-09','2025-02-18','w','y','w','b','z','s','l','g'),(49,3,-4,8,-10,1403956186045671649,null,6070888386247696636,6864331793839321007,null,'2024-01-08','2027-01-16','2026-01-18','p','z','b','i','t','u','q','f'); """ sql "sync" + sql """ set enable_common_expr_pushdown = true """ qt_sql """ SELECT COUNT( *) AS field1, col_int_undef_signed AS field2 FROM ${indexTbName1} WHERE( col_date_undef_signed_not_null_index_inverted == '2024-01-01' OR day( col_date_undef_signed_not_null_index_inverted ) !=0 ) GROUP BY field2 ORDER BY field2; """ diff --git a/regression-test/suites/inverted_index_p0/test_null_index.groovy b/regression-test/suites/inverted_index_p0/test_null_index.groovy index 797200ab01..61b613470e 100644 --- a/regression-test/suites/inverted_index_p0/test_null_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_null_index.groovy @@ -44,7 +44,7 @@ suite("test_null_index", "p0"){ "replication_allocation" = "tag.location.default: 1" ); """ - + sql """ set enable_common_expr_pushdown = true """ sql "INSERT INTO $indexTblName VALUES (1, 'a', null, [null], [1]), (2, 'b', 'b', ['b'], [2]), (3, 'c', 'c', ['c'], [3]);" qt_sql "SELECT * FROM $indexTblName WHERE str match null order by id;" qt_sql "SELECT * FROM $indexTblName WHERE str_null match null order by id;" diff --git a/regression-test/suites/inverted_index_p0/test_or_not_match.groovy b/regression-test/suites/inverted_index_p0/test_or_not_match.groovy index 95af26480c..488639aae6 100644 --- a/regression-test/suites/inverted_index_p0/test_or_not_match.groovy +++ b/regression-test/suites/inverted_index_p0/test_or_not_match.groovy @@ -64,6 +64,7 @@ suite("test_or_not_match", "p0") { sql "set enable_nereids_planner = true" sql "set enable_fallback_to_original_planner = false" + sql """ set enable_common_expr_pushdown = true """ qt_sql "select request from ${tableName} where request like '1.0' or not request MATCH 'GETA' order by request limit 2;" } diff --git a/regression-test/suites/inverted_index_p0/test_pk_no_need_read_data.groovy b/regression-test/suites/inverted_index_p0/test_pk_no_need_read_data.groovy index 4aa969debd..b8cf3f4c0b 100644 --- a/regression-test/suites/inverted_index_p0/test_pk_no_need_read_data.groovy +++ b/regression-test/suites/inverted_index_p0/test_pk_no_need_read_data.groovy @@ -54,6 +54,7 @@ suite("test_pk_no_need_read_data", "p0"){ // case1: enable count on index sql "set enable_count_on_index_pushdown = true" + sql """ set enable_common_expr_pushdown = true """ qt_select_0 "SELECT COUNT() FROM ${table1} WHERE date='2017-10-01'" qt_select_1 "SELECT COUNT() FROM ${table1} WHERE year(date)='2017'" diff --git a/regression-test/suites/inverted_index_p0/test_stopwords.groovy b/regression-test/suites/inverted_index_p0/test_stopwords.groovy index 4f7c577dc5..f9f8c8339a 100644 --- a/regression-test/suites/inverted_index_p0/test_stopwords.groovy +++ b/regression-test/suites/inverted_index_p0/test_stopwords.groovy @@ -42,6 +42,7 @@ suite("test_stopwords", "p0"){ try { sql "sync" + sql """ set enable_common_expr_pushdown = true """ qt_sql """ select * from ${indexTbName} where b match 'a'; """ qt_sql """ select * from ${indexTbName} where b match 'are'; """ diff --git a/regression-test/suites/inverted_index_p0/topn_clear_block.groovy b/regression-test/suites/inverted_index_p0/topn_clear_block.groovy index 586173e0f3..78d541bcb4 100644 --- a/regression-test/suites/inverted_index_p0/topn_clear_block.groovy +++ b/regression-test/suites/inverted_index_p0/topn_clear_block.groovy @@ -40,6 +40,7 @@ suite("test_clear_block") { } sql """ set enable_match_without_inverted_index = false; """ + sql """ set enable_common_expr_pushdown = true """ // sql """ set def dupTableName = "dup_httplogs" sql """ drop table if exists ${dupTableName} """ diff --git a/regression-test/suites/inverted_index_p0/unique_with_mow/test_mow_with_null_sequence.groovy b/regression-test/suites/inverted_index_p0/unique_with_mow/test_mow_with_null_sequence.groovy index 92615e44d4..2da3509f1d 100644 --- a/regression-test/suites/inverted_index_p0/unique_with_mow/test_mow_with_null_sequence.groovy +++ b/regression-test/suites/inverted_index_p0/unique_with_mow/test_mow_with_null_sequence.groovy @@ -38,6 +38,7 @@ suite("test_mow_with_null_sequence", "inverted_index") { ); """ + sql """ set enable_common_expr_pushdown = true """ sql """ insert into $tableName values('a', 'zhang san', 'address1', NULL) """ sql """ insert into $tableName values('a', 'zhang si', 'address2', '2022-10-20') """ diff --git a/regression-test/suites/inverted_index_p0/unique_with_mow/test_pk_uk_case.groovy b/regression-test/suites/inverted_index_p0/unique_with_mow/test_pk_uk_case.groovy index 309a458dd0..e433ccaa06 100644 --- a/regression-test/suites/inverted_index_p0/unique_with_mow/test_pk_uk_case.groovy +++ b/regression-test/suites/inverted_index_p0/unique_with_mow/test_pk_uk_case.groovy @@ -192,6 +192,7 @@ suite("test_pk_uk_case_unique_with_mow", "inverted_index") { """ sql "sync" + sql """ set enable_common_expr_pushdown = true """ // count(*) def result0 = sql """ SELECT count(*) FROM ${tableNamePk}; """ diff --git a/regression-test/suites/inverted_index_p0/unique_with_mow/test_primary_key_simple_case.groovy b/regression-test/suites/inverted_index_p0/unique_with_mow/test_primary_key_simple_case.groovy index b0ffda2975..8884ff837a 100644 --- a/regression-test/suites/inverted_index_p0/unique_with_mow/test_primary_key_simple_case.groovy +++ b/regression-test/suites/inverted_index_p0/unique_with_mow/test_primary_key_simple_case.groovy @@ -66,6 +66,7 @@ suite("test_primary_key_simple_case", "inverted_index") { sql """ INSERT INTO ${tableName} VALUES (5, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 20) """ + sql """ set enable_common_expr_pushdown = true """ result = sql """ SELECT * FROM ${tableName} t ORDER BY user_id; """ assertTrue(result.size() == 5) diff --git a/regression-test/suites/inverted_index_p0/unique_with_mow/test_unique_mow_sequence.groovy b/regression-test/suites/inverted_index_p0/unique_with_mow/test_unique_mow_sequence.groovy index 9c461f7371..c2a8b673f0 100644 --- a/regression-test/suites/inverted_index_p0/unique_with_mow/test_unique_mow_sequence.groovy +++ b/regression-test/suites/inverted_index_p0/unique_with_mow/test_unique_mow_sequence.groovy @@ -76,6 +76,7 @@ suite("test_unique_mow_sequence", "inverted_index") { } sql "sync" + sql """ set enable_common_expr_pushdown = true """ order_qt_sql "select * from $tableName where c_custkey < 6;" diff --git a/regression-test/suites/inverted_index_up_down_p0/inverted_index.groovy b/regression-test/suites/inverted_index_up_down_p0/inverted_index.groovy index 913602905e..b15846a944 100644 --- a/regression-test/suites/inverted_index_up_down_p0/inverted_index.groovy +++ b/regression-test/suites/inverted_index_up_down_p0/inverted_index.groovy @@ -21,6 +21,7 @@ suite("test_upgrade_downgrade_compatibility_inverted_index","p0,inverted_index,r def alter_res = "null" def useTime = 0 + sql "SET enable_common_expr_pushdown = true" sql "SET enable_match_without_inverted_index = false" def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> diff --git a/regression-test/suites/variant_p0/with_index/load.groovy b/regression-test/suites/variant_p0/with_index/load.groovy index ba46e7a9ee..4eeff1bfac 100644 --- a/regression-test/suites/variant_p0/with_index/load.groovy +++ b/regression-test/suites/variant_p0/with_index/load.groovy @@ -61,7 +61,8 @@ suite("regression_test_variant_with_index", "nonConcurrent"){ properties("replication_num" = "1", "disable_auto_compaction" = "true"); """ sql """insert into var_with_index values(1, '{"a" : 0, "b": 3}', 'hello world'), (2, '{"a" : 123}', 'world'),(3, '{"a" : 123}', 'hello world')""" - sql """set enable_match_without_inverted_index = false""" + sql """set enable_match_without_inverted_index = false""" + sql """ set enable_common_expr_pushdown = true """ qt_sql_inv_1 """select v["a"] from var_with_index where inv match 'hello' order by k""" qt_sql_inv_2 """select v["a"] from var_with_index where inv match 'hello' and cast(v['a'] as int) > 0 order by k""" qt_sql_inv_3 """select * from var_with_index where inv match 'hello' and cast(v["a"] as int) > 0 order by k""" diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy b/regression-test/suites/variant_p0/with_index/var_index.groovy index 67f7236c26..ab7b41eb83 100644 --- a/regression-test/suites/variant_p0/with_index/var_index.groovy +++ b/regression-test/suites/variant_p0/with_index/var_index.groovy @@ -33,6 +33,7 @@ suite("regression_test_variant_var_index", "p0"){ sql """insert into var_index values(2, '{"a" : 18811, "b" : "hello world", "c" : 1181111}')""" sql """insert into var_index values(3, '{"a" : 18811, "b" : "hello wworld", "c" : 11111}')""" sql """insert into var_index values(4, '{"a" : 1234, "b" : "hello xxx world", "c" : 8181111}')""" + sql """ set enable_common_expr_pushdown = true """ sql """set enable_match_without_inverted_index = false""" qt_sql """select * from var_index where cast(v["a"] as smallint) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 1024 order by k""" sql """set enable_match_without_inverted_index = true"""