database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](inverted index) disable range query in StringTypeInvertedIndexReader (#38218) (#38926)

Browse Source 
## Proposed changes

pick from master #38218

Issue Number: close #xxx

<!--Describe your changes.-->
This commit is contained in:
Sun Chenyang
2024-08-07 10:44:02 +08:00
committed by GitHub
parent e400859531
commit b856530b09
3 changed files with 61 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
be/src/olap/rowset/segment_v2/segment_iterator.cpp
View File

@ -890,7 +890,8 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool
if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_inverted_index_query) {
return false;
}
if (_inverted_index_iterators[pred->column_id()] == nullptr) {
auto pred_column_id = pred->column_id();
if (_inverted_index_iterators[pred_column_id] == nullptr) {
//this column without inverted index
return false;
}
@ -905,13 +906,21 @@ bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool
return false;
}
// UNTOKENIZED strings exceed ignore_above, they are written as null, causing range query errors
if (PredicateTypeTraits::is_range(pred->type()) &&
_inverted_index_iterators[pred_column_id] != nullptr &&
_inverted_index_iterators[pred_column_id]->get_inverted_index_reader_type() ==
InvertedIndexReaderType::STRING_TYPE) {
return false;
}
// Function filter no apply inverted index
if (dynamic_cast<LikeColumnPredicate<TYPE_CHAR>*>(pred) != nullptr ||
dynamic_cast<LikeColumnPredicate<TYPE_STRING>*>(pred) != nullptr) {
return false;
}
bool handle_by_fulltext = _column_has_fulltext_index(pred->column_id());
bool handle_by_fulltext = _column_has_fulltext_index(pred_column_id);
if (handle_by_fulltext) {
// when predicate in compound condition which except leafNode of andNode,
// only can apply match query for fulltext index,

3
regression-test/data/inverted_index_p0/test_ignore_above_in_index.out
View File

@ -2,3 +2,6 @@
-- !sql --
3
-- !sql --
772

47
regression-test/suites/inverted_index_p0/test_ignore_above_in_index.groovy
View File

@ -39,4 +39,51 @@ suite("test_ignore_above_in_index", "p0") {
sql "insert into ${tableName} values (20, '1234567890');"
sql "insert into ${tableName} values (20, '1234567890');"
qt_sql "select count() from ${tableName} where c = '1234567890';"
def tableName2 = "test_ignore_above_in_index2"
sql "DROP TABLE IF EXISTS ${tableName2}"
sql """
CREATE TABLE ${tableName2} (
`@timestamp` int(11) NULL COMMENT "",
`clientip` string NULL COMMENT "",
`request` string NULL COMMENT "",
`status` int NULL COMMENT "",
`size` int NULL COMMENT "",
INDEX clientip_idx (`clientip`) USING INVERTED PROPERTIES("ignore_above"="5") COMMENT '',
INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true") COMMENT '',
INDEX status_idx (`status`) USING INVERTED COMMENT '',
INDEX size_idx (`size`) USING INVERTED COMMENT ''
) ENGINE=OLAP
DUPLICATE KEY(`@timestamp`)
COMMENT "OLAP"
DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
);
"""
// load the json data
streamLoad {
table "${tableName2}"
set 'read_json_by_line', 'true'
set 'format', 'json'
file 'documents-1000.json' // import json file
time 10000 // limit inflight 10s
// if declared a check callback, the default check condition will ignore.
// So you must check all condition
check { result, exception, startTime, endTime ->
if (exception != null) {
throw exception
}
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("success", json.Status.toLowerCase())
assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows)
assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
}
}
qt_sql "select count() from ${tableName2} where clientip > '17.0';"
}