database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](inverted index) skip read index column data only for DUP and MOW table (#32594)

Browse Source
This commit is contained in:
Kang
2024-03-21 17:44:56 +08:00
committed by yiguolei
parent 26efde2925
commit 55b7f7f019
5 changed files with 223 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
be/src/olap/iterators.h
View File

@ -97,6 +97,7 @@ public:
int block_row_max = 4096 - 32; // see https://github.com/apache/doris/pull/11816
TabletSchemaSPtr tablet_schema = nullptr;
bool enable_unique_key_merge_on_write = false;
bool record_rowids = false;
// flag for enable topn opt
bool use_topn_opt = false;

2
be/src/olap/rowset/beta_rowset_reader.cpp
View File

@ -215,6 +215,8 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
}
_read_options.use_page_cache = _read_context->use_page_cache;
_read_options.tablet_schema = _read_context->tablet_schema;
_read_options.enable_unique_key_merge_on_write =
_read_context->enable_unique_key_merge_on_write;
_read_options.record_rowids = _read_context->record_rowids;
_read_options.use_topn_opt = _read_context->use_topn_opt;
_read_options.topn_filter_source_node_ids = _read_context->topn_filter_source_node_ids;

6
be/src/olap/rowset/segment_v2/segment_iterator.cpp
View File

@ -1086,6 +1086,12 @@ Status SegmentIterator::_apply_inverted_index_on_block_column_predicate(
}
bool SegmentIterator::_need_read_data(ColumnId cid) {
// only support DUP_KEYS and UNIQUE_KEYS with MOW
if (!((_opts.tablet_schema->keys_type() == KeysType::DUP_KEYS ||
(_opts.tablet_schema->keys_type() == KeysType::UNIQUE_KEYS &&
_opts.enable_unique_key_merge_on_write)))) {
return true;
}
// if there is delete predicate, we always need to read data
if (_opts.delete_condition_predicates->num_of_column_predicate() > 0) {
return true;

83
regression-test/data/inverted_index_p0/test_index_skip_read_data.out Normal file
View File

@ -0,0 +1,83 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql10 --
1 20 300
1 20 400
-- !sql11 --
1 20 300
1 20 400
-- !sql12 --
1 20 300
1 20 400
-- !sql13 --
1 20 300
-- !sql14 --
1 20 400
-- !sql15 --
20
20
-- !sql16 --
1
1
-- !sql17 --
1 20
-- !sql18 --
1 20
-- !sql20 --
1 20 400
-- !sql21 --
1 20 400
-- !sql22 --
1 20 400
-- !sql23 --
-- !sql24 --
1 20 400
-- !sql25 --
20
-- !sql26 --
1
-- !sql27 --
-- !sql28 --
1 20
-- !sql30 --
1 20 400
-- !sql31 --
1 20 400
-- !sql32 --
1 20 400
-- !sql33 --
-- !sql34 --
1 20 400
-- !sql35 --
20
-- !sql36 --
1
-- !sql37 --
-- !sql38 --
1 20

131
regression-test/suites/inverted_index_p0/test_index_skip_read_data.groovy Normal file
View File

@ -0,0 +1,131 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_index_skip_read_data", "p0"){
def indexTbName1 = "test_index_skip_read_data_dup"
def indexTbName2 = "test_index_skip_read_data_mow"
def indexTbName3 = "test_index_skip_read_data_mor"
// dup
sql "DROP TABLE IF EXISTS ${indexTbName1}"
sql """
CREATE TABLE ${indexTbName1} (
`k1` int(11) NULL COMMENT "",
`k2` varchar(20) NULL COMMENT "",
`data` text NULL COMMENT "",
INDEX idx_k1 (`k1`) USING INVERTED COMMENT '',
INDEX idx_k2 (`k2`) USING BITMAP COMMENT '',
INDEX idx_data (`data`) USING INVERTED COMMENT ''
) ENGINE=OLAP
DUPLICATE KEY(`k1`, `k2`)
COMMENT "OLAP"
DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"disable_auto_compaction" = "true"
);
"""
sql """ INSERT INTO ${indexTbName1} VALUES (1, 20, 300); """
sql """ INSERT INTO ${indexTbName1} VALUES (1, 20, 400); """
qt_sql10 """ SELECT * FROM ${indexTbName1} ORDER BY k1,k2,data; """
qt_sql11 """ SELECT * FROM ${indexTbName1} WHERE k1 = 1 ORDER BY k1,k2,data; """
qt_sql12 """ SELECT * FROM ${indexTbName1} WHERE k2 = 20 ORDER BY k1,k2,data; """
qt_sql13 """ SELECT * FROM ${indexTbName1} WHERE data = 300 ORDER BY k1,k2,data; """
qt_sql14 """ SELECT * FROM ${indexTbName1} WHERE data = 400 ORDER BY k1,k2,data; """
qt_sql15 """ SELECT k2 FROM ${indexTbName1} WHERE k1 = 1 ORDER BY k1,k2,data; """
qt_sql16 """ SELECT k1 FROM ${indexTbName1} WHERE k2 = 20 ORDER BY k1,k2,data; """
qt_sql17 """ SELECT k1, k2 FROM ${indexTbName1} WHERE data = 300 ORDER BY k1,k2,data; """
qt_sql18 """ SELECT k1, k2 FROM ${indexTbName1} WHERE data = 400 ORDER BY k1,k2,data; """
// mow
sql "DROP TABLE IF EXISTS ${indexTbName2}"
sql """
CREATE TABLE ${indexTbName2} (
`k1` int(11) NULL COMMENT "",
`k2` varchar(20) NULL COMMENT "",
`data` text NULL COMMENT "",
INDEX idx_k1 (`k1`) USING INVERTED COMMENT '',
INDEX idx_k2 (`k2`) USING BITMAP COMMENT '',
INDEX idx_data (`data`) USING INVERTED COMMENT ''
) ENGINE=OLAP
UNIQUE KEY(`k1`, `k2`)
COMMENT "OLAP"
DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"enable_unique_key_merge_on_write" = "true",
"disable_auto_compaction" = "true"
);
"""
sql """ INSERT INTO ${indexTbName2} VALUES (1, 20, 300); """
sql """ INSERT INTO ${indexTbName2} VALUES (1, 20, 400); """
qt_sql20 """ SELECT * FROM ${indexTbName2} ORDER BY k1,k2,data; """
qt_sql21 """ SELECT * FROM ${indexTbName2} WHERE k1 = 1 ORDER BY k1,k2,data; """
qt_sql22 """ SELECT * FROM ${indexTbName2} WHERE k2 = 20 ORDER BY k1,k2,data; """
qt_sql23 """ SELECT * FROM ${indexTbName2} WHERE data = 300 ORDER BY k1,k2,data; """
qt_sql24 """ SELECT * FROM ${indexTbName2} WHERE data = 400 ORDER BY k1,k2,data; """
qt_sql25 """ SELECT k2 FROM ${indexTbName2} WHERE k1 = 1 ORDER BY k1,k2,data; """
qt_sql26 """ SELECT k1 FROM ${indexTbName2} WHERE k2 = 20 ORDER BY k1,k2,data; """
qt_sql27 """ SELECT k1, k2 FROM ${indexTbName2} WHERE data = 300 ORDER BY k1,k2,data; """
qt_sql28 """ SELECT k1, k2 FROM ${indexTbName2} WHERE data = 400 ORDER BY k1,k2,data; """
// mor
sql "DROP TABLE IF EXISTS ${indexTbName3}"
sql """
CREATE TABLE ${indexTbName3} (
`k1` int(11) NULL COMMENT "",
`k2` varchar(20) NULL COMMENT "",
`data` text NULL COMMENT "",
INDEX idx_k1 (`k1`) USING INVERTED COMMENT '',
INDEX idx_k2 (`k2`) USING BITMAP COMMENT '',
INDEX idx_data (`data`) USING INVERTED COMMENT ''
) ENGINE=OLAP
UNIQUE KEY(`k1`, `k2`)
COMMENT "OLAP"
DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1",
"enable_unique_key_merge_on_write" = "false",
"disable_auto_compaction" = "true"
);
"""
sql """ INSERT INTO ${indexTbName3} VALUES (1, 20, 300); """
sql """ INSERT INTO ${indexTbName3} VALUES (1, 20, 400); """
qt_sql30 """ SELECT * FROM ${indexTbName3} ORDER BY k1,k2,data; """
qt_sql31 """ SELECT * FROM ${indexTbName3} WHERE k1 = 1 ORDER BY k1,k2,data; """
qt_sql32 """ SELECT * FROM ${indexTbName3} WHERE k2 = 20 ORDER BY k1,k2,data; """
qt_sql33 """ SELECT * FROM ${indexTbName3} WHERE data = 300 ORDER BY k1,k2,data; """
qt_sql34 """ SELECT * FROM ${indexTbName3} WHERE data = 400 ORDER BY k1,k2,data; """
qt_sql35 """ SELECT k2 FROM ${indexTbName3} WHERE k1 = 1 ORDER BY k1,k2,data; """
qt_sql36 """ SELECT k1 FROM ${indexTbName3} WHERE k2 = 20 ORDER BY k1,k2,data; """
qt_sql37 """ SELECT k1, k2 FROM ${indexTbName3} WHERE data = 300 ORDER BY k1,k2,data; """
qt_sql38 """ SELECT k1, k2 FROM ${indexTbName3} WHERE data = 400 ORDER BY k1,k2,data; """
}