database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bugfix](index) index page should not be bitshuffle decoded (#12231)

Browse Source 
* [bugfix](index) index page should not be bitshuffle decoded

* minor change
This commit is contained in:
TengJianPing
2022-09-01 11:56:44 +08:00
committed by GitHub
parent fc05d54f0d
commit f294d33332
7 changed files with 109 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
View File

@ -72,7 +72,7 @@ Status IndexedColumnReader::load_index_page(const PagePointerPB& pp, PageHandle*
std::unique_ptr<BlockCompressionCodec> local_compress_codec;
RETURN_IF_ERROR(get_block_compression_codec(_meta.compression(), local_compress_codec));
RETURN_IF_ERROR(read_page(PagePointer(pp), handle, &body, &footer, INDEX_PAGE,
local_compress_codec.get()));
local_compress_codec.get(), false));
RETURN_IF_ERROR(reader->parse(body, footer.index_page_footer()));
_mem_size += body.get_size();
return Status::OK();
@ -80,7 +80,7 @@ Status IndexedColumnReader::load_index_page(const PagePointerPB& pp, PageHandle*
Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle, Slice* body,
PageFooterPB* footer, PageTypePB type,
BlockCompressionCodec* codec) const {
BlockCompressionCodec* codec, bool pre_decode) const {
PageReadOptions opts;
opts.file_reader = _file_reader.get();
opts.page_pointer = pp;
@ -91,6 +91,7 @@ Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle,
opts.kept_in_memory = _kept_in_memory;
opts.type = type;
opts.encoding_info = _encoding_info;
opts.pre_decode = pre_decode;
return PageIO::read_and_decompress_page(opts, handle, body, footer);
}
@ -106,8 +107,8 @@ Status IndexedColumnIterator::_read_data_page(const PagePointer& pp) {
PageHandle handle;
Slice body;
PageFooterPB footer;
RETURN_IF_ERROR(
_reader->read_page(pp, &handle, &body, &footer, DATA_PAGE, _compress_codec.get()));
RETURN_IF_ERROR(_reader->read_page(pp, &handle, &body, &footer, DATA_PAGE,
_compress_codec.get(), true));
// parse data page
// note that page_index is not used in IndexedColumnIterator, so we pass 0
PageDecoderOptions opts;

2
be/src/olap/rowset/segment_v2/indexed_column_reader.h
View File

@ -54,7 +54,7 @@ public:
// read a page specified by `pp' from `file' into `handle'
Status read_page(const PagePointer& pp, PageHandle* handle, Slice* body, PageFooterPB* footer,
PageTypePB type, BlockCompressionCodec* codec) const;
PageTypePB type, BlockCompressionCodec* codec, bool pre_decode) const;
int64_t num_values() const { return _num_values; }
const EncodingInfo* encoding_info() const { return _encoding_info; }

2
be/src/olap/rowset/segment_v2/page_io.cpp
View File

@ -197,7 +197,7 @@ Status PageIO::read_and_decompress_page(const PageReadOptions& opts, PageHandle*
opts.stats->uncompressed_bytes_read += body_size;
}
if (opts.encoding_info) {
if (opts.pre_decode && opts.encoding_info) {
auto* pre_decoder = opts.encoding_info->get_data_page_pre_decoder();
if (pre_decoder) {
RETURN_IF_ERROR(pre_decoder->decode(

3
be/src/olap/rowset/segment_v2/page_io.h
View File

@ -67,6 +67,9 @@ struct PageReadOptions {
const EncodingInfo* encoding_info = nullptr;
// index_page should not be pre-decoded
bool pre_decode = true;
void sanity_check() const {
CHECK_NOTNULL(file_reader);
CHECK_NOTNULL(stats);

16
regression-test/data/index_p0/test_decimal_bitmap_index_multi_page.out Normal file
View File

@ -0,0 +1,16 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
a DECIMAL(12,6) No true \N
-- !sql --
default_cluster:regression_test_index_p0.test_decimal_bitmap_index_multi_page bitmap_index_multi_page a BITMAP
-- !sql --
0.000001
1.000001
2.000001
-- !sql --
131072.000001
131071.000001
131070.000001

58
regression-test/suites/index_p0/load.groovy Normal file
View File

@ -0,0 +1,58 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_bitmap_index_load") {
def tbName = "test_decimal_bitmap_index_multi_page"
sql """
drop TABLE if exists `${tbName}` force;
"""
sql """
CREATE TABLE `${tbName}` (
`a` decimal(12, 6) NOT NULL
) ENGINE = OLAP
DUPLICATE KEY(`a`)
DISTRIBUTED BY HASH(`a`) BUCKETS 1
PROPERTIES (
"replication_allocation" = "tag.location.default: 1");
"""
sql """
create index bitmap_index_multi_page on ${tbName}(a) using bitmap;
"""
streamLoad {
table "${tbName}"
set 'column_separator', '|'
set 'columns', 'a,temp'
file """${context.sf1DataPath}/regression/bitmap_index_test.csv"""
time 10000 // limit inflight 10s
// if declared a check callback, the default check condition will ignore.
// So you must check all condition
check { result, exception, startTime, endTime ->
if (exception != null) {
throw exception
}
log.info("Stream load result: ${result}".toString())
def json = parseJson(result)
assertEquals("success", json.Status.toLowerCase())
assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
}
}
}

25
regression-test/suites/index_p0/test_decimal_bitmap_index_multi_page.groovy Normal file
View File

@ -0,0 +1,25 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
suite("test_decimal_bitmap_index_multi_page") {
def tbName = "test_decimal_bitmap_index_multi_page"
qt_sql "desc ${tbName};"
qt_sql "SHOW INDEX FROM ${tbName};"
qt_sql "select * from ${tbName} order by a ASC limit 3;"
qt_sql "select * from ${tbName} order by a DESC limit 3;"
}