diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp index 9437b218dc..02f27aa6e6 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp @@ -72,7 +72,7 @@ Status IndexedColumnReader::load_index_page(const PagePointerPB& pp, PageHandle* std::unique_ptr local_compress_codec; RETURN_IF_ERROR(get_block_compression_codec(_meta.compression(), local_compress_codec)); RETURN_IF_ERROR(read_page(PagePointer(pp), handle, &body, &footer, INDEX_PAGE, - local_compress_codec.get())); + local_compress_codec.get(), false)); RETURN_IF_ERROR(reader->parse(body, footer.index_page_footer())); _mem_size += body.get_size(); return Status::OK(); @@ -80,7 +80,7 @@ Status IndexedColumnReader::load_index_page(const PagePointerPB& pp, PageHandle* Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle, Slice* body, PageFooterPB* footer, PageTypePB type, - BlockCompressionCodec* codec) const { + BlockCompressionCodec* codec, bool pre_decode) const { PageReadOptions opts; opts.file_reader = _file_reader.get(); opts.page_pointer = pp; @@ -91,6 +91,7 @@ Status IndexedColumnReader::read_page(const PagePointer& pp, PageHandle* handle, opts.kept_in_memory = _kept_in_memory; opts.type = type; opts.encoding_info = _encoding_info; + opts.pre_decode = pre_decode; return PageIO::read_and_decompress_page(opts, handle, body, footer); } @@ -106,8 +107,8 @@ Status IndexedColumnIterator::_read_data_page(const PagePointer& pp) { PageHandle handle; Slice body; PageFooterPB footer; - RETURN_IF_ERROR( - _reader->read_page(pp, &handle, &body, &footer, DATA_PAGE, _compress_codec.get())); + RETURN_IF_ERROR(_reader->read_page(pp, &handle, &body, &footer, DATA_PAGE, + _compress_codec.get(), true)); // parse data page // note that page_index is not used in IndexedColumnIterator, so we pass 0 PageDecoderOptions opts; diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.h b/be/src/olap/rowset/segment_v2/indexed_column_reader.h index e70b2aae5c..15b4c175d5 100644 --- a/be/src/olap/rowset/segment_v2/indexed_column_reader.h +++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.h @@ -54,7 +54,7 @@ public: // read a page specified by `pp' from `file' into `handle' Status read_page(const PagePointer& pp, PageHandle* handle, Slice* body, PageFooterPB* footer, - PageTypePB type, BlockCompressionCodec* codec) const; + PageTypePB type, BlockCompressionCodec* codec, bool pre_decode) const; int64_t num_values() const { return _num_values; } const EncodingInfo* encoding_info() const { return _encoding_info; } diff --git a/be/src/olap/rowset/segment_v2/page_io.cpp b/be/src/olap/rowset/segment_v2/page_io.cpp index d0d8db629a..4388feb6dc 100644 --- a/be/src/olap/rowset/segment_v2/page_io.cpp +++ b/be/src/olap/rowset/segment_v2/page_io.cpp @@ -197,7 +197,7 @@ Status PageIO::read_and_decompress_page(const PageReadOptions& opts, PageHandle* opts.stats->uncompressed_bytes_read += body_size; } - if (opts.encoding_info) { + if (opts.pre_decode && opts.encoding_info) { auto* pre_decoder = opts.encoding_info->get_data_page_pre_decoder(); if (pre_decoder) { RETURN_IF_ERROR(pre_decoder->decode( diff --git a/be/src/olap/rowset/segment_v2/page_io.h b/be/src/olap/rowset/segment_v2/page_io.h index 7b8bee7358..86cff00f89 100644 --- a/be/src/olap/rowset/segment_v2/page_io.h +++ b/be/src/olap/rowset/segment_v2/page_io.h @@ -67,6 +67,9 @@ struct PageReadOptions { const EncodingInfo* encoding_info = nullptr; + // index_page should not be pre-decoded + bool pre_decode = true; + void sanity_check() const { CHECK_NOTNULL(file_reader); CHECK_NOTNULL(stats); diff --git a/regression-test/data/index_p0/test_decimal_bitmap_index_multi_page.out b/regression-test/data/index_p0/test_decimal_bitmap_index_multi_page.out new file mode 100644 index 0000000000..538cbb9404 --- /dev/null +++ b/regression-test/data/index_p0/test_decimal_bitmap_index_multi_page.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +a DECIMAL(12,6) No true \N + +-- !sql -- +default_cluster:regression_test_index_p0.test_decimal_bitmap_index_multi_page bitmap_index_multi_page a BITMAP + +-- !sql -- +0.000001 +1.000001 +2.000001 + +-- !sql -- +131072.000001 +131071.000001 +131070.000001 diff --git a/regression-test/suites/index_p0/load.groovy b/regression-test/suites/index_p0/load.groovy new file mode 100644 index 0000000000..0e9d856b56 --- /dev/null +++ b/regression-test/suites/index_p0/load.groovy @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_bitmap_index_load") { + def tbName = "test_decimal_bitmap_index_multi_page" + + sql """ + drop TABLE if exists `${tbName}` force; + """ + sql """ + CREATE TABLE `${tbName}` ( + `a` decimal(12, 6) NOT NULL + ) ENGINE = OLAP + DUPLICATE KEY(`a`) + DISTRIBUTED BY HASH(`a`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1"); + """ + sql """ + create index bitmap_index_multi_page on ${tbName}(a) using bitmap; + """ + + streamLoad { + table "${tbName}" + set 'column_separator', '|' + set 'columns', 'a,temp' + file """${context.sf1DataPath}/regression/bitmap_index_test.csv""" + time 10000 // limit inflight 10s + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } +} + diff --git a/regression-test/suites/index_p0/test_decimal_bitmap_index_multi_page.groovy b/regression-test/suites/index_p0/test_decimal_bitmap_index_multi_page.groovy new file mode 100644 index 0000000000..6607c06ad4 --- /dev/null +++ b/regression-test/suites/index_p0/test_decimal_bitmap_index_multi_page.groovy @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_decimal_bitmap_index_multi_page") { + def tbName = "test_decimal_bitmap_index_multi_page" + + qt_sql "desc ${tbName};" + qt_sql "SHOW INDEX FROM ${tbName};" + qt_sql "select * from ${tbName} order by a ASC limit 3;" + qt_sql "select * from ${tbName} order by a DESC limit 3;" + +}