database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](multi-catalog) read parquet file by start/offset (#10843)

Browse Source 
To avoid reading the repeat row group, we should align offsets
This commit is contained in:
slothever
2022-07-18 20:51:08 +08:00
committed by GitHub
parent 60dd322aba
commit 8a366c9ba2
14 changed files with 169 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
be/src/vec/exec/varrow_scanner.cpp
View File

@ -76,8 +76,9 @@ Status VArrowScanner::_open_next_reader() {
if (range.__isset.num_of_columns_from_file) {
num_of_columns_from_file = range.num_of_columns_from_file;
}
_cur_file_reader = _new_arrow_reader(file_reader.release(), _state->batch_size(),
num_of_columns_from_file);
_cur_file_reader =
_new_arrow_reader(file_reader.release(), _state->batch_size(),
num_of_columns_from_file, range.start_offset, range.size);
auto tuple_desc = _state->desc_tbl().get_tuple_descriptor(_tupleId);
Status status = _cur_file_reader->init_reader(tuple_desc, _src_slot_descs, _conjunct_ctxs,
_state->timezone());