database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](iceberg_orc)Fixed the bug that the iceberg reader did not perform position delete when reading the orc file without a predicate. (#34814) (#34882)

Browse Source 
bp #34814
This commit is contained in:
daidai
2024-05-15 11:31:29 +08:00
committed by GitHub
parent e13ce905cf
commit 02084fd91f
2 changed files with 63 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
be/src/vec/exec/format/orc/vorc_reader.cpp
View File

@ -1747,9 +1747,16 @@ Status OrcReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
}
} else {
if (_delete_rows_filter_ptr) {
_execute_filter_position_delete_rowids(*_delete_rows_filter_ptr);
RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(block, columns_to_filter,
(*_delete_rows_filter_ptr)));
} else {
std::unique_ptr<IColumn::Filter> filter(new IColumn::Filter(block->rows(), 1));
_execute_filter_position_delete_rowids(*filter);
RETURN_IF_CATCH_EXCEPTION(
Block::filter_block_internal(block, columns_to_filter, (*filter)));
}
Block::erase_useless_column(block, column_to_keep);
static_cast<void>(_convert_dict_cols_to_string_cols(block, &batch_vec));
}

56
regression-test/suites/external_table_p2/iceberg/iceberg_position_delete.groovy
View File

@ -101,6 +101,62 @@ suite("iceberg_position_delete", "p2,external,iceberg,external_remote,external_r
qt_parquet_19 """ select count(*) from iceberg_position_parquet where name != 'final entryxxxxxx' ;"""
qt_parquet_20 """ select count(*) from iceberg_position_parquet; """
List<List<Object>> iceberg_position_orc = sql """ select * from iceberg_position_orc ;"""
List<List<Object>> iceberg_position_parquet = sql """ select * from iceberg_position_parquet;"""
List<List<Object>> iceberg_position_gen = sql """ select * from iceberg_position_gen_data;"""
assertTrue(iceberg_position_orc.size() == iceberg_position_gen.size())
assertTrue(iceberg_position_orc.size() == iceberg_position_parquet.size())
assertTrue(iceberg_position_orc.size() == 5632)
List<List<Object>> iceberg_position_orc_1 = sql """select * from iceberg_position_orc where id != 1;"""
List<List<Object>> iceberg_position_orc_2 = sql """select * from iceberg_position_orc where name != "hello word" ;"""
List<List<Object>> iceberg_position_orc_3 = sql """select id from iceberg_position_orc where id != 1;"""
List<List<Object>> iceberg_position_orc_4 = sql """select name from iceberg_position_orc where id != 1;"""
List<List<Object>> iceberg_position_orc_5 = sql """select name from iceberg_position_orc where name != "hello word" ;"""
List<List<Object>> iceberg_position_orc_6 = sql """select id from iceberg_position_orc where name != "hello word" ;"""
List<List<Object>> iceberg_position_orc_7 = sql """select * from iceberg_position_orc where id != 1 and name != "33333";"""
assertTrue(iceberg_position_orc_1.size() == 5632)
assertTrue(iceberg_position_orc_2.size() == 5632)
assertTrue(iceberg_position_orc_3.size() == 5632)
assertTrue(iceberg_position_orc_4.size() == 5632)
assertTrue(iceberg_position_orc_5.size() == 5632)
assertTrue(iceberg_position_orc_6.size() == 5632)
assertTrue(iceberg_position_orc_7.size() == 5632)
List<List<Object>> iceberg_position_gen_1 = sql """select * from iceberg_position_gen_data where id != 1 and name != "hello";"""
assertTrue(iceberg_position_gen_1.size() == 5632)
List<List<Object>> iceberg_position_gen_2 = sql """select * from iceberg_position_gen_data where id != 2;"""
assertTrue(iceberg_position_gen_2.size() == 5120)
List<List<Object>> iceberg_position_gen_22 = sql """select * from iceberg_position_gen_data where id != 5;"""
assertTrue(iceberg_position_gen_22.size() == 5632)
List<List<Object>> iceberg_position_gen_3 = sql """select * from iceberg_position_gen_data where name != "hello word" ;"""
assertTrue(iceberg_position_gen_3.size() == 5632)
List<List<Object>> iceberg_position_gen_4 = sql """select id from iceberg_position_gen_data where id != 2;"""
assertTrue(iceberg_position_gen_4.size() == 5120)
List<List<Object>> iceberg_position_gen_44 = sql """select id from iceberg_position_gen_data where id != 5;"""
assertTrue(iceberg_position_gen_44.size() == 5632)
List<List<Object>> iceberg_position_gen_5 = sql """select name from iceberg_position_gen_data where id != 2;"""
assertTrue(iceberg_position_gen_5.size() == 5120)
List<List<Object>> iceberg_position_gen_55 = sql """select name from iceberg_position_gen_data where id != 5;"""
assertTrue(iceberg_position_gen_55.size() == 5632)
List<List<Object>> iceberg_position_gen_6 = sql """select name from iceberg_position_gen_data where name != "hello wordxx" ;"""
assertTrue(iceberg_position_gen_6.size() == 5632)
List<List<Object>> iceberg_position_gen_7 = sql """select id from iceberg_position_gen_data where name != "hello word" ;"""
assertTrue(iceberg_position_gen_7.size() == 5632)
sql """drop catalog ${catalog_name}"""
}
}