[Fix](orc-reader) Fix orc complex types when late materialization was turned on by disabling late materialization in this case. (#26548)

Fix orc complex types when late materialization was turned on in orc reader by disabling late materialization in this case.
This commit is contained in:
Qi Chen
2023-11-09 12:05:43 +08:00
committed by GitHub
parent f6b7046a6e
commit d1438a8563
4 changed files with 25 additions and 1 deletions

View File

@ -734,6 +734,13 @@ Status OrcReader::set_fill_columns(
}
}
for (auto& each : _tuple_descriptor->slots()) {
PrimitiveType column_type = each->col_type();
if (column_type == TYPE_ARRAY || column_type == TYPE_MAP || column_type == TYPE_STRUCT) {
_has_complex_type = true;
}
}
for (auto& kv : partition_columns) {
auto iter = predicate_columns.find(kv.first);
if (iter == predicate_columns.end()) {
@ -754,7 +761,8 @@ Status OrcReader::set_fill_columns(
}
}
if (_enable_lazy_mat && _lazy_read_ctx.predicate_columns.first.size() > 0 &&
if (!_has_complex_type && _enable_lazy_mat &&
_lazy_read_ctx.predicate_columns.first.size() > 0 &&
_lazy_read_ctx.lazy_read_columns.size() > 0) {
_lazy_read_ctx.can_lazy_read = true;
}

View File

@ -552,6 +552,7 @@ private:
std::shared_ptr<ObjectPool> _obj_pool;
std::unique_ptr<orc::StringDictFilter> _string_dict_filter;
bool _is_dict_cols_converted;
bool _has_complex_type = false;
};
class ORCFileInputStream : public orc::InputStream {

File diff suppressed because one or more lines are too long

View File

@ -105,6 +105,9 @@ suite("test_hive_basic_type", "external_docker,hive,external_docker_hive,p0,exte
// hive tables in rcbinary format are not supported
//order_qt_37 """select * from ${catalog_name}.${ex_db_name}.rcbinary_all_types limit 1;"""
// orc_all_types_t predicate test
order_qt_41 """select * from ${catalog_name}.${ex_db_name}.orc_all_types_t where t_int = 3;"""
//sql """drop catalog if exists ${catalog_name} """
}
}