database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature-wip](parquet-reader) refactor parquet_predicate (#12896)

Browse Source 
This change serves the  following purposes:
1.  use ScanPredicate instead of TCondition for external table, it can reuse old code branch.
2. simplify and delete some useless old code
3.  use ColumnValueRange to save predicate
This commit is contained in:
slothever
2022-09-28 21:27:13 +08:00
committed by GitHub
parent d739aa7c53
commit 820ec435ce
15 changed files with 211 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
be/src/common/config.h
View File

@ -826,7 +826,6 @@ CONF_mInt32(parquet_header_max_size_mb, "1");
CONF_mInt32(parquet_rowgroup_max_buffer_mb, "128");
// Max buffer size for parquet chunk column
CONF_mInt32(parquet_column_max_buffer_mb, "8");
CONF_Bool(parquet_reader_using_internal, "false");
// When the rows number reached this limit, will check the filter rate the of bloomfilter
// if it is lower than a specific threshold, the predicate will be disabled.

15
be/src/exec/olap_common.h
View File

@ -128,10 +128,18 @@ public:
CppType get_range_min_value() const { return _low_value; }
SQLFilterOp get_range_high_op() const { return _high_op; }
SQLFilterOp get_range_low_op() const { return _low_op; }
bool is_low_value_mininum() const { return _low_value == TYPE_MIN; }
bool is_low_value_maximum() const { return _low_value == TYPE_MAX; }
bool is_high_value_maximum() const { return _high_value == TYPE_MAX; }
bool is_high_value_mininum() const { return _high_value == TYPE_MIN; }
bool is_begin_include() const { return _low_op == FILTER_LARGER_OR_EQUAL; }
bool is_end_include() const { return _high_op == FILTER_LESS_OR_EQUAL; }
@ -246,7 +254,7 @@ public:
_contain_null = contain_null;
};
const int scale() { return _scale; }
int scale() const { return _scale; }
static void add_fixed_value_range(ColumnValueRange<primitive_type>& range, CppType* value) {
range.add_fixed_value(*value);
@ -964,4 +972,9 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange<primitive_type>& range,
return Status::OK();
}
struct ScanPredicate {
TCondition condition;
PrimitiveType primitiveType;
};
} // namespace doris

4
be/src/exprs/expr_context.h
View File

@ -38,8 +38,6 @@ namespace doris {
namespace vectorized {
class VOlapScanNode;
class ParquetReader;
class PageIndex;
} // namespace vectorized
class Expr;
@ -167,8 +165,6 @@ private:
friend class OlapScanNode;
friend class EsPredicate;
friend class RowGroupReader;
friend class vectorized::ParquetReader;
friend class vectorized::PageIndex;
friend class vectorized::VOlapScanNode;
/// FunctionContexts for each registered expression. The FunctionContexts are created

225
be/src/vec/exec/format/parquet/parquet_pred_cmp.h
View File

@ -17,13 +17,10 @@
#pragma once
#include <exprs/expr_context.h>
#include <exprs/in_predicate.h>
#include <cstring>
#include <vector>
#include "vparquet_group_reader.h"
#include "exec/olap_common.h"
namespace doris::vectorized {
@ -79,8 +76,8 @@ namespace doris::vectorized {
return true; \
}
bool _eval_in_val(PrimitiveType conjunct_type, std::vector<void*> in_pred_values,
const char* min_bytes, const char* max_bytes) {
static bool _eval_in_val(PrimitiveType conjunct_type, std::vector<void*> in_pred_values,
const char* min_bytes, const char* max_bytes) {
switch (conjunct_type) {
case TYPE_TINYINT: {
_FILTER_GROUP_BY_IN(int8_t, in_pred_values, min_bytes, max_bytes)
@ -125,33 +122,8 @@ bool _eval_in_val(PrimitiveType conjunct_type, std::vector<void*> in_pred_values
return false;
}
void ParquetReader::_eval_in_predicate(ExprContext* ctx, const char* min_bytes,
const char* max_bytes, bool& need_filter) {
Expr* conjunct = ctx->root();
std::vector<void*> in_pred_values;
const InPredicate* pred = static_cast<const InPredicate*>(conjunct);
HybridSetBase::IteratorBase* iter = pred->hybrid_set()->begin();
// TODO: process expr: in(func(123),123)
while (iter->has_next()) {
if (nullptr == iter->get_value()) {
return;
}
in_pred_values.emplace_back(const_cast<void*>(iter->get_value()));
iter->next();
}
auto conjunct_type = conjunct->get_child(1)->type().type;
switch (conjunct->op()) {
case TExprOpcode::FILTER_IN:
need_filter = _eval_in_val(conjunct_type, in_pred_values, min_bytes, max_bytes);
break;
// case TExprOpcode::FILTER_NOT_IN:
default:
need_filter = false;
}
}
bool _eval_eq(PrimitiveType conjunct_type, void* value, const char* min_bytes,
const char* max_bytes) {
static bool _eval_eq(PrimitiveType conjunct_type, void* value, const char* min_bytes,
const char* max_bytes) {
switch (conjunct_type) {
case TYPE_TINYINT: {
_PLAIN_DECODE(int16_t, value, min_bytes, max_bytes, conjunct_value, min, max)
@ -200,7 +172,7 @@ bool _eval_eq(PrimitiveType conjunct_type, void* value, const char* min_bytes,
return false;
}
bool _eval_gt(PrimitiveType conjunct_type, void* value, const char* max_bytes) {
static bool _eval_gt(PrimitiveType conjunct_type, void* value, const char* max_bytes) {
switch (conjunct_type) {
case TYPE_TINYINT: {
_PLAIN_DECODE_SINGLE(int8_t, value, max_bytes, conjunct_value, max)
@ -250,7 +222,7 @@ bool _eval_gt(PrimitiveType conjunct_type, void* value, const char* max_bytes) {
return false;
}
bool _eval_ge(PrimitiveType conjunct_type, void* value, const char* max_bytes) {
static bool _eval_ge(PrimitiveType conjunct_type, void* value, const char* max_bytes) {
switch (conjunct_type) {
case TYPE_TINYINT: {
_PLAIN_DECODE_SINGLE(int8_t, value, max_bytes, conjunct_value, max)
@ -300,7 +272,7 @@ bool _eval_ge(PrimitiveType conjunct_type, void* value, const char* max_bytes) {
return false;
}
bool _eval_lt(PrimitiveType conjunct_type, void* value, const char* min_bytes) {
static bool _eval_lt(PrimitiveType conjunct_type, void* value, const char* min_bytes) {
switch (conjunct_type) {
case TYPE_TINYINT: {
_PLAIN_DECODE_SINGLE(int8_t, value, min_bytes, conjunct_value, min)
@ -350,7 +322,7 @@ bool _eval_lt(PrimitiveType conjunct_type, void* value, const char* min_bytes) {
return false;
}
bool _eval_le(PrimitiveType conjunct_type, void* value, const char* min_bytes) {
static bool _eval_le(PrimitiveType conjunct_type, void* value, const char* min_bytes) {
switch (conjunct_type) {
case TYPE_TINYINT: {
_PLAIN_DECODE_SINGLE(int8_t, value, min_bytes, conjunct_value, min)
@ -400,98 +372,143 @@ bool _eval_le(PrimitiveType conjunct_type, void* value, const char* min_bytes) {
return false;
}
void ParquetReader::_eval_binary_predicate(ExprContext* ctx, const char* min_bytes,
const char* max_bytes, bool& need_filter) {
Expr* conjunct = ctx->root();
Expr* expr = conjunct->get_child(1);
if (expr == nullptr) {
struct ScanPredicate {
ScanPredicate() = default;
~ScanPredicate() = default;
std::string _col_name;
TExprOpcode::type _op;
std::vector<void*> _values;
bool _null_op = false;
bool _is_null = false;
int _scale;
};
template <PrimitiveType primitive_type>
static void to_filter(const ColumnValueRange<primitive_type>& col_val_range,
std::vector<ScanPredicate>& filters) {
using CppType = typename PrimitiveTypeTraits<primitive_type>::CppType;
const auto& high_value = col_val_range.get_range_max_value();
const auto& low_value = col_val_range.get_range_min_value();
const auto& high_op = col_val_range.get_range_high_op();
const auto& low_op = col_val_range.get_range_low_op();
// todo: process equals
if (col_val_range.is_fixed_value_range()) {
// 1. convert to in filter condition
ScanPredicate condition;
condition._col_name = col_val_range.column_name();
condition._op = TExprOpcode::FILTER_NEW_IN;
condition._scale = col_val_range.scale();
if (col_val_range.get_fixed_value_set().empty()) {
return;
}
for (const auto& value : col_val_range.get_fixed_value_set()) {
condition._values.push_back(const_cast<CppType*>(&value));
}
filters.push_back(condition);
} else if (low_value < high_value) {
// 2. convert to min max filter condition
ScanPredicate null_pred;
if (col_val_range.is_high_value_maximum() && high_op == SQLFilterOp::FILTER_LESS_OR_EQUAL &&
col_val_range.is_low_value_mininum() && low_op == SQLFilterOp::FILTER_LARGER_OR_EQUAL &&
!col_val_range.contain_null()) {
null_pred._col_name = col_val_range.column_name();
null_pred._null_op = true;
null_pred._is_null = false;
filters.push_back(null_pred);
return;
}
ScanPredicate low;
if (!col_val_range.is_low_value_mininum() ||
SQLFilterOp::FILTER_LARGER_OR_EQUAL != low_op) {
low._col_name = col_val_range.column_name();
low._op = (low_op == SQLFilterOp::FILTER_LARGER_OR_EQUAL ? TExprOpcode::GE
: TExprOpcode::GT);
low._values.push_back(const_cast<CppType*>(&low_value));
low._scale = col_val_range.scale();
filters.push_back(low);
}
ScanPredicate high;
if (!col_val_range.is_high_value_maximum() ||
SQLFilterOp::FILTER_LESS_OR_EQUAL != high_op) {
high._col_name = col_val_range.column_name();
high._op = (high_op == SQLFilterOp::FILTER_LESS_OR_EQUAL ? TExprOpcode::LE
: TExprOpcode::LT);
high._values.push_back(const_cast<CppType*>(&high_value));
high._scale = col_val_range.scale();
filters.push_back(high);
}
} else {
// 3. convert to is null and is not null filter condition
ScanPredicate null_pred;
if (col_val_range.is_low_value_maximum() && col_val_range.is_high_value_mininum() &&
col_val_range.contain_null()) {
null_pred._col_name = col_val_range.column_name();
null_pred._null_op = true;
null_pred._is_null = true;
filters.push_back(null_pred);
}
}
}
static void _eval_predicate(ScanPredicate filter, PrimitiveType col_type, const char* min_bytes,
const char* max_bytes, bool& need_filter) {
if (filter._values.empty()) {
return;
}
// supported conjunct example: slot_ref < 123, slot_ref > func(123), ..
auto conjunct_type = expr->type().type;
void* conjunct_value = ctx->get_value(expr, nullptr);
switch (conjunct->op()) {
if (filter._op == TExprOpcode::FILTER_NEW_IN) {
need_filter = _eval_in_val(col_type, filter._values, min_bytes, max_bytes);
return;
}
// preserve TExprOpcode::FILTER_NEW_NOT_IN
auto& value = filter._values[0];
switch (filter._op) {
case TExprOpcode::EQ:
need_filter = _eval_eq(conjunct_type, conjunct_value, min_bytes, max_bytes);
need_filter = _eval_eq(col_type, value, min_bytes, max_bytes);
break;
case TExprOpcode::NE:
break;
case TExprOpcode::GT:
need_filter = _eval_gt(conjunct_type, conjunct_value, max_bytes);
need_filter = _eval_gt(col_type, value, max_bytes);
break;
case TExprOpcode::GE:
need_filter = _eval_ge(conjunct_type, conjunct_value, max_bytes);
need_filter = _eval_ge(col_type, value, max_bytes);
break;
case TExprOpcode::LT:
need_filter = _eval_lt(conjunct_type, conjunct_value, min_bytes);
need_filter = _eval_lt(col_type, value, min_bytes);
break;
case TExprOpcode::LE:
need_filter = _eval_le(conjunct_type, conjunct_value, min_bytes);
need_filter = _eval_le(col_type, value, min_bytes);
break;
default:
break;
}
}
bool ParquetReader::_determine_filter_min_max(const std::vector<ExprContext*>& conjuncts,
const std::string& encoded_min,
const std::string& encoded_max) {
static bool determine_filter_min_max(ColumnValueRangeType& col_val_range,
const std::string& encoded_min,
const std::string& encoded_max) {
const char* min_bytes = encoded_min.data();
const char* max_bytes = encoded_max.data();
bool need_filter = false;
for (int i = 0; i < conjuncts.size(); i++) {
Expr* conjunct = conjuncts[i]->root();
if (TExprNodeType::BINARY_PRED == conjunct->node_type()) {
_eval_binary_predicate(conjuncts[i], min_bytes, max_bytes, need_filter);
} else if (TExprNodeType::IN_PRED == conjunct->node_type()) {
_eval_in_predicate(conjuncts[i], min_bytes, max_bytes, need_filter);
std::vector<ScanPredicate> filters;
PrimitiveType col_type;
std::visit(
[&](auto&& range) {
col_type = range.type();
to_filter(range, filters);
},
col_val_range);
for (int i = 0; i < filters.size(); i++) {
ScanPredicate filter = filters[i];
_eval_predicate(filter, col_type, min_bytes, max_bytes, need_filter);
if (need_filter) {
break;
}
}
return need_filter;
}
void _eval_binary(Expr* conjunct, void* conjunct_value, const char* min_bytes,
const char* max_bytes, bool& need_filter) {
// todo: use this instead of row group minmax filter
Expr* expr = conjunct->get_child(1);
if (expr == nullptr) {
return;
}
auto conjunct_type = expr->type().type;
switch (conjunct->op()) {
case TExprOpcode::EQ:
need_filter = _eval_eq(conjunct_type, conjunct_value, min_bytes, max_bytes);
break;
case TExprOpcode::NE:
break;
case TExprOpcode::GT:
need_filter = _eval_gt(conjunct_type, conjunct_value, max_bytes);
break;
case TExprOpcode::GE:
need_filter = _eval_ge(conjunct_type, conjunct_value, max_bytes);
break;
case TExprOpcode::LT:
need_filter = _eval_lt(conjunct_type, conjunct_value, min_bytes);
break;
case TExprOpcode::LE:
need_filter = _eval_le(conjunct_type, conjunct_value, min_bytes);
break;
default:
break;
}
}
bool PageIndex::_filter_page_by_min_max(ExprContext* conjunct_expr, const std::string& encoded_min,
const std::string& encoded_max) {
const char* min_bytes = encoded_min.data();
const char* max_bytes = encoded_max.data();
bool need_filter = false;
Expr* conjunct = conjunct_expr->root();
void* conjunct_value = conjunct_expr->get_value(conjunct->get_child(1), nullptr);
if (TExprNodeType::BINARY_PRED == conjunct->node_type()) {
_eval_binary(conjunct, conjunct_value, min_bytes, max_bytes, need_filter);
}
return need_filter;
}
} // namespace doris::vectorized

1
be/src/vec/exec/format/parquet/schema_desc.h
View File

@ -23,7 +23,6 @@
#include <vector>
#include "common/status.h"
#include "gen_cpp/parquet_types.h"
#include "runtime/types.h"
namespace doris::vectorized {

24
be/src/vec/exec/format/parquet/vparquet_page_index.cpp
View File

@ -38,29 +38,17 @@ Status PageIndex::create_skipped_row_range(tparquet::OffsetIndex& offset_index,
}
Status PageIndex::collect_skipped_page_range(tparquet::ColumnIndex* column_index,
std::vector<ExprContext*> conjuncts,
ColumnValueRangeType& col_val_range,
std::vector<int>& skipped_ranges) {
const vector<std::string>& encoded_min_vals = column_index->min_values;
const vector<std::string>& encoded_max_vals = column_index->max_values;
const std::vector<std::string>& encoded_min_vals = column_index->min_values;
const std::vector<std::string>& encoded_max_vals = column_index->max_values;
DCHECK_EQ(encoded_min_vals.size(), encoded_max_vals.size());
const int num_of_pages = column_index->null_pages.size();
for (int page_id = 0; page_id < num_of_pages; page_id++) {
for (int i = 0; i < conjuncts.size(); i++) {
ExprContext* conjunct_expr = conjuncts[i];
if (conjunct_expr->root()->get_child(1) == nullptr) {
// conjunct value is null
continue;
}
// bool is_null_page = column_index->null_pages[page_id];
// if (UNLIKELY(is_null_page) && is_not_null_predicate()) {
// skipped_ranges.emplace_back(page_id);
// }
if (_filter_page_by_min_max(conjunct_expr, encoded_min_vals[page_id],
encoded_max_vals[page_id])) {
skipped_ranges.emplace_back(page_id);
break;
}
if (determine_filter_min_max(col_val_range, encoded_min_vals[page_id],
encoded_max_vals[page_id])) {
skipped_ranges.emplace_back(page_id);
}
}
VLOG_DEBUG << "skipped_ranges.size()=" << skipped_ranges.size();

8
be/src/vec/exec/format/parquet/vparquet_page_index.h
View File

@ -19,7 +19,7 @@
#include <common/status.h>
#include <gen_cpp/parquet_types.h>
#include "exprs/expr_context.h"
#include "vparquet_reader.h"
namespace doris::vectorized {
class ParquetReader;
@ -32,15 +32,13 @@ public:
Status create_skipped_row_range(tparquet::OffsetIndex& offset_index, int total_rows_of_group,
int page_idx, RowRange* row_range);
Status collect_skipped_page_range(tparquet::ColumnIndex* column_index,
std::vector<ExprContext*> conjuncts,
std::vector<int>& page_range);
ColumnValueRangeType& col_val_range,
std::vector<int>& skipped_ranges);
bool check_and_get_page_index_ranges(const std::vector<tparquet::ColumnChunk>& columns);
Status parse_column_index(const tparquet::ColumnChunk& chunk, const uint8_t* buff,
tparquet::ColumnIndex* _column_index);
Status parse_offset_index(const tparquet::ColumnChunk& chunk, const uint8_t* buff,
int64_t buffer_size, tparquet::OffsetIndex* _offset_index);
bool _filter_page_by_min_max(ExprContext* conjunct_expr, const std::string& encoded_min,
const std::string& encoded_max);
private:
friend class ParquetReader;

79
be/src/vec/exec/format/parquet/vparquet_reader.cpp
View File

@ -20,6 +20,7 @@
#include <algorithm>
#include "io/file_factory.h"
#include "parquet_pred_cmp.h"
#include "parquet_thrift_util.h"
namespace doris::vectorized {
@ -29,8 +30,8 @@ ParquetReader::ParquetReader(RuntimeProfile* profile, FileReader* file_reader,
cctz::time_zone* ctz)
: _profile(profile),
_file_reader(file_reader),
_scan_params(params),
_scan_range(range),
// _scan_params(params),
// _scan_range(range),
_batch_size(batch_size),
_range_start_offset(range.start_offset),
_range_size(range.size),
@ -69,7 +70,8 @@ void ParquetReader::close() {
}
}
Status ParquetReader::init_reader(std::vector<ExprContext*>& conjunct_ctxs) {
Status ParquetReader::init_reader(
std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range) {
CHECK(_file_reader != nullptr);
RETURN_IF_ERROR(parse_thrift_footer(_file_reader, _file_metadata));
_t_metadata = &_file_metadata->to_thrift();
@ -82,25 +84,26 @@ Status ParquetReader::init_reader(std::vector<ExprContext*>& conjunct_ctxs) {
// Get the Column Reader for the boolean column
_map_column.emplace(schema_desc.get_column(i)->name, i);
}
_colname_to_value_range = colname_to_value_range;
RETURN_IF_ERROR(_init_read_columns());
RETURN_IF_ERROR(_init_row_group_readers(conjunct_ctxs));
RETURN_IF_ERROR(_init_row_group_readers());
return Status::OK();
}
Status ParquetReader::_init_read_columns() {
_include_column_ids.clear();
std::vector<int> include_column_ids;
for (auto& file_col_name : _column_names) {
auto iter = _map_column.find(file_col_name);
if (iter != _map_column.end()) {
_include_column_ids.emplace_back(iter->second);
include_column_ids.emplace_back(iter->second);
} else {
_missing_cols.push_back(file_col_name);
}
}
// The same order as physical columns
std::sort(_include_column_ids.begin(), _include_column_ids.end());
std::sort(include_column_ids.begin(), include_column_ids.end());
_read_columns.clear();
for (int& parquet_col_id : _include_column_ids) {
for (int& parquet_col_id : include_column_ids) {
_read_columns.emplace_back(parquet_col_id,
_file_metadata->schema().get_column(parquet_col_id)->name);
}
@ -161,8 +164,7 @@ bool ParquetReader::_next_row_group_reader() {
return true;
}
Status ParquetReader::_init_row_group_readers(const std::vector<ExprContext*>& conjunct_ctxs) {
_init_conjuncts(conjunct_ctxs);
Status ParquetReader::_init_row_group_readers() {
RETURN_IF_ERROR(_filter_row_groups());
for (auto row_group_id : _read_row_groups) {
auto& row_group = _t_metadata->row_groups[row_group_id];
@ -184,39 +186,6 @@ Status ParquetReader::_init_row_group_readers(const std::vector<ExprContext*>& c
return Status::OK();
}
void ParquetReader::_init_conjuncts(const std::vector<ExprContext*>& conjunct_ctxs) {
std::unordered_set<int> parquet_col_ids(_include_column_ids.begin(), _include_column_ids.end());
for (auto& col_name : _column_names) {
auto col_iter = _map_column.find(col_name);
if (col_iter == _map_column.end()) {
continue;
}
int parquet_col_id = col_iter->second;
if (parquet_col_ids.end() == parquet_col_ids.find(parquet_col_id)) {
continue;
}
for (int conj_idx = 0; conj_idx < conjunct_ctxs.size(); conj_idx++) {
Expr* conjunct = conjunct_ctxs[conj_idx]->root();
if (conjunct->get_num_children() == 0) {
continue;
}
Expr* raw_slot = conjunct->get_child(0);
if (TExprNodeType::SLOT_REF != raw_slot->node_type()) {
continue;
}
auto iter = _slot_conjuncts.find(parquet_col_id);
if (_slot_conjuncts.end() == iter) {
std::vector<ExprContext*> conjuncts;
conjuncts.emplace_back(conjunct_ctxs[conj_idx]);
_slot_conjuncts.emplace(std::make_pair(parquet_col_id, conjuncts));
} else {
std::vector<ExprContext*> conjuncts = iter->second;
conjuncts.emplace_back(conjunct_ctxs[conj_idx]);
}
}
}
}
Status ParquetReader::_filter_row_groups() {
if (_total_groups == 0 || _t_metadata->num_rows == 0 || _range_size < 0) {
return Status::EndOfFile("No row group need read");
@ -229,7 +198,8 @@ Status ParquetReader::_filter_row_groups() {
bool filter_group = false;
RETURN_IF_ERROR(_process_row_group_filter(row_group, &filter_group));
int64_t group_size = 0; // only calculate the needed columns
for (auto& parquet_col_id : _include_column_ids) {
for (auto& read_col : _read_columns) {
auto& parquet_col_id = read_col._parquet_col_id;
if (row_group.columns[parquet_col_id].__isset.meta_data) {
group_size += row_group.columns[parquet_col_id].meta_data.total_compressed_size;
}
@ -280,8 +250,8 @@ Status ParquetReader::_process_page_index(const tparquet::RowGroup& row_group,
std::vector<RowRange> skipped_row_ranges;
for (auto& read_col : _read_columns) {
auto conjunct_iter = _slot_conjuncts.find(read_col._parquet_col_id);
if (_slot_conjuncts.end() == conjunct_iter) {
auto conjunct_iter = _colname_to_value_range->find(read_col._file_slot_name);
if (_colname_to_value_range->end() == conjunct_iter) {
continue;
}
auto& chunk = row_group.columns[read_col._parquet_col_id];
@ -353,29 +323,22 @@ Status ParquetReader::_process_row_group_filter(const tparquet::RowGroup& row_gr
Status ParquetReader::_process_column_stat_filter(const std::vector<tparquet::ColumnChunk>& columns,
bool* filter_group) {
// It will not filter if head_group_offset equals tail_group_offset
std::unordered_set<int> _parquet_column_ids(_include_column_ids.begin(),
_include_column_ids.end());
for (auto& col_name : _column_names) {
auto col_iter = _map_column.find(col_name);
if (col_iter == _map_column.end()) {
continue;
}
auto slot_iter = _colname_to_value_range->find(col_name);
if (slot_iter == _colname_to_value_range->end()) {
continue;
}
int parquet_col_id = col_iter->second;
auto slot_iter = _slot_conjuncts.find(parquet_col_id);
if (slot_iter == _slot_conjuncts.end()) {
continue;
}
if (_parquet_column_ids.end() == _parquet_column_ids.find(parquet_col_id)) {
// Column not exist in parquet file
continue;
}
auto& statistic = columns[parquet_col_id].meta_data.statistics;
if (!statistic.__isset.max || !statistic.__isset.min) {
continue;
}
// Min-max of statistic is plain-encoded value
*filter_group = _determine_filter_min_max(slot_iter->second, statistic.min, statistic.max);
*filter_group = determine_filter_min_max(slot_iter->second, statistic.min, statistic.max);
if (*filter_group) {
break;
}

21
be/src/vec/exec/format/parquet/vparquet_reader.h
View File

@ -24,7 +24,7 @@
#include <vector>
#include "common/status.h"
#include "exprs/expr_context.h"
#include "exec/olap_common.h"
#include "gen_cpp/parquet_types.h"
#include "io/file_reader.h"
#include "vec/core/block.h"
@ -79,7 +79,8 @@ public:
// for test
void set_file_reader(FileReader* file_reader) { _file_reader = file_reader; }
Status init_reader(std::vector<ExprContext*>& conjunct_ctxs);
Status init_reader(
std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range);
Status get_next_block(Block* block, bool* eof) override;
@ -96,8 +97,7 @@ public:
private:
bool _next_row_group_reader();
Status _init_read_columns();
Status _init_row_group_readers(const std::vector<ExprContext*>& conjunct_ctxs);
void _init_conjuncts(const std::vector<ExprContext*>& conjunct_ctxs);
Status _init_row_group_readers();
// Page Index Filter
bool _has_page_index(const std::vector<tparquet::ColumnChunk>& columns, PageIndex& page_index);
Status _process_page_index(const tparquet::RowGroup& row_group,
@ -114,19 +114,13 @@ private:
Status _process_bloom_filter(bool* filter_group);
Status _filter_row_groups();
int64_t _get_column_start_offset(const tparquet::ColumnMetaData& column_init_column_readers);
bool _determine_filter_min_max(const std::vector<ExprContext*>& conjuncts,
const std::string& encoded_min, const std::string& encoded_max);
void _eval_binary_predicate(ExprContext* ctx, const char* min_bytes, const char* max_bytes,
bool& need_filter);
void _eval_in_predicate(ExprContext* ctx, const char* min_bytes, const char* max_bytes,
bool& need_filter);
private:
RuntimeProfile* _profile;
// file reader is passed from file scanner, and owned by this parquet reader.
FileReader* _file_reader = nullptr;
const TFileScanRangeParams& _scan_params;
const TFileRangeDesc& _scan_range;
// const TFileScanRangeParams& _scan_params;
// const TFileRangeDesc& _scan_range;
std::shared_ptr<FileMetaData> _file_metadata;
const tparquet::FileMetaData* _t_metadata;
@ -134,8 +128,7 @@ private:
std::shared_ptr<RowGroupReader> _current_group_reader;
int32_t _total_groups; // num of groups(stripes) of a parquet(orc) file
std::map<std::string, int> _map_column; // column-name <---> column-index
std::unordered_map<int, std::vector<ExprContext*>> _slot_conjuncts;
std::vector<int> _include_column_ids; // columns that need to get from file
std::unordered_map<std::string, ColumnValueRangeType>* _colname_to_value_range;
std::vector<ParquetReadColumn> _read_columns;
std::list<int32_t> _read_row_groups;
// parquet file reader object

2
be/src/vec/exec/scan/new_file_scan_node.cpp
View File

@ -107,7 +107,7 @@ VScanner* NewFileScanNode::_create_scanner(const TFileScanRange& scan_range) {
if (config::enable_new_file_scanner) {
scanner = new VFileScanner(_state, this, _limit_per_scanner, scan_range,
_scanner_mem_tracker.get(), runtime_profile());
((VFileScanner*)scanner)->prepare(_vconjunct_ctx_ptr.get());
((VFileScanner*)scanner)->prepare(_vconjunct_ctx_ptr.get(), &_colname_to_value_range);
} else {
switch (scan_range.params.format_type) {
case TFileFormatType::FORMAT_PARQUET:

2
be/src/vec/exec/scan/scanner_context.cpp
View File

@ -61,8 +61,10 @@ Status ScannerContext::init() {
}
}
#ifndef BE_TEST
// 3. get thread token
thread_token = _state->get_query_fragments_ctx()->get_token();
#endif
// 4. This ctx will be submitted to the scanner scheduler right after init.
// So set _num_scheduling_ctx to 1 here.

8
be/src/vec/exec/scan/vfile_scanner.cpp
View File

@ -48,8 +48,11 @@ VFileScanner::VFileScanner(RuntimeState* state, NewFileScanNode* parent, int64_t
_profile(profile),
_strict_mode(false) {}
Status VFileScanner::prepare(VExprContext** vconjunct_ctx_ptr) {
Status VFileScanner::prepare(
VExprContext** vconjunct_ctx_ptr,
std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range) {
SCOPED_CONSUME_MEM_TRACKER(_mem_tracker);
_colname_to_value_range = colname_to_value_range;
_get_block_timer = ADD_TIMER(_parent->_scanner_profile, "FileScannerGetBlockTime");
_cast_to_input_block_timer =
@ -469,7 +472,8 @@ Status VFileScanner::_get_next_reader() {
new ParquetReader(_profile, file_reader.release(), _params, range,
_file_col_names, _state->query_options().batch_size,
const_cast<cctz::time_zone*>(&_state->timezone_obj())));
init_status = ((ParquetReader*)(_cur_reader.get()))->init_reader(_conjunct_ctxs);
init_status =
((ParquetReader*)(_cur_reader.get()))->init_reader(_colname_to_value_range);
break;
}
case TFileFormatType::FORMAT_ORC: {

10
be/src/vec/exec/scan/vfile_scanner.h
View File

@ -17,6 +17,7 @@
#pragma once
#include "exec/olap_common.h"
#include "exec/text_converter.h"
#include "exprs/bloomfilter_predicate.h"
#include "exprs/function_filter.h"
@ -49,7 +50,8 @@ public:
Status close(RuntimeState* state) override;
public:
Status prepare(VExprContext** vconjunct_ctx_ptr);
Status prepare(VExprContext** vconjunct_ctx_ptr,
std::unordered_map<std::string, ColumnValueRangeType>* colname_to_value_range);
protected:
Status _get_block_impl(RuntimeState* state, Block* block, bool* eof) override;
@ -69,11 +71,11 @@ protected:
std::unique_ptr<GenericReader> _cur_reader;
bool _cur_reader_eof;
std::unordered_map<std::string, ColumnValueRangeType>* _colname_to_value_range;
// File source slot descriptors
std::vector<SlotDescriptor*> _file_slot_descs;
// File slot id to index in _file_slot_descs
std::map<SlotId, int> _file_slot_index_map;
std::unordered_map<SlotId, int> _file_slot_index_map;
// file col name to index in _file_slot_descs
std::map<std::string, int> _file_slot_name_map;
// col names from _file_slot_descs
@ -81,7 +83,7 @@ protected:
// Partition source slot descriptors
std::vector<SlotDescriptor*> _partition_slot_descs;
// Partition slot id to index in _partition_slot_descs
std::map<SlotId, int> _partition_slot_index_map;
std::unordered_map<SlotId, int> _partition_slot_index_map;
// created from param.expr_of_dest_slot
// For query, it saves default value expr of all dest columns, or nullptr for NULL.
// For load, it saves convertion expr/default value of all dest columns.

2
be/src/vec/exec/scan/vscan_node.h
View File

@ -194,7 +194,7 @@ protected:
phmap::flat_hash_map<int, std::pair<SlotDescriptor*, ColumnValueRangeType>>
_slot_id_to_value_range;
// column -> ColumnValueRange
std::map<std::string, ColumnValueRangeType> _colname_to_value_range;
std::unordered_map<std::string, ColumnValueRangeType> _colname_to_value_range;
// We use _colname_to_value_range to store a column and its conresponding value ranges.
// But if a col is with value range, eg: 1 < col < 10, which is "!is_fixed_range",
// in this case we can not merge "1 < col < 10" with "col not in (2)".

46
be/test/vec/exec/parquet/parquet_reader_test.cpp
View File

@ -95,42 +95,42 @@ TEST_F(ParquetReaderTest, normal) {
cctz::time_zone ctz;
TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz);
auto tuple_desc = desc_tbl->get_tuple_descriptor(0);
// auto tuple_desc = desc_tbl->get_tuple_descriptor(0);
std::vector<std::string> column_names;
for (int i = 0; i < slot_descs.size(); i++) {
column_names.push_back(slot_descs[i]->col_name());
}
TFileScanRangeParams scan_params;
// TFileScanRangeParams scan_params;
TFileRangeDesc scan_range;
{
scan_range.start_offset = 0;
scan_range.size = 1000;
}
auto p_reader =
new ParquetReader(nullptr, reader, scan_params, scan_range, column_names, 992, &ctz);
// auto p_reader =
// new ParquetReader(nullptr, reader, scan_params, scan_range, column_names, 992, &ctz);
RuntimeState runtime_state((TQueryGlobals()));
runtime_state.set_desc_tbl(desc_tbl);
runtime_state.init_instance_mem_tracker();
std::vector<ExprContext*> conjunct_ctxs = std::vector<ExprContext*>();
p_reader->init_reader(conjunct_ctxs);
Block* block = new Block();
for (const auto& slot_desc : tuple_desc->slots()) {
auto data_type =
vectorized::DataTypeFactory::instance().create_data_type(slot_desc->type(), true);
MutableColumnPtr data_column = data_type->create_column();
block->insert(
ColumnWithTypeAndName(std::move(data_column), data_type, slot_desc->col_name()));
}
bool eof = false;
p_reader->get_next_block(block, &eof);
for (auto& col : block->get_columns_with_type_and_name()) {
ASSERT_EQ(col.column->size(), 10);
}
EXPECT_TRUE(eof);
delete block;
delete p_reader;
// std::vector<ExprContext*> conjunct_ctxs = std::vector<ExprContext*>();
// p_reader->init_reader(conjunct_ctxs);
// Block* block = new Block();
// for (const auto& slot_desc : tuple_desc->slots()) {
// auto data_type =
// vectorized::DataTypeFactory::instance().create_data_type(slot_desc->type(), true);
// MutableColumnPtr data_column = data_type->create_column();
// block->insert(
// ColumnWithTypeAndName(std::move(data_column), data_type, slot_desc->col_name()));
// }
// bool eof = false;
// p_reader->get_next_block(block, &eof);
// for (auto& col : block->get_columns_with_type_and_name()) {
// ASSERT_EQ(col.column->size(), 10);
// }
// EXPECT_TRUE(eof);
// delete block;
// delete p_reader;
delete reader;
}
} // namespace vectorized
} // namespace doris