database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](multi catalog)Return emtpy block while external table scanner couldn't find the file (#14997)

Browse Source 
FE file path cache for external table may out of date. In this case, BE may fail to find the not exist file from FE cache. 
This pr is to handle this case: instead of throw an error message to the user, we return empty result set to the user.
This commit is contained in:
Jibing-Li
2022-12-16 09:36:35 +08:00
committed by GitHub
parent 52e09e6b04
commit e0d528980f
3 changed files with 17 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
be/src/io/hdfs_file_reader.cpp
View File

@ -72,6 +72,9 @@ Status HdfsFileReader::open() {
RETURN_IF_ERROR(HdfsFsCache::instance()->get_connection(_hdfs_params, &_fs_handle));
_hdfs_fs = _fs_handle->hdfs_fs;
if (hdfsExists(_hdfs_fs, _path.c_str()) != 0) {
return Status::NotFound("{} not exists!", _path);
}
_hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_RDONLY, 0, 0, 0);
if (_hdfs_file == nullptr) {
if (_fs_handle->from_cache) {

12
be/src/vec/exec/scan/scanner_scheduler.cpp
View File

@ -26,6 +26,7 @@
#include "vec/core/block.h"
#include "vec/exec/scan/vscanner.h"
#include "vec/exprs/vexpr.h"
#include "vfile_scanner.h"
namespace doris::vectorized {
@ -232,12 +233,21 @@ void ScannerScheduler::_scanner_scan(ScannerScheduler* scheduler, ScannerContext
auto block = ctx->get_free_block(&get_free_block);
status = scanner->get_block(state, block, &eos);
VLOG_ROW << "VOlapScanNode input rows: " << block->rows() << ", eos: " << eos;
if (!status.ok()) {
// The VFileScanner for external table may try to open not exist files,
// Because FE file cache for external table may out of date.
if (!status.ok() && (typeid(*scanner) == typeid(doris::vectorized::VFileScanner) &&
!status.is<ErrorCode::NOT_FOUND>())) {
LOG(WARNING) << "Scan thread read VOlapScanner failed: " << status.to_string();
// Add block ptr in blocks, prevent mem leak in read failed
blocks.push_back(block);
break;
}
if (status.is<ErrorCode::NOT_FOUND>()) {
// The only case in this if branch is external table file delete and fe cache has not been updated yet.
// Set status to OK.
status = Status::OK();
eos = true;
}
raw_bytes_read += block->bytes();
num_rows_in_block += block->rows();

3
be/src/vec/exec/scan/vfile_scanner.cpp
View File

@ -531,6 +531,9 @@ Status VFileScanner::_get_next_reader() {
if (init_status.is<END_OF_FILE>()) {
continue;
} else if (!init_status.ok()) {
if (init_status.is<ErrorCode::NOT_FOUND>()) {
return init_status;
}
return Status::InternalError("failed to init reader for file {}, err: {}", range.path,
init_status.to_string());
}