database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[libhdfs] Add errno for hdfs writer. when no dir, hdfs writer open failed, the dir need to be created. (#7050)

Browse Source 
1. Add errno message for hdfs writer failed.
2. When call openWrite for hdfs, the dir will be created when it doesn't exist,
This commit is contained in:
pengxiangyu
2021-11-11 15:21:21 +08:00
committed by GitHub
parent c47beb4d3a
commit 632f8fcc75
4 changed files with 41 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
be/src/exec/hdfs_file_reader.cpp
View File

@ -20,6 +20,7 @@
#include <unistd.h>
#include "common/logging.h"
#include "service/backend_options.h"
namespace doris {
@ -80,7 +81,8 @@ Status HdfsFileReader::open() {
_hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_RDONLY, 0, 0, 0);
if (_hdfs_file == nullptr) {
std::stringstream ss;
ss << "open file failed. " << _namenode << _path;
ss << "open file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
<< _namenode << _path << ", err: " << strerror(errno);;
return Status::InternalError(ss.str());
}
LOG(INFO) << "open file. " << _namenode << _path;
@ -139,7 +141,8 @@ Status HdfsFileReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r
int ret = hdfsSeek(_hdfs_fs, _hdfs_file, position);
if (ret != 0) { // check fseek return value
std::stringstream ss;
ss << "hdfsSeek failed. " << _namenode << _path;
ss << "hdfsSeek failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
<< _namenode << _path << ", err: " << strerror(errno);;
return Status::InternalError(ss.str());
}
}
@ -147,7 +150,8 @@ Status HdfsFileReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r
*bytes_read = hdfsRead(_hdfs_fs, _hdfs_file, out, nbytes);
if (*bytes_read < 0) {
std::stringstream ss;
ss << "Read hdfs file failed. " << _namenode << _path;
ss << "Read hdfs file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
<< _namenode << _path << ", err: " << strerror(errno);;
return Status::InternalError(ss.str());
}
_current_offset += *bytes_read; // save offset with file
@ -165,7 +169,7 @@ int64_t HdfsFileReader::size() {
}
hdfsFileInfo* file_info = hdfsGetPathInfo(_hdfs_fs, _path.c_str());
if (file_info == nullptr) {
LOG(WARNING) << "get path info failed: " << _namenode << _path;
LOG(WARNING) << "get path info failed: " << _namenode << _path << ", err: " << strerror(errno);;
close();
return -1;
}
@ -183,8 +187,8 @@ Status HdfsFileReader::seek(int64_t position) {
if (res != 0) {
char err_buf[64];
std::stringstream ss;
ss << "Seek to offset failed. offset=" << position
<< ", error=" << strerror_r(errno, err_buf, 64);
ss << "Seek to offset failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
<< " offset=" << position << ", err: " << strerror(errno);
return Status::InternalError(ss.str());
}
return Status::OK();

32
be/src/exec/hdfs_writer.cpp
View File

@ -17,7 +17,10 @@
#include "exec/hdfs_writer.h"
#include <filesystem>
#include "common/logging.h"
#include "service/backend_options.h"
namespace doris {
const static std::string FS_KEY = "fs.defaultFS";
@ -47,11 +50,30 @@ Status HDFSWriter::open() {
// the path already exists
return Status::AlreadyExist(_path + " already exists.");
}
std::filesystem::path hdfs_path(_path);
std::string hdfs_dir = hdfs_path.parent_path().string();
exists = hdfsExists(_hdfs_fs, hdfs_dir.c_str());
if (exists != 0) {
LOG(INFO) << "hdfs dir doesn't exist, create it: " << hdfs_dir;
int ret = hdfsCreateDirectory(_hdfs_fs, hdfs_dir.c_str());
if (ret != 0) {
std::stringstream ss;
ss << "create dir failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
<< " namenode: " << _namenode << " path: " << hdfs_dir
<< ", err: " << strerror(errno);
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
}
// open file
_hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_WRONLY, 0, 0, 0);
if (_hdfs_file == nullptr) {
std::stringstream ss;
ss << "open file failed. namenode:" << _namenode << " path:" << _path;
ss << "open file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
<< " namenode:" << _namenode << " path:" << _path
<< ", err: " << strerror(errno);
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
LOG(INFO) << "open file. namenode:" << _namenode << " path:" << _path;
@ -66,7 +88,9 @@ Status HDFSWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_len
int32_t result = hdfsWrite(_hdfs_fs, _hdfs_file, buf, buf_len);
if (result < 0) {
std::stringstream ss;
ss << "write file failed. namenode:" << _namenode << " path:" << _path;
ss << "write file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
<< "namenode:" << _namenode << " path:" << _path
<< ", err: " << strerror(errno);
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
@ -91,7 +115,9 @@ Status HDFSWriter::close() {
int result = hdfsFlush(_hdfs_fs, _hdfs_file);
if (result == -1) {
std::stringstream ss;
ss << "failed to flush hdfs file. namenode:" << _namenode << " path:" << _path;
ss << "failed to flush hdfs file. " << "(BE: " << BackendOptions::get_localhost() << ")"
<< "namenode:" << _namenode << " path:" << _path
<< ", err: " << strerror(errno);
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}

1
be/src/exprs/bitmap_function.h
View File

@ -110,6 +110,7 @@ public:
const BigIntVal& range_start, const BigIntVal& cardinality_limit);
static StringVal sub_bitmap(FunctionContext* ctx, const StringVal& src,
const BigIntVal& offset, const BigIntVal& cardinality_limit);
};
} // namespace doris
#endif //DORIS_BE_SRC_QUERY_EXPRS_BITMAP_FUNCTION_H

2
be/src/util/bitmap_value.h
View File

@ -1489,7 +1489,7 @@ public:
break;
}
}
return count;
return count;
}
/**