From 632f8fcc754d23f5674e32ddef2ddec9664b759e Mon Sep 17 00:00:00 2001 From: pengxiangyu Date: Thu, 11 Nov 2021 15:21:21 +0800 Subject: [PATCH] [libhdfs] Add errno for hdfs writer. when no dir, hdfs writer open failed, the dir need to be created. (#7050) 1. Add errno message for hdfs writer failed. 2. When call openWrite for hdfs, the dir will be created when it doesn't exist, --- be/src/exec/hdfs_file_reader.cpp | 16 ++++++++++------ be/src/exec/hdfs_writer.cpp | 32 +++++++++++++++++++++++++++++--- be/src/exprs/bitmap_function.h | 1 + be/src/util/bitmap_value.h | 2 +- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/be/src/exec/hdfs_file_reader.cpp b/be/src/exec/hdfs_file_reader.cpp index a6f509ff1c..0ba1a3ab15 100644 --- a/be/src/exec/hdfs_file_reader.cpp +++ b/be/src/exec/hdfs_file_reader.cpp @@ -20,6 +20,7 @@ #include #include "common/logging.h" +#include "service/backend_options.h" namespace doris { @@ -80,7 +81,8 @@ Status HdfsFileReader::open() { _hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_RDONLY, 0, 0, 0); if (_hdfs_file == nullptr) { std::stringstream ss; - ss << "open file failed. " << _namenode << _path; + ss << "open file failed. " << "(BE: " << BackendOptions::get_localhost() << ")" + << _namenode << _path << ", err: " << strerror(errno);; return Status::InternalError(ss.str()); } LOG(INFO) << "open file. " << _namenode << _path; @@ -139,7 +141,8 @@ Status HdfsFileReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r int ret = hdfsSeek(_hdfs_fs, _hdfs_file, position); if (ret != 0) { // check fseek return value std::stringstream ss; - ss << "hdfsSeek failed. " << _namenode << _path; + ss << "hdfsSeek failed. " << "(BE: " << BackendOptions::get_localhost() << ")" + << _namenode << _path << ", err: " << strerror(errno);; return Status::InternalError(ss.str()); } } @@ -147,7 +150,8 @@ Status HdfsFileReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r *bytes_read = hdfsRead(_hdfs_fs, _hdfs_file, out, nbytes); if (*bytes_read < 0) { std::stringstream ss; - ss << "Read hdfs file failed. " << _namenode << _path; + ss << "Read hdfs file failed. " << "(BE: " << BackendOptions::get_localhost() << ")" + << _namenode << _path << ", err: " << strerror(errno);; return Status::InternalError(ss.str()); } _current_offset += *bytes_read; // save offset with file @@ -165,7 +169,7 @@ int64_t HdfsFileReader::size() { } hdfsFileInfo* file_info = hdfsGetPathInfo(_hdfs_fs, _path.c_str()); if (file_info == nullptr) { - LOG(WARNING) << "get path info failed: " << _namenode << _path; + LOG(WARNING) << "get path info failed: " << _namenode << _path << ", err: " << strerror(errno);; close(); return -1; } @@ -183,8 +187,8 @@ Status HdfsFileReader::seek(int64_t position) { if (res != 0) { char err_buf[64]; std::stringstream ss; - ss << "Seek to offset failed. offset=" << position - << ", error=" << strerror_r(errno, err_buf, 64); + ss << "Seek to offset failed. " << "(BE: " << BackendOptions::get_localhost() << ")" + << " offset=" << position << ", err: " << strerror(errno); return Status::InternalError(ss.str()); } return Status::OK(); diff --git a/be/src/exec/hdfs_writer.cpp b/be/src/exec/hdfs_writer.cpp index a00413f358..d608a77bc6 100644 --- a/be/src/exec/hdfs_writer.cpp +++ b/be/src/exec/hdfs_writer.cpp @@ -17,7 +17,10 @@ #include "exec/hdfs_writer.h" +#include + #include "common/logging.h" +#include "service/backend_options.h" namespace doris { const static std::string FS_KEY = "fs.defaultFS"; @@ -47,11 +50,30 @@ Status HDFSWriter::open() { // the path already exists return Status::AlreadyExist(_path + " already exists."); } + + std::filesystem::path hdfs_path(_path); + std::string hdfs_dir = hdfs_path.parent_path().string(); + exists = hdfsExists(_hdfs_fs, hdfs_dir.c_str()); + if (exists != 0) { + LOG(INFO) << "hdfs dir doesn't exist, create it: " << hdfs_dir; + int ret = hdfsCreateDirectory(_hdfs_fs, hdfs_dir.c_str()); + if (ret != 0) { + std::stringstream ss; + ss << "create dir failed. " << "(BE: " << BackendOptions::get_localhost() << ")" + << " namenode: " << _namenode << " path: " << hdfs_dir + << ", err: " << strerror(errno); + LOG(WARNING) << ss.str(); + return Status::InternalError(ss.str()); + } + } // open file _hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_WRONLY, 0, 0, 0); if (_hdfs_file == nullptr) { std::stringstream ss; - ss << "open file failed. namenode:" << _namenode << " path:" << _path; + ss << "open file failed. " << "(BE: " << BackendOptions::get_localhost() << ")" + << " namenode:" << _namenode << " path:" << _path + << ", err: " << strerror(errno); + LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } LOG(INFO) << "open file. namenode:" << _namenode << " path:" << _path; @@ -66,7 +88,9 @@ Status HDFSWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_len int32_t result = hdfsWrite(_hdfs_fs, _hdfs_file, buf, buf_len); if (result < 0) { std::stringstream ss; - ss << "write file failed. namenode:" << _namenode << " path:" << _path; + ss << "write file failed. " << "(BE: " << BackendOptions::get_localhost() << ")" + << "namenode:" << _namenode << " path:" << _path + << ", err: " << strerror(errno); LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } @@ -91,7 +115,9 @@ Status HDFSWriter::close() { int result = hdfsFlush(_hdfs_fs, _hdfs_file); if (result == -1) { std::stringstream ss; - ss << "failed to flush hdfs file. namenode:" << _namenode << " path:" << _path; + ss << "failed to flush hdfs file. " << "(BE: " << BackendOptions::get_localhost() << ")" + << "namenode:" << _namenode << " path:" << _path + << ", err: " << strerror(errno); LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } diff --git a/be/src/exprs/bitmap_function.h b/be/src/exprs/bitmap_function.h index 363b9b7012..2cba6854c7 100644 --- a/be/src/exprs/bitmap_function.h +++ b/be/src/exprs/bitmap_function.h @@ -110,6 +110,7 @@ public: const BigIntVal& range_start, const BigIntVal& cardinality_limit); static StringVal sub_bitmap(FunctionContext* ctx, const StringVal& src, const BigIntVal& offset, const BigIntVal& cardinality_limit); + }; } // namespace doris #endif //DORIS_BE_SRC_QUERY_EXPRS_BITMAP_FUNCTION_H diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 6d3710f4b1..34f0aba762 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1489,7 +1489,7 @@ public: break; } } - return count; + return count; } /**