database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](cooldown) Fix hdfs path (#33315)

Browse Source
This commit is contained in:
plat1ko
2024-04-09 12:55:53 +08:00
committed by GitHub
parent a1f80eaa7a
commit 97850cf2bb
2 changed files with 16 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
be/src/io/fs/hdfs_file_system.cpp
View File

@ -283,11 +283,12 @@ Status HdfsFileSystem::list_impl(const Path& path, bool only_file, std::vector<F
if (only_file && file.mKind == kObjectKindDirectory) {
continue;
}
FileInfo file_info;
file_info.file_name = file.mName;
auto& file_info = files->emplace_back();
std::string_view fname(file.mName);
fname.remove_prefix(fname.rfind('/') + 1);
file_info.file_name = fname;
file_info.file_size = file.mSize;
file_info.is_file = (file.mKind != kObjectKindDirectory);
files->emplace_back(std::move(file_info));
}
hdfsFreeFileInfo(hdfs_file_info, numEntries);
return Status::OK();

22
be/src/util/hdfs_util.cpp
View File

@ -42,17 +42,19 @@ hdfsFS HDFSHandle::create_hdfs_fs(HDFSCommonBuilder& hdfs_builder) {
}
Path convert_path(const Path& path, const std::string& namenode) {
Path real_path(path);
if (path.string().find(namenode) != std::string::npos) {
std::string real_path_str = path.string().substr(namenode.size());
if (!real_path_str.starts_with("/")) {
// The real path must starts with "/"
// Or the hadoop client will add a prefix like "/user/hadoop".
real_path_str = "/" + real_path_str;
}
real_path = real_path_str;
std::string fs_path;
if (path.native().starts_with(namenode)) {
// `path` is URI format, remove the namenode part in `path`
fs_path = path.native().substr(namenode.size());
} else {
fs_path = path;
}
return real_path;
// Always use absolute path (start with '/') in hdfs
if (fs_path.empty() || fs_path[0] != '/') {
fs_path.insert(fs_path.begin(), '/');
}
return fs_path;
}
bool is_hdfs(const std::string& path_or_fs) {