f66f161017
Sometimes, the partitions of a hive table may on different storage, eg, some is on HDFS, others on object storage(cos, etc).
This PR mainly changes:
1. Fix the bug of accessing files via cosn.
2. Add a new field `fs_name` in TFileRangeDesc
This is because, when accessing a file, the BE will get a hdfs client from hdfs client cache, and different file in one query
request may have different fs name, eg, some of are `hdfs://`, some of are `cosn://`, so we need to specify fs name
for each file, otherwise, it may return error:
`reason: IllegalArgumentException: Wrong FS: cosn://doris-build-1308700295/xxxx, expected: hdfs://[172.xxxx:4007](http://172.xxxxx:4007/)`
59 lines
1.8 KiB
C++
59 lines
1.8 KiB
C++
// Licensed to the Apache Software Foundation (ASF) under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
#include "util/hdfs_util.h"
|
|
|
|
#include <ostream>
|
|
|
|
#include "common/logging.h"
|
|
#include "io/fs/err_utils.h"
|
|
#include "io/hdfs_builder.h"
|
|
|
|
namespace doris {
|
|
namespace io {
|
|
|
|
HDFSHandle& HDFSHandle::instance() {
|
|
static HDFSHandle hdfs_handle;
|
|
return hdfs_handle;
|
|
}
|
|
|
|
hdfsFS HDFSHandle::create_hdfs_fs(HDFSCommonBuilder& hdfs_builder) {
|
|
hdfsFS hdfs_fs = hdfsBuilderConnect(hdfs_builder.get());
|
|
if (hdfs_fs == nullptr) {
|
|
LOG(WARNING) << "connect to hdfs failed."
|
|
<< ", error: " << hdfs_error();
|
|
return nullptr;
|
|
}
|
|
return hdfs_fs;
|
|
}
|
|
|
|
Path convert_path(const Path& path, const std::string& namenode) {
|
|
Path real_path(path);
|
|
if (path.string().find(namenode) != std::string::npos) {
|
|
std::string real_path_str = path.string().substr(namenode.size());
|
|
real_path = real_path_str;
|
|
}
|
|
return real_path;
|
|
}
|
|
|
|
bool is_hdfs(const std::string& path_or_fs) {
|
|
return path_or_fs.rfind("hdfs://") == 0;
|
|
}
|
|
|
|
} // namespace io
|
|
} // namespace doris
|