diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 53a09999cd..78001d53c3 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1404,10 +1404,14 @@ void TaskWorkerPool::_upload_worker_thread_callback() { std::map> tablet_files; std::unique_ptr loader = std::make_unique( _env, upload_request.job_id, agent_task_req.signature, upload_request.broker_addr, - upload_request.broker_prop, + upload_request.broker_prop); + Status status = loader->init( upload_request.__isset.storage_backend ? upload_request.storage_backend - : TStorageBackendType::type::BROKER); - Status status = loader->upload(upload_request.src_dest_map, &tablet_files); + : TStorageBackendType::type::BROKER, + upload_request.__isset.location ? upload_request.location : ""); + if (status.ok()) { + status = loader->upload(upload_request.src_dest_map, &tablet_files); + } if (!status.ok()) { LOG_WARNING("failed to upload") @@ -1456,10 +1460,14 @@ void TaskWorkerPool::_download_worker_thread_callback() { std::unique_ptr loader = std::make_unique( _env, download_request.job_id, agent_task_req.signature, - download_request.broker_addr, download_request.broker_prop, + download_request.broker_addr, download_request.broker_prop); + Status status = loader->init( download_request.__isset.storage_backend ? download_request.storage_backend - : TStorageBackendType::type::BROKER); - Status status = loader->download(download_request.src_dest_map, &downloaded_tablet_ids); + : TStorageBackendType::type::BROKER, + download_request.__isset.location ? download_request.location : ""); + if (status.ok()) { + status = loader->download(download_request.src_dest_map, &downloaded_tablet_ids); + } if (!status.ok()) { LOG_WARNING("failed to download") @@ -1769,6 +1777,7 @@ void TaskWorkerPool::_push_storage_policy_worker_thread_callback() { continue; } if (resource.__isset.s3_storage_param) { + Status st; S3Conf s3_conf; s3_conf.ak = std::move(resource.s3_storage_param.ak); s3_conf.sk = std::move(resource.s3_storage_param.sk); @@ -1779,14 +1788,13 @@ void TaskWorkerPool::_push_storage_policy_worker_thread_callback() { s3_conf.connect_timeout_ms = resource.s3_storage_param.conn_timeout_ms; s3_conf.max_connections = resource.s3_storage_param.max_conn; s3_conf.request_timeout_ms = resource.s3_storage_param.request_timeout_ms; - std::shared_ptr s3_fs; + std::shared_ptr fs; if (existed_resource.fs == nullptr) { - s3_fs = io::S3FileSystem::create(s3_conf, std::to_string(resource.id)); + st = io::S3FileSystem::create(s3_conf, std::to_string(resource.id), &fs); } else { - s3_fs = std::static_pointer_cast(existed_resource.fs); - s3_fs->set_conf(s3_conf); + fs = std::static_pointer_cast(existed_resource.fs); + fs->set_conf(s3_conf); } - auto st = s3_fs->connect(); if (!st.ok()) { LOG(WARNING) << "update s3 resource failed: " << st; } else { @@ -1794,7 +1802,7 @@ void TaskWorkerPool::_push_storage_policy_worker_thread_callback() { .tag("resource_id", resource.id) .tag("resource_name", resource.name) .tag("s3_conf", s3_conf.to_string()); - put_storage_resource(resource.id, {std::move(s3_fs), resource.version}); + put_storage_resource(resource.id, {std::move(fs), resource.version}); } } else { LOG(WARNING) << "unknown resource=" << resource; diff --git a/be/src/common/status.cpp b/be/src/common/status.cpp index 32aafe35ed..85f4d69938 100644 --- a/be/src/common/status.cpp +++ b/be/src/common/status.cpp @@ -8,6 +8,7 @@ #include #include "gen_cpp/types.pb.h" // for PStatus +#include "service/backend_options.h" namespace doris { @@ -52,8 +53,8 @@ void Status::to_thrift(TStatus* s) const { return; } s->status_code = (int16_t)_code > 0 ? (TStatusCode::type)_code : TStatusCode::INTERNAL_ERROR; - s->error_msgs.push_back( - fmt::format("[{}]{}", code_as_string(), _err_msg ? _err_msg->_msg : "")); + s->error_msgs.push_back(fmt::format("({})[{}]{}", BackendOptions::get_localhost(), + code_as_string(), _err_msg ? _err_msg->_msg : "")); s->__isset.error_msgs = true; } @@ -67,7 +68,8 @@ void Status::to_protobuf(PStatus* s) const { s->clear_error_msgs(); s->set_status_code((int)_code); if (!ok() && _err_msg) { - s->add_error_msgs(_err_msg->_msg); + s->add_error_msgs(fmt::format("({})[{}]{}", BackendOptions::get_localhost(), + code_as_string(), _err_msg ? _err_msg->_msg : "")); } } diff --git a/be/src/common/status.h b/be/src/common/status.h index 772eaa2230..12aa7b6b6c 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -13,7 +13,6 @@ #include "common/compiler_util.h" #include "gen_cpp/Status_types.h" // for TStatus -#include "service/backend_options.h" #ifdef ENABLE_STACKTRACE #include "util/stack_util.h" #endif @@ -301,7 +300,6 @@ public: if (rhs._err_msg) { _err_msg = std::make_unique(*rhs._err_msg); } - _be_ip = rhs._be_ip; return *this; } @@ -479,12 +477,10 @@ private: #endif }; std::unique_ptr _err_msg; - std::string_view _be_ip = BackendOptions::get_localhost(); }; inline std::ostream& operator<<(std::ostream& ostr, const Status& status) { ostr << '[' << status.code_as_string() << ']'; - ostr << '[' << status._be_ip << ']'; ostr << (status._err_msg ? status._err_msg->_msg : ""); #ifdef ENABLE_STACKTRACE if (status._err_msg && !status._err_msg->_stack.empty()) { @@ -510,7 +506,7 @@ inline std::string Status::to_string() const { } while (false) #define RETURN_ERROR_IF_NON_VEC \ - return Status::NotSupported("Non-vectorized engine is not supported since Doris 1.3+."); + return Status::NotSupported("Non-vectorized engine is not supported since Doris 2.0."); // End _get_next_span after last call to get_next method #define RETURN_IF_ERROR_AND_CHECK_SPAN(stmt, get_next_span, done) \ diff --git a/be/src/exec/arrow/arrow_reader.cpp b/be/src/exec/arrow/arrow_reader.cpp index c03d7791dd..d0dbf3f992 100644 --- a/be/src/exec/arrow/arrow_reader.cpp +++ b/be/src/exec/arrow/arrow_reader.cpp @@ -23,6 +23,7 @@ #include "common/logging.h" #include "gen_cpp/PaloBrokerService_types.h" #include "gen_cpp/TPaloBrokerService.h" +#include "io/io_common.h" #include "olap/iterators.h" #include "runtime/broker_mgr.h" #include "runtime/client_cache.h" @@ -216,8 +217,7 @@ arrow::Result ArrowFile::ReadAt(int64_t position, int64_t nbytes, void* while (bytes_read < nbytes) { size_t reads = 0; Slice file_slice((uint8_t*)out, nbytes); - IOContext io_ctx; - Status result = _file_reader->read_at(_pos, file_slice, io_ctx, &reads); + Status result = _file_reader->read_at(_pos, file_slice, &reads); if (!result.ok()) { return arrow::Status::IOError("Readat failed."); } diff --git a/be/src/exec/line_reader.h b/be/src/exec/line_reader.h index 80a8f94977..0efdf43e57 100644 --- a/be/src/exec/line_reader.h +++ b/be/src/exec/line_reader.h @@ -20,12 +20,15 @@ #include "common/status.h" namespace doris { - +namespace io { +class IOContext; +} // This class is used for CSV scanner, to read content line by line class LineReader { public: virtual ~LineReader() = default; - virtual Status read_line(const uint8_t** ptr, size_t* size, bool* eof) = 0; + virtual Status read_line(const uint8_t** ptr, size_t* size, bool* eof, + const io::IOContext* io_ctx) = 0; virtual void close() = 0; }; diff --git a/be/src/http/web_page_handler.cpp b/be/src/http/web_page_handler.cpp index 9c81f2a1c7..4a5b873a08 100644 --- a/be/src/http/web_page_handler.cpp +++ b/be/src/http/web_page_handler.cpp @@ -30,7 +30,6 @@ #include "http/http_response.h" #include "http/http_status.h" #include "http/utils.h" -#include "olap/file_helper.h" #include "util/cpu_info.h" #include "util/debug_util.h" #include "util/disk_info.h" diff --git a/be/src/io/CMakeLists.txt b/be/src/io/CMakeLists.txt index 3dae0d9263..1c953f1e55 100644 --- a/be/src/io/CMakeLists.txt +++ b/be/src/io/CMakeLists.txt @@ -22,26 +22,27 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/io") set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/io") set(IO_FILES - broker_writer.cpp - buffered_reader.cpp file_factory.cpp hdfs_builder.cpp - hdfs_writer.cpp - local_file_writer.cpp - s3_writer.cpp fs/file_reader_options.cpp - fs/local_file_reader.cpp + fs/file_reader.cpp + fs/file_system.cpp + fs/remote_file_system.cpp fs/local_file_system.cpp + fs/local_file_reader.cpp fs/local_file_writer.cpp - fs/s3_file_reader.cpp fs/s3_file_system.cpp + fs/s3_file_reader.cpp fs/s3_file_writer.cpp fs/hdfs_file_system.cpp fs/hdfs_file_reader.cpp + fs/hdfs_file_writer.cpp fs/broker_file_system.cpp fs/broker_file_reader.cpp - fs/remote_file_system.cpp + fs/broker_file_writer.cpp + fs/buffered_reader.cpp fs/stream_load_pipe.cpp + fs/fs_utils.cpp cache/dummy_file_cache.cpp cache/file_cache.cpp cache/file_cache_manager.cpp diff --git a/be/src/io/cache/block/block_file_segment.cpp b/be/src/io/cache/block/block_file_segment.cpp index ae98270897..020fc85de1 100644 --- a/be/src/io/cache/block/block_file_segment.cpp +++ b/be/src/io/cache/block/block_file_segment.cpp @@ -26,7 +26,6 @@ #include #include "common/status.h" -#include "io/fs/file_reader.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "olap/iterators.h" @@ -171,13 +170,11 @@ Status FileBlock::read_at(Slice buffer, size_t offset) { std::lock_guard segment_lock(_mutex); if (!_cache_reader) { auto download_path = get_path_in_local_cache(); - RETURN_IF_ERROR( - global_local_filesystem()->open_file(download_path, &_cache_reader, nullptr)); + RETURN_IF_ERROR(global_local_filesystem()->open_file(download_path, &_cache_reader)); } } size_t bytes_reads = buffer.size; - IOContext io_ctx; - RETURN_IF_ERROR(_cache_reader->read_at(offset, buffer, io_ctx, &bytes_reads)); + RETURN_IF_ERROR(_cache_reader->read_at(offset, buffer, &bytes_reads)); DCHECK(bytes_reads == buffer.size); return Status::OK(); } diff --git a/be/src/io/cache/block/block_lru_file_cache.cpp b/be/src/io/cache/block/block_lru_file_cache.cpp index e2b1063007..22dbfe80f0 100644 --- a/be/src/io/cache/block/block_lru_file_cache.cpp +++ b/be/src/io/cache/block/block_lru_file_cache.cpp @@ -790,9 +790,8 @@ std::string LRUFileCache::read_file_cache_version() const { fs->file_size(version_path, &file_size); char version[file_size]; - IOContext io_ctx; - fs->open_file(version_path, &version_reader, &io_ctx); - version_reader->read_at(0, Slice(version, file_size), io_ctx, &file_size); + fs->open_file(version_path, &version_reader); + version_reader->read_at(0, Slice(version, file_size), &file_size); version_reader->close(); return std::string(version, file_size); } diff --git a/be/src/io/cache/block/cached_remote_file_reader.cpp b/be/src/io/cache/block/cached_remote_file_reader.cpp index 1b82916e11..c3a3eee9f6 100644 --- a/be/src/io/cache/block/cached_remote_file_reader.cpp +++ b/be/src/io/cache/block/cached_remote_file_reader.cpp @@ -28,8 +28,8 @@ namespace doris { namespace io { CachedRemoteFileReader::CachedRemoteFileReader(FileReaderSPtr remote_file_reader, - const std::string& cache_path, IOContext* io_ctx) - : _remote_file_reader(std::move(remote_file_reader)), _io_ctx(io_ctx) { + const std::string& cache_path) + : _remote_file_reader(std::move(remote_file_reader)) { _cache_key = IFileCache::hash(cache_path); _cache = FileCacheFactory::instance().get_by_path(_cache_key); _disposable_cache = FileCacheFactory::instance().get_disposable_cache(_cache_key); @@ -57,21 +57,10 @@ std::pair CachedRemoteFileReader::_align_size(size_t offset, return std::make_pair(align_left, align_size); } -Status CachedRemoteFileReader::read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) { - if (bthread_self() == 0) { - return read_at_impl(offset, result, io_ctx, bytes_read); - } - Status s; - auto task = [&] { s = read_at_impl(offset, result, io_ctx, bytes_read); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - -Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, - const IOContext& /*io_ctx*/, size_t* bytes_read) { +Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) { DCHECK(!closed()); - DCHECK(_io_ctx); + DCHECK(io_ctx); if (offset > size()) { return Status::IOError( fmt::format("offset exceeds file size(offset: {), file size: {}, path: {})", offset, @@ -83,23 +72,23 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, *bytes_read = 0; return Status::OK(); } - CloudFileCachePtr cache = _io_ctx->use_disposable_cache ? _disposable_cache : _cache; + CloudFileCachePtr cache = io_ctx->use_disposable_cache ? _disposable_cache : _cache; // cache == nullptr since use_disposable_cache = true and don't set disposable cache in conf if (cache == nullptr) { - return _remote_file_reader->read_at(offset, result, *_io_ctx, bytes_read); + return _remote_file_reader->read_at(offset, result, bytes_read, io_ctx); } ReadStatistics stats; stats.bytes_read = bytes_req; // if state == nullptr, the method is called for read footer // if state->read_segment_index, read all the end of file size_t align_left = offset, align_size = size() - offset; - if (!_io_ctx->read_segment_index) { + if (!io_ctx->read_segment_index) { auto pair = _align_size(offset, bytes_req); align_left = pair.first; align_size = pair.second; } - bool is_persistent = _io_ctx->is_persistent; - TUniqueId query_id = _io_ctx->query_id ? *(_io_ctx->query_id) : TUniqueId(); + bool is_persistent = io_ctx->is_persistent; + TUniqueId query_id = io_ctx->query_id ? *(io_ctx->query_id) : TUniqueId(); FileBlocksHolder holder = cache->get_or_set(_cache_key, align_left, align_size, is_persistent, query_id); std::vector empty_segments; @@ -122,8 +111,8 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, empty_end = empty_segments.back()->range().right; size_t size = empty_end - empty_start + 1; std::unique_ptr buffer(new char[size]); - RETURN_IF_ERROR(_remote_file_reader->read_at(empty_start, Slice(buffer.get(), size), - *_io_ctx, &size)); + RETURN_IF_ERROR(_remote_file_reader->read_at(empty_start, Slice(buffer.get(), size), &size, + io_ctx)); for (auto& segment : empty_segments) { if (segment->state() == FileBlock::State::SKIP_CACHE) { continue; @@ -194,7 +183,7 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset, Slice result, current_offset = right + 1; } DCHECK(*bytes_read == bytes_req); - _update_state(stats, _io_ctx->file_cache_stats); + _update_state(stats, io_ctx->file_cache_stats); DorisMetrics::instance()->s3_bytes_read_total->increment(*bytes_read); return Status::OK(); } diff --git a/be/src/io/cache/block/cached_remote_file_reader.h b/be/src/io/cache/block/cached_remote_file_reader.h index 496f151404..179a61a066 100644 --- a/be/src/io/cache/block/cached_remote_file_reader.h +++ b/be/src/io/cache/block/cached_remote_file_reader.h @@ -31,18 +31,12 @@ namespace io { class CachedRemoteFileReader final : public FileReader { public: - CachedRemoteFileReader(FileReaderSPtr remote_file_reader, const std::string& cache_path, - IOContext* io_ctx); + CachedRemoteFileReader(FileReaderSPtr remote_file_reader, const std::string& cache_path); ~CachedRemoteFileReader() override; Status close() override; - Status read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) override; - - Status read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, size_t* bytes_read); - const Path& path() const override { return _remote_file_reader->path(); } size_t size() const override { return _remote_file_reader->size(); } @@ -51,6 +45,10 @@ public: FileSystemSPtr fs() const override { return _remote_file_reader->fs(); } +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override; + private: std::pair _align_size(size_t offset, size_t size) const; @@ -59,8 +57,6 @@ private: CloudFileCachePtr _cache; CloudFileCachePtr _disposable_cache; - IOContext* _io_ctx; - struct ReadStatistics { bool hit_cache = false; int64_t bytes_read = 0; diff --git a/be/src/io/cache/dummy_file_cache.h b/be/src/io/cache/dummy_file_cache.h index 7af7451130..653d47b1d6 100644 --- a/be/src/io/cache/dummy_file_cache.h +++ b/be/src/io/cache/dummy_file_cache.h @@ -38,11 +38,6 @@ public: Status close() override { return Status::OK(); } - Status read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) override { - return Status::NotSupported("dummy file cache only used for GC"); - } - const Path& path() const override { return _cache_dir; } size_t size() const override { return 0; } @@ -71,6 +66,12 @@ public: FileSystemSPtr fs() const override { return nullptr; } +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override { + return Status::NotSupported("dummy file cache only used for GC"); + } + private: void _add_file_cache(const Path& data_file); void _load(); diff --git a/be/src/io/cache/file_cache.cpp b/be/src/io/cache/file_cache.cpp index 5a4382a6f7..4dc08b52c9 100644 --- a/be/src/io/cache/file_cache.cpp +++ b/be/src/io/cache/file_cache.cpp @@ -21,6 +21,7 @@ #include "common/status.h" #include "gutil/strings/util.h" #include "io/fs/local_file_system.h" +#include "io/fs/local_file_writer.h" #include "olap/iterators.h" namespace doris { @@ -50,10 +51,8 @@ Status FileCache::download_cache_to_local(const Path& cache_file, const Path& ca } Slice file_slice(file_buf, need_req_size); size_t bytes_read = 0; - IOContext io_ctx; RETURN_NOT_OK_STATUS_WITH_WARN( - remote_file_reader->read_at(offset + count_bytes_read, file_slice, io_ctx, - &bytes_read), + remote_file_reader->read_at(offset + count_bytes_read, file_slice, &bytes_read), fmt::format("read remote file failed. {}. offset: {}, request size: {}", remote_file_reader->path().native(), offset + count_bytes_read, need_req_size)); @@ -129,19 +128,20 @@ Status FileCache::_get_dir_files_and_remove_unfinished(const Path& cache_dir, } // list all files - std::vector cache_file_names; + std::vector cache_files; + bool exists = true; RETURN_NOT_OK_STATUS_WITH_WARN( - io::global_local_filesystem()->list(cache_dir, &cache_file_names), + io::global_local_filesystem()->list(cache_dir, true, &cache_files, &exists), fmt::format("List dir failed: {}", cache_dir.native())) // separate DATA file and DONE file std::set cache_names_temp; std::list done_names_temp; - for (auto& cache_file_name : cache_file_names) { - if (ends_with(cache_file_name.native(), CACHE_DONE_FILE_SUFFIX)) { - done_names_temp.push_back(std::move(cache_file_name)); + for (auto& cache_file : cache_files) { + if (ends_with(cache_file.file_name, CACHE_DONE_FILE_SUFFIX)) { + done_names_temp.push_back(cache_file.file_name); } else { - cache_names_temp.insert(std::move(cache_file_name)); + cache_names_temp.insert(cache_file.file_name); } } @@ -181,11 +181,12 @@ Status FileCache::_check_and_delete_empty_dir(const Path& cache_dir) { return Status::OK(); } - std::vector cache_file_names; + std::vector cache_files; + bool exists = true; RETURN_NOT_OK_STATUS_WITH_WARN( - io::global_local_filesystem()->list(cache_dir, &cache_file_names), + io::global_local_filesystem()->list(cache_dir, true, &cache_files, &exists), fmt::format("List dir failed: {}", cache_dir.native())); - if (cache_file_names.empty()) { + if (cache_files.empty()) { RETURN_NOT_OK_STATUS_WITH_WARN(io::global_local_filesystem()->delete_directory(cache_dir), fmt::format("Delete dir failed: {}", cache_dir.native())); LOG(INFO) << "Delete empty dir: " << cache_dir.native(); diff --git a/be/src/io/cache/file_cache.h b/be/src/io/cache/file_cache.h index 692cd00e16..68ffc1a3a1 100644 --- a/be/src/io/cache/file_cache.h +++ b/be/src/io/cache/file_cache.h @@ -61,6 +61,11 @@ public: virtual int64_t get_oldest_match_time() const = 0; protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override { + return Status::NotSupported("dummy file cache only used for GC"); + } + Status _remove_file(const Path& file, size_t* cleaned_size); Status _remove_cache_and_done(const Path& cache_file, const Path& cache_done_file, diff --git a/be/src/io/cache/file_cache_manager.cpp b/be/src/io/cache/file_cache_manager.cpp index 33b6a2e145..d3bf4942de 100644 --- a/be/src/io/cache/file_cache_manager.cpp +++ b/be/src/io/cache/file_cache_manager.cpp @@ -120,11 +120,14 @@ void FileCacheManager::_add_file_cache_for_gc_by_disk(std::vector& result) { std::vector tablets = StorageEngine::instance()->tablet_manager()->get_all_tablet(); + bool exists = true; for (const auto& tablet : tablets) { - std::vector seg_file_paths; - if (io::global_local_filesystem()->list(tablet->tablet_path(), &seg_file_paths).ok()) { - for (Path seg_file : seg_file_paths) { - std::string seg_filename = seg_file.native(); + std::vector seg_files; + if (io::global_local_filesystem() + ->list(tablet->tablet_path(), true, &seg_files, &exists) + .ok()) { + for (auto& seg_file : seg_files) { + std::string seg_filename = seg_file.file_name; // check if it is a dir name if (!BetaRowset::is_segment_cache_dir(seg_filename)) { continue; diff --git a/be/src/io/cache/sub_file_cache.cpp b/be/src/io/cache/sub_file_cache.cpp index 4b21e37505..f23e122132 100644 --- a/be/src/io/cache/sub_file_cache.cpp +++ b/be/src/io/cache/sub_file_cache.cpp @@ -49,22 +49,11 @@ SubFileCache::SubFileCache(const Path& cache_dir, int64_t alive_time_sec, SubFileCache::~SubFileCache() {} -Status SubFileCache::read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) { - if (bthread_self() == 0) { - return read_at_impl(offset, result, io_ctx, bytes_read); - } - Status s; - auto task = [&] { s = read_at_impl(offset, result, io_ctx, bytes_read); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - -Status SubFileCache::read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) { +Status SubFileCache::read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) { RETURN_IF_ERROR(_init()); - if (io_ctx.reader_type != READER_QUERY) { - return _remote_file_reader->read_at(offset, result, io_ctx, bytes_read); + if (io_ctx->reader_type != READER_QUERY) { + return _remote_file_reader->read_at(offset, result, bytes_read, io_ctx); } std::vector need_cache_offsets; RETURN_IF_ERROR(_get_need_cache_offsets(offset, result.size, &need_cache_offsets)); @@ -125,7 +114,7 @@ Status SubFileCache::read_at_impl(size_t offset, Slice result, const IOContext& Slice read_slice(result.mutable_data() + offset_begin - offset, req_size); size_t sub_bytes_read = -1; RETURN_NOT_OK_STATUS_WITH_WARN( - _cache_file_readers[*iter]->read_at(offset_begin - *iter, read_slice, io_ctx, + _cache_file_readers[*iter]->read_at(offset_begin - *iter, read_slice, &sub_bytes_read), fmt::format("Read local cache file failed: {}", _cache_file_readers[*iter]->path().native())); @@ -204,7 +193,7 @@ Status SubFileCache::_generate_cache_reader(size_t offset, size_t req_size) { } } io::FileReaderSPtr cache_reader; - RETURN_IF_ERROR(io::global_local_filesystem()->open_file(cache_file, &cache_reader, nullptr)); + RETURN_IF_ERROR(io::global_local_filesystem()->open_file(cache_file, &cache_reader)); _cache_file_readers.emplace(offset, cache_reader); _last_match_times.emplace(offset, time(nullptr)); LOG(INFO) << "Create cache file from remote file successfully: " diff --git a/be/src/io/cache/sub_file_cache.h b/be/src/io/cache/sub_file_cache.h index f8a1a48ba5..4766a4b6a6 100644 --- a/be/src/io/cache/sub_file_cache.h +++ b/be/src/io/cache/sub_file_cache.h @@ -36,11 +36,6 @@ public: Status close() override { return _remote_file_reader->close(); } - Status read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) override; - - Status read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, size_t* bytes_read); - const Path& path() const override { return _remote_file_reader->path(); } size_t size() const override { return _remote_file_reader->size(); } @@ -65,6 +60,10 @@ public: FileSystemSPtr fs() const override { return _remote_file_reader->fs(); } +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override; + private: Status _generate_cache_reader(size_t offset, size_t req_size); diff --git a/be/src/io/cache/whole_file_cache.cpp b/be/src/io/cache/whole_file_cache.cpp index aa98df1501..5f4935cd16 100644 --- a/be/src/io/cache/whole_file_cache.cpp +++ b/be/src/io/cache/whole_file_cache.cpp @@ -36,28 +36,18 @@ WholeFileCache::WholeFileCache(const Path& cache_dir, int64_t alive_time_sec, WholeFileCache::~WholeFileCache() {} -Status WholeFileCache::read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) { - if (bthread_self() == 0) { - return read_at_impl(offset, result, io_ctx, bytes_read); - } - Status s; - auto task = [&] { s = read_at_impl(offset, result, io_ctx, bytes_read); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - -Status WholeFileCache::read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) { - if (io_ctx.reader_type != READER_QUERY) { - return _remote_file_reader->read_at(offset, result, io_ctx, bytes_read); +Status WholeFileCache::read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) { + DCHECK(io_ctx); + if (io_ctx->reader_type != READER_QUERY) { + return _remote_file_reader->read_at(offset, result, bytes_read, io_ctx); } if (_cache_file_reader == nullptr) { RETURN_IF_ERROR(_generate_cache_reader(offset, result.size)); } std::shared_lock rlock(_cache_lock); RETURN_NOT_OK_STATUS_WITH_WARN( - _cache_file_reader->read_at(offset, result, io_ctx, bytes_read), + _cache_file_reader->read_at(offset, result, bytes_read, io_ctx), fmt::format("Read local cache file failed: {}", _cache_file_reader->path().native())); if (*bytes_read != result.size) { LOG(ERROR) << "read cache file failed: " << _cache_file_reader->path().native() @@ -137,8 +127,7 @@ Status WholeFileCache::_generate_cache_reader(size_t offset, size_t req_size) { return st; } } - RETURN_IF_ERROR( - io::global_local_filesystem()->open_file(cache_file, &_cache_file_reader, nullptr)); + RETURN_IF_ERROR(io::global_local_filesystem()->open_file(cache_file, &_cache_file_reader)); _cache_file_size = _cache_file_reader->size(); LOG(INFO) << "Create cache file from remote file successfully: " << _remote_file_reader->path().native() << " -> " << cache_file.native(); diff --git a/be/src/io/cache/whole_file_cache.h b/be/src/io/cache/whole_file_cache.h index 556827f67c..0036678114 100644 --- a/be/src/io/cache/whole_file_cache.h +++ b/be/src/io/cache/whole_file_cache.h @@ -36,11 +36,6 @@ public: Status close() override { return _remote_file_reader->close(); } - Status read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) override; - - Status read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, size_t* bytes_read); - const Path& path() const override { return _remote_file_reader->path(); } size_t size() const override { return _remote_file_reader->size(); } @@ -63,6 +58,10 @@ public: FileSystemSPtr fs() const override { return _remote_file_reader->fs(); } +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override; + private: Status _generate_cache_reader(size_t offset, size_t req_size); diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index d17faec53b..09ee20d8c6 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -19,21 +19,21 @@ #include "common/config.h" #include "common/status.h" -#include "io/broker_writer.h" #include "io/fs/broker_file_system.h" +#include "io/fs/broker_file_writer.h" #include "io/fs/file_reader_options.h" #include "io/fs/file_system.h" #include "io/fs/hdfs_file_system.h" +#include "io/fs/hdfs_file_writer.h" #include "io/fs/local_file_system.h" +#include "io/fs/local_file_writer.h" #include "io/fs/remote_file_system.h" #include "io/fs/s3_file_system.h" -#include "io/hdfs_writer.h" -#include "io/local_file_writer.h" -#include "io/s3_writer.h" -#include "olap/iterators.h" +#include "io/fs/s3_file_writer.h" #include "runtime/exec_env.h" #include "runtime/stream_load/new_load_stream_mgr.h" #include "runtime/stream_load/stream_load_context.h" +#include "util/s3_uri.h" namespace doris { @@ -41,23 +41,34 @@ Status FileFactory::create_file_writer(TFileType::type type, ExecEnv* env, const std::vector& broker_addresses, const std::map& properties, const std::string& path, int64_t start_offset, - std::unique_ptr& file_writer) { + std::unique_ptr& file_writer) { switch (type) { case TFileType::FILE_LOCAL: { - file_writer.reset(new LocalFileWriter(path, start_offset)); + RETURN_IF_ERROR(io::global_local_filesystem()->create_file(path, &file_writer)); break; } case TFileType::FILE_BROKER: { - file_writer.reset(new BrokerWriter(env, broker_addresses, properties, path, start_offset)); + std::shared_ptr fs; + RETURN_IF_ERROR(io::BrokerFileSystem::create(broker_addresses[0], properties, 0, &fs)); + RETURN_IF_ERROR(fs->create_file(path, &file_writer)); break; } case TFileType::FILE_S3: { - file_writer.reset(new S3Writer(properties, path, start_offset)); + S3URI s3_uri(path); + RETURN_IF_ERROR(s3_uri.parse()); + S3Conf s3_conf; + RETURN_IF_ERROR( + S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf)); + std::shared_ptr fs; + RETURN_IF_ERROR(io::S3FileSystem::create(s3_conf, "", &fs)); + RETURN_IF_ERROR(fs->create_file(path, &file_writer)); break; } case TFileType::FILE_HDFS: { - RETURN_IF_ERROR(create_hdfs_writer( - const_cast&>(properties), path, file_writer)); + THdfsParams hdfs_params = parse_properties(properties); + std::shared_ptr fs; + RETURN_IF_ERROR(io::HdfsFileSystem::create(hdfs_params, "", &fs)); + RETURN_IF_ERROR(fs->create_file(path, &file_writer)); break; } default: @@ -71,34 +82,34 @@ Status FileFactory::create_file_reader(RuntimeProfile* /*profile*/, const FileSystemProperties& system_properties, const FileDescription& file_description, std::shared_ptr* file_system, - io::FileReaderSPtr* file_reader, IOContext* io_ctx) { + io::FileReaderSPtr* file_reader) { TFileType::type type = system_properties.system_type; auto cache_policy = io::FileCachePolicy::NO_CACHE; - if (config::enable_file_cache && io_ctx->enable_file_cache) { + if (config::enable_file_cache) { cache_policy = io::FileCachePolicy::FILE_BLOCK_CACHE; } io::FileBlockCachePathPolicy file_block_cache; io::FileReaderOptions reader_options(cache_policy, file_block_cache); switch (type) { case TFileType::FILE_LOCAL: { - RETURN_IF_ERROR(io::global_local_filesystem()->open_file( - file_description.path, reader_options, file_reader, io_ctx)); + RETURN_IF_ERROR(io::global_local_filesystem()->open_file(file_description.path, + reader_options, file_reader)); break; } case TFileType::FILE_S3: { RETURN_IF_ERROR(create_s3_reader(system_properties.properties, file_description.path, - file_system, file_reader, reader_options, io_ctx)); + file_system, file_reader, reader_options)); break; } case TFileType::FILE_HDFS: { RETURN_IF_ERROR(create_hdfs_reader(system_properties.hdfs_params, file_description.path, - file_system, file_reader, reader_options, io_ctx)); + file_system, file_reader, reader_options)); break; } case TFileType::FILE_BROKER: { RETURN_IF_ERROR(create_broker_reader(system_properties.broker_addresses[0], system_properties.properties, file_description, - file_system, file_reader, reader_options, io_ctx)); + file_system, file_reader, reader_options)); break; } default: @@ -120,18 +131,11 @@ Status FileFactory::create_pipe_reader(const TUniqueId& load_id, io::FileReaderS Status FileFactory::create_hdfs_reader(const THdfsParams& hdfs_params, const std::string& path, std::shared_ptr* hdfs_file_system, io::FileReaderSPtr* reader, - const io::FileReaderOptions& reader_options, - IOContext* io_ctx) { - *hdfs_file_system = io::HdfsFileSystem::create(hdfs_params, ""); - RETURN_IF_ERROR((std::static_pointer_cast(*hdfs_file_system))->connect()); - RETURN_IF_ERROR((*hdfs_file_system)->open_file(path, reader_options, reader, io_ctx)); - return Status::OK(); -} - -Status FileFactory::create_hdfs_writer(const std::map& properties, - const std::string& path, - std::unique_ptr& writer) { - writer.reset(new HDFSWriter(properties, path)); + const io::FileReaderOptions& reader_options) { + std::shared_ptr fs; + RETURN_IF_ERROR(io::HdfsFileSystem::create(hdfs_params, "", &fs)); + RETURN_IF_ERROR(fs->open_file(path, reader_options, reader)); + *hdfs_file_system = std::move(fs); return Status::OK(); } @@ -139,18 +143,15 @@ Status FileFactory::create_s3_reader(const std::map& p const std::string& path, std::shared_ptr* s3_file_system, io::FileReaderSPtr* reader, - const io::FileReaderOptions& reader_options, - IOContext* io_ctx) { + const io::FileReaderOptions& reader_options) { S3URI s3_uri(path); - if (!s3_uri.parse()) { - return Status::InvalidArgument("s3 uri is invalid: {}", path); - } + RETURN_IF_ERROR(s3_uri.parse()); S3Conf s3_conf; - RETURN_IF_ERROR(ClientFactory::convert_properties_to_s3_conf(prop, s3_uri, &s3_conf)); - std::shared_ptr tmp_fs = io::S3FileSystem::create(std::move(s3_conf), ""); - RETURN_IF_ERROR(tmp_fs->connect()); - RETURN_IF_ERROR(tmp_fs->open_file(s3_uri.get_key(), reader_options, reader, io_ctx)); - *s3_file_system = std::move(tmp_fs); + RETURN_IF_ERROR(S3ClientFactory::convert_properties_to_s3_conf(prop, s3_uri, &s3_conf)); + std::shared_ptr fs; + RETURN_IF_ERROR(io::S3FileSystem::create(std::move(s3_conf), "", &fs)); + RETURN_IF_ERROR(fs->open_file(path, reader_options, reader)); + *s3_file_system = std::move(fs); return Status::OK(); } @@ -159,14 +160,12 @@ Status FileFactory::create_broker_reader(const TNetworkAddress& broker_addr, const FileDescription& file_description, std::shared_ptr* broker_file_system, io::FileReaderSPtr* reader, - const io::FileReaderOptions& reader_options, - IOContext* io_ctx) { - *broker_file_system = - io::BrokerFileSystem::create(broker_addr, prop, file_description.file_size); + const io::FileReaderOptions& reader_options) { + std::shared_ptr fs; RETURN_IF_ERROR( - (std::static_pointer_cast(*broker_file_system))->connect()); - RETURN_IF_ERROR((*broker_file_system) - ->open_file(file_description.path, reader_options, reader, io_ctx)); + io::BrokerFileSystem::create(broker_addr, prop, file_description.file_size, &fs)); + RETURN_IF_ERROR(fs->open_file(file_description.path, reader_options, reader)); + *broker_file_system = std::move(fs); return Status::OK(); } } // namespace doris diff --git a/be/src/io/file_factory.h b/be/src/io/file_factory.h index 413795b7cd..2a7a2fe9b4 100644 --- a/be/src/io/file_factory.h +++ b/be/src/io/file_factory.h @@ -18,8 +18,8 @@ #include "gen_cpp/PlanNodes_types.h" #include "gen_cpp/Types_types.h" -#include "io/file_writer.h" #include "io/fs/file_reader.h" +#include "io/fs/file_writer.h" namespace doris { namespace io { @@ -45,18 +45,19 @@ struct FileDescription { class FileFactory { public: - // Create FileWriter + /// Create FileWriter static Status create_file_writer(TFileType::type type, ExecEnv* env, const std::vector& broker_addresses, const std::map& properties, const std::string& path, int64_t start_offset, - std::unique_ptr& file_writer); + std::unique_ptr& file_writer); + /// Create FileReader static Status create_file_reader(RuntimeProfile* profile, const FileSystemProperties& system_properties, const FileDescription& file_description, std::shared_ptr* file_system, - io::FileReaderSPtr* file_reader, IOContext* io_ctx); + io::FileReaderSPtr* file_reader); // Create FileReader for stream load pipe static Status create_pipe_reader(const TUniqueId& load_id, io::FileReaderSPtr* file_reader); @@ -64,25 +65,20 @@ public: static Status create_hdfs_reader(const THdfsParams& hdfs_params, const std::string& path, std::shared_ptr* hdfs_file_system, io::FileReaderSPtr* reader, - const io::FileReaderOptions& reader_options, - IOContext* io_ctx); - - static Status create_hdfs_writer(const std::map& properties, - const std::string& path, std::unique_ptr& writer); + const io::FileReaderOptions& reader_options); static Status create_s3_reader(const std::map& prop, const std::string& path, std::shared_ptr* s3_file_system, io::FileReaderSPtr* reader, - const io::FileReaderOptions& reader_options, IOContext* io_ctx); + const io::FileReaderOptions& reader_options); static Status create_broker_reader(const TNetworkAddress& broker_addr, const std::map& prop, const FileDescription& file_description, std::shared_ptr* hdfs_file_system, io::FileReaderSPtr* reader, - const io::FileReaderOptions& reader_options, - IOContext* io_ctx); + const io::FileReaderOptions& reader_options); static TFileType::type convert_storage_type(TStorageBackendType::type type) { switch (type) { diff --git a/be/src/io/fs/broker_file_reader.cpp b/be/src/io/fs/broker_file_reader.cpp index b708627660..40ddfe6b44 100644 --- a/be/src/io/fs/broker_file_reader.cpp +++ b/be/src/io/fs/broker_file_reader.cpp @@ -77,8 +77,8 @@ Status BrokerFileReader::close() { return Status::OK(); } -Status BrokerFileReader::read_at(size_t offset, Slice result, const IOContext& /*io_ctx*/, - size_t* bytes_read) { +Status BrokerFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* /*io_ctx*/) { DCHECK(!closed()); size_t bytes_req = result.size; char* to = result.data; diff --git a/be/src/io/fs/broker_file_reader.h b/be/src/io/fs/broker_file_reader.h index 5a753462bd..288c5d25d4 100644 --- a/be/src/io/fs/broker_file_reader.h +++ b/be/src/io/fs/broker_file_reader.h @@ -39,9 +39,6 @@ public: Status close() override; - Status read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) override; - const Path& path() const override { return _path; } size_t size() const override { return _file_size; } @@ -50,6 +47,10 @@ public: FileSystemSPtr fs() const override { return _fs; } +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override; + private: const Path& _path; size_t _file_size; diff --git a/be/src/io/fs/broker_file_system.cpp b/be/src/io/fs/broker_file_system.cpp index 681fa014d4..f962cdb810 100644 --- a/be/src/io/fs/broker_file_system.cpp +++ b/be/src/io/fs/broker_file_system.cpp @@ -21,10 +21,11 @@ #include #include "io/fs/broker_file_reader.h" +#include "io/fs/broker_file_writer.h" +#include "io/fs/local_file_system.h" #include "runtime/broker_mgr.h" #include "runtime/exec_env.h" #include "util/defer_op.h" -#include "util/storage_backend.h" namespace doris { namespace io { @@ -50,17 +51,17 @@ inline const std::string& client_id(const TNetworkAddress& addr) { #endif #ifndef CHECK_BROKER_CLIENT -#define CHECK_BROKER_CLIENT(client) \ - if (!client) { \ - return Status::InternalError("init Broker client error"); \ +#define CHECK_BROKER_CLIENT(client) \ + if (!client) { \ + return Status::IOError("init Broker client error"); \ } #endif -std::shared_ptr BrokerFileSystem::create( - const TNetworkAddress& broker_addr, const std::map& broker_prop, - size_t file_size) { - return std::shared_ptr( - new BrokerFileSystem(broker_addr, broker_prop, file_size)); +Status BrokerFileSystem::create(const TNetworkAddress& broker_addr, + const std::map& broker_prop, + size_t file_size, std::shared_ptr* fs) { + (*fs).reset(new BrokerFileSystem(broker_addr, broker_prop, file_size)); + return (*fs)->connect(); } BrokerFileSystem::BrokerFileSystem(const TNetworkAddress& broker_addr, @@ -71,25 +72,24 @@ BrokerFileSystem::BrokerFileSystem(const TNetworkAddress& broker_addr, _broker_prop(broker_prop), _file_size(file_size) {} -Status BrokerFileSystem::connect() { +Status BrokerFileSystem::connect_impl() { Status status = Status::OK(); _client.reset(new BrokerServiceConnection(client_cache(), _broker_addr, config::thrift_rpc_timeout_ms, &status)); - if (!status.ok()) { - std::stringstream ss; - ss << "failed to get broker client. " - << "broker addr: " << _broker_addr << ". msg: " << status; - status = Status::InternalError(ss.str()); - } return status; } -Status BrokerFileSystem::open_file(const Path& path, FileReaderSPtr* reader, - IOContext* /*io_ctx*/) { +Status BrokerFileSystem::create_file_impl(const Path& path, FileWriterPtr* writer) { + *writer = std::make_unique(ExecEnv::GetInstance(), _broker_addr, _broker_prop, + path, 0 /* offset */, getSPtr()); + return Status::OK(); +} + +Status BrokerFileSystem::open_file_internal(const Path& file, FileReaderSPtr* reader) { CHECK_BROKER_CLIENT(_client); TBrokerOpenReaderRequest request; request.__set_version(TBrokerVersion::VERSION_ONE); - request.__set_path(path); + request.__set_path(file); request.__set_startOffset(0); request.__set_clientId(client_id(_broker_addr)); request.__set_properties(_broker_prop); @@ -105,39 +105,31 @@ Status BrokerFileSystem::open_file(const Path& path, FileReaderSPtr* reader, (*_client)->openReader(*response, request); } } catch (apache::thrift::TException& e) { - std::stringstream ss; - ss << "Open broker reader failed, broker:" << _broker_addr << " failed: " << e.what(); - return Status::RpcError(ss.str()); + return Status::RpcError("failed to open file {}: {}", file.native(), error_msg(e.what())); } if (response->opStatus.statusCode != TBrokerOperationStatusCode::OK) { - std::stringstream ss; - ss << "Open broker reader failed, broker: " << _broker_addr - << " failed: " << response->opStatus.message; - return Status::InternalError(ss.str()); + return Status::IOError("failed to open file {}: {}", file.native(), + error_msg(response->opStatus.message)); } - // TODO(cmy): The file size is no longer got from openReader() method. - // But leave the code here for compatibility. - // This will be removed later. - TBrokerFD fd; - if (response->__isset.size) { - _file_size = response->size; - } - fd = response->fd; *reader = std::make_shared( - _broker_addr, path, _file_size, fd, + _broker_addr, file, _file_size, response->fd, std::static_pointer_cast(shared_from_this())); return Status::OK(); } -Status BrokerFileSystem::delete_file(const Path& path) { +Status BrokerFileSystem::create_directory_impl(const Path& /*path*/) { + return Status::NotSupported("create directory not implemented!"); +} + +Status BrokerFileSystem::delete_file_impl(const Path& file) { CHECK_BROKER_CLIENT(_client); try { // rm file from remote path TBrokerDeletePathRequest del_req; TBrokerOperationStatus del_rep; del_req.__set_version(TBrokerVersion::VERSION_ONE); - del_req.__set_path(path); + del_req.__set_path(file); del_req.__set_properties(_broker_prop); try { @@ -150,27 +142,27 @@ Status BrokerFileSystem::delete_file(const Path& path) { if (del_rep.statusCode == TBrokerOperationStatusCode::OK) { return Status::OK(); } else { - std::stringstream ss; - ss << "failed to delete from remote path: " << path << ", msg: " << del_rep.message; - return Status::InternalError(ss.str()); + return Status::IOError("failed to delete file {}: {}", file.native(), + error_msg(del_rep.message)); } } catch (apache::thrift::TException& e) { - std::stringstream ss; - ss << "failed to delete file in remote path: " << path << ", msg: " << e.what(); - return Status::RpcError(ss.str()); + return Status::RpcError("failed to delete file {}: {}", file.native(), error_msg(e.what())); } } -Status BrokerFileSystem::create_directory(const Path& /*path*/) { - return Status::NotSupported("create directory not implemented!"); -} - // Delete all files under path. -Status BrokerFileSystem::delete_directory(const Path& path) { - return delete_file(path); +Status BrokerFileSystem::delete_directory_impl(const Path& dir) { + return delete_file_impl(dir); } -Status BrokerFileSystem::exists(const Path& path, bool* res) const { +Status BrokerFileSystem::batch_delete_impl(const std::vector& files) { + for (auto& file : files) { + RETURN_IF_ERROR(delete_file_impl(file)); + } + return Status::OK(); +} + +Status BrokerFileSystem::exists_impl(const Path& path, bool* res) const { CHECK_BROKER_CLIENT(_client); *res = false; try { @@ -188,29 +180,32 @@ Status BrokerFileSystem::exists(const Path& path, bool* res) const { } if (check_rep.opStatus.statusCode != TBrokerOperationStatusCode::OK) { - std::stringstream ss; - ss << "failed to check exist: " << path << ", msg: " << check_rep.opStatus.message; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); + return Status::IOError("failed to check exist of path {}: {}", path.native(), + error_msg(check_rep.opStatus.message)); } else if (!check_rep.isPathExist) { - return Status::NotFound("{} not exists!", path.string()); + *res = false; + return Status::OK(); } else { *res = true; return Status::OK(); } } catch (apache::thrift::TException& e) { - std::stringstream ss; - ss << "failed to check exist: " << path << ", msg: " << e.what(); - return Status::RpcError(ss.str()); + return Status::RpcError("failed to check exist of path {}: {}", path.native(), + error_msg(e.what())); } } -Status BrokerFileSystem::file_size(const Path& path, size_t* file_size) const { +Status BrokerFileSystem::file_size_impl(const Path& path, size_t* file_size) const { *file_size = _file_size; return Status::OK(); } -Status BrokerFileSystem::list(const Path& path, std::vector* files) { +Status BrokerFileSystem::list_impl(const Path& dir, bool only_file, std::vector* files, + bool* exists) { + RETURN_IF_ERROR(exists_impl(dir, exists)); + if (!(*exists)) { + return Status::OK(); + } CHECK_BROKER_CLIENT(_client); Status status = Status::OK(); try { @@ -218,7 +213,7 @@ Status BrokerFileSystem::list(const Path& path, std::vector* files) { TBrokerListResponse list_rep; TBrokerListPathRequest list_req; list_req.__set_version(TBrokerVersion::VERSION_ONE); - list_req.__set_path(path / "*"); + list_req.__set_path(dir / "*"); list_req.__set_isRecursive(false); list_req.__set_properties(_broker_prop); list_req.__set_fileNameOnly(true); // we only need file name, not abs path @@ -231,52 +226,204 @@ Status BrokerFileSystem::list(const Path& path, std::vector* files) { } if (list_rep.opStatus.statusCode == TBrokerOperationStatusCode::FILE_NOT_FOUND) { - LOG(INFO) << "path does not exist: " << path; + LOG(INFO) << "path does not exist: " << dir; + *exists = false; return Status::OK(); } else if (list_rep.opStatus.statusCode != TBrokerOperationStatusCode::OK) { - std::stringstream ss; - ss << "failed to list files from remote path: " << path - << ", msg: " << list_rep.opStatus.message; - return Status::InternalError(ss.str()); + return Status::IOError("failed to list dir {}: {}", dir.native(), + error_msg(list_rep.opStatus.message)); } LOG(INFO) << "finished to list files from remote path. file num: " << list_rep.files.size(); + *exists = true; // split file name and checksum for (const auto& file : list_rep.files) { - if (file.isDir) { + if (only_file && file.isDir) { // this is not a file continue; } - - const std::string& file_name = file.path; - size_t pos = file_name.find_last_of('.'); - if (pos == std::string::npos || pos == file_name.size() - 1) { - // Not found checksum separator, ignore this file - continue; - } - - FileStat stat = {std::string(file_name, 0, pos), std::string(file_name, pos + 1), - file.size}; - files->emplace_back(std::string(file_name, 0, pos)); - VLOG(2) << "split remote file: " << std::string(file_name, 0, pos) - << ", checksum: " << std::string(file_name, pos + 1); + FileInfo file_info; + file_info.file_name = file.path; + file_info.file_size = file.size; + file_info.is_file = !file.isDir; + files->emplace_back(std::move(file_info)); } LOG(INFO) << "finished to split files. valid file num: " << files->size(); - } catch (apache::thrift::TException& e) { std::stringstream ss; - ss << "failed to list files in remote path: " << path << ", msg: " << e.what(); - return Status::RpcError(ss.str()); + ss << "failed to list files in remote path: " << dir << ", msg: " << e.what(); + return Status::RpcError("failed to list dir {}: {}", dir.native(), error_msg(e.what())); } return status; } +Status BrokerFileSystem::rename_impl(const Path& orig_name, const Path& new_name) { + CHECK_BROKER_CLIENT(_client); + try { + TBrokerOperationStatus op_status; + TBrokerRenamePathRequest rename_req; + rename_req.__set_version(TBrokerVersion::VERSION_ONE); + rename_req.__set_srcPath(orig_name); + rename_req.__set_destPath(new_name); + rename_req.__set_properties(_broker_prop); + + try { + (*_client)->renamePath(op_status, rename_req); + } catch (apache::thrift::transport::TTransportException& e) { + RETURN_IF_ERROR((*_client).reopen()); + (*_client)->renamePath(op_status, rename_req); + } + + if (op_status.statusCode != TBrokerOperationStatusCode::OK) { + return Status::IOError("failed to rename from {} to {}: {}", orig_name.native(), + new_name.native(), error_msg(op_status.message)); + } + } catch (apache::thrift::TException& e) { + return Status::RpcError("failed to rename from {} to {}: {}", orig_name.native(), + new_name.native(), error_msg(e.what())); + } + + LOG(INFO) << "finished to rename file. orig: " << orig_name << ", new: " << new_name; + return Status::OK(); +} + +Status BrokerFileSystem::rename_dir_impl(const Path& orig_name, const Path& new_name) { + return rename_impl(orig_name, new_name); +} + +Status BrokerFileSystem::upload_impl(const Path& local_file, const Path& remote_file) { + // 1. open local file for read + FileSystemSPtr local_fs = global_local_filesystem(); + FileReaderSPtr local_reader = nullptr; + RETURN_IF_ERROR(local_fs->open_file(local_file, &local_reader)); + + size_t file_len = local_reader->size(); + if (file_len == -1) { + return Status::IOError("failed to get length of file: {}: {}", local_file.native(), + error_msg("")); + } + + // NOTICE: broker writer must be closed before calling rename + FileWriterPtr broker_writer = nullptr; + RETURN_IF_ERROR(create_file_impl(remote_file, &broker_writer)); + + constexpr size_t buf_sz = 1024 * 1024; + char read_buf[buf_sz]; + size_t left_len = file_len; + size_t read_offset = 0; + size_t bytes_read = 0; + while (left_len > 0) { + size_t read_len = left_len > buf_sz ? buf_sz : left_len; + RETURN_IF_ERROR(local_reader->read_at(read_offset, {read_buf, read_len}, &bytes_read)); + // write through broker + RETURN_IF_ERROR(broker_writer->append({read_buf, read_len})); + + read_offset += read_len; + left_len -= read_len; + } + + // close manually, because we need to check its close status + RETURN_IF_ERROR(broker_writer->close()); + LOG(INFO) << "finished to write file via broker. file: " << local_file + << ", length: " << file_len; + return Status::OK(); +} + +Status BrokerFileSystem::batch_upload_impl(const std::vector& local_files, + const std::vector& remote_files) { + DCHECK(local_files.size() == remote_files.size()); + for (int i = 0; i < local_files.size(); ++i) { + RETURN_IF_ERROR(upload_impl(local_files[i], remote_files[i])); + } + return Status::OK(); +} + +Status BrokerFileSystem::direct_upload_impl(const Path& remote_file, const std::string& content) { + FileWriterPtr broker_writer = nullptr; + RETURN_IF_ERROR(create_file_impl(remote_file, &broker_writer)); + RETURN_IF_ERROR(broker_writer->append({content})); + return broker_writer->close(); +} + +Status BrokerFileSystem::upload_with_checksum_impl(const Path& local_file, const Path& remote_file, + const std::string& checksum) { + std::string temp = remote_file.string() + ".part"; + std::string final_file = remote_file.string() + "." + checksum; + RETURN_IF_ERROR(upload_impl(local_file, temp)); + return rename_impl(temp, final_file); +} + +Status BrokerFileSystem::download_impl(const Path& remote_file, const Path& local_file) { + // 1. open remote file for read + FileReaderSPtr broker_reader = nullptr; + RETURN_IF_ERROR(open_file_internal(remote_file, &broker_reader)); + + // 2. remove the existing local file if exist + if (std::filesystem::remove(local_file)) { + VLOG(2) << "remove the previously exist local file: " << local_file; + } + + // 3. open local file for write + FileSystemSPtr local_fs = global_local_filesystem(); + FileWriterPtr local_writer = nullptr; + RETURN_IF_ERROR(local_fs->create_file(local_file, &local_writer)); + + // 4. read remote and write to local + VLOG(2) << "read remote file: " << remote_file << " to local: " << local_file; + constexpr size_t buf_sz = 1024 * 1024; + std::unique_ptr read_buf(new uint8_t[buf_sz]); + size_t write_offset = 0; + size_t cur_offset = 0; + while (true) { + size_t read_len = 0; + Slice file_slice(read_buf.get(), buf_sz); + RETURN_IF_ERROR(broker_reader->read_at(cur_offset, file_slice, &read_len)); + cur_offset += read_len; + if (read_len == 0) { + break; + } + + RETURN_IF_ERROR(local_writer->write_at(write_offset, {read_buf.get(), read_len})); + write_offset += read_len; + } // file_handler should be closed before calculating checksum + + return Status::OK(); +} + +Status BrokerFileSystem::direct_download_impl(const Path& remote_impl, std::string* content) { + // 1. open remote file for read + FileReaderSPtr broker_reader = nullptr; + RETURN_IF_ERROR(open_file_internal(remote_impl, &broker_reader)); + + constexpr size_t buf_sz = 1024 * 1024; + std::unique_ptr read_buf(new char[buf_sz]); + size_t write_offset = 0; + size_t cur_offset = 0; + while (true) { + size_t read_len = 0; + Slice file_slice(read_buf.get(), buf_sz); + RETURN_IF_ERROR(broker_reader->read_at(cur_offset, file_slice, &read_len)); + cur_offset += read_len; + if (read_len == 0) { + break; + } + + content->insert(write_offset, read_buf.get(), read_len); + write_offset += read_len; + } + return Status::OK(); +} + Status BrokerFileSystem::get_client(std::shared_ptr* client) const { CHECK_BROKER_CLIENT(_client); *client = _client; return Status::OK(); } +std::string BrokerFileSystem::error_msg(const std::string& err) const { + return fmt::format("({}:{}), {}", _broker_addr.hostname, _broker_addr.port, err); +} + } // namespace io } // namespace doris diff --git a/be/src/io/fs/broker_file_system.h b/be/src/io/fs/broker_file_system.h index bf55d49a53..0328add3ca 100644 --- a/be/src/io/fs/broker_file_system.h +++ b/be/src/io/fs/broker_file_system.h @@ -24,51 +24,42 @@ namespace doris { namespace io { class BrokerFileSystem final : public RemoteFileSystem { public: - static std::shared_ptr create( - const TNetworkAddress& broker_addr, - const std::map& broker_prop, size_t file_size); + static Status create(const TNetworkAddress& broker_addr, + const std::map& broker_prop, size_t file_size, + std::shared_ptr* fs); ~BrokerFileSystem() override = default; - Status create_file(const Path& /*path*/, FileWriterPtr* /*writer*/) override { - return Status::NotSupported("Currently not support to create file through broker."); - } - - Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override; - - Status delete_file(const Path& path) override; - - Status create_directory(const Path& path) override; - - // Delete all files under path. - Status delete_directory(const Path& path) override; - - Status link_file(const Path& /*src*/, const Path& /*dest*/) override { - return Status::NotSupported("Not supported link file through broker."); - } - - Status exists(const Path& path, bool* res) const override; - - Status file_size(const Path& path, size_t* file_size) const override; - - Status list(const Path& path, std::vector* files) override; - - Status upload(const Path& /*local_path*/, const Path& /*dest_path*/) override { - return Status::NotSupported("Currently not support to upload file to HDFS"); - } - - Status batch_upload(const std::vector& /*local_paths*/, - const std::vector& /*dest_paths*/) override { - return Status::NotSupported("Currently not support to batch upload file to HDFS"); - } - - Status connect() override; - Status get_client(std::shared_ptr* client) const; +protected: + Status connect_impl() override; + Status create_file_impl(const Path& file, FileWriterPtr* writer) override; + Status open_file_internal(const Path& file, FileReaderSPtr* reader) override; + Status create_directory_impl(const Path& dir) override; + Status delete_file_impl(const Path& file) override; + Status delete_directory_impl(const Path& dir) override; + Status batch_delete_impl(const std::vector& files) override; + Status exists_impl(const Path& path, bool* res) const override; + Status file_size_impl(const Path& file, size_t* file_size) const override; + Status list_impl(const Path& dir, bool only_file, std::vector* files, + bool* exists) override; + Status rename_impl(const Path& orig_name, const Path& new_name) override; + Status rename_dir_impl(const Path& orig_name, const Path& new_name) override; + + Status upload_impl(const Path& local_file, const Path& remote_file) override; + Status batch_upload_impl(const std::vector& local_files, + const std::vector& remote_files) override; + Status direct_upload_impl(const Path& remote_file, const std::string& content) override; + Status upload_with_checksum_impl(const Path& local_file, const Path& remote_file, + const std::string& checksum) override; + Status download_impl(const Path& remote_file, const Path& local_file) override; + Status direct_download_impl(const Path& remote_file, std::string* content) override; + private: BrokerFileSystem(const TNetworkAddress& broker_addr, const std::map& broker_prop, size_t file_size); + std::string error_msg(const std::string& err) const; const TNetworkAddress& _broker_addr; const std::map& _broker_prop; diff --git a/be/src/io/broker_writer.cpp b/be/src/io/fs/broker_file_writer.cpp similarity index 68% rename from be/src/io/broker_writer.cpp rename to be/src/io/fs/broker_file_writer.cpp index 398b172c07..463c44946f 100644 --- a/be/src/io/broker_writer.cpp +++ b/be/src/io/fs/broker_file_writer.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "io/broker_writer.h" +#include "io/fs/broker_file_writer.h" #include @@ -29,20 +29,22 @@ #include "runtime/exec_env.h" namespace doris { +namespace io { -BrokerWriter::BrokerWriter(ExecEnv* env, const std::vector& broker_addresses, - const std::map& properties, - const std::string& path, int64_t start_offset) - : _env(env), - _addresses(broker_addresses), +BrokerFileWriter::BrokerFileWriter(ExecEnv* env, const TNetworkAddress& broker_address, + const std::map& properties, + const std::string& path, int64_t start_offset, FileSystemSPtr fs) + : FileWriter(path, fs), + _env(env), + _address(broker_address), _properties(properties), - _path(path), - _cur_offset(start_offset), - _is_closed(false), - _addr_idx(0) {} + _cur_offset(start_offset) {} -BrokerWriter::~BrokerWriter() { - close(); +BrokerFileWriter::~BrokerFileWriter() { + if (_opened) { + close(); + } + CHECK(!_opened || _closed) << "open: " << _opened << ", closed: " << _closed; } #ifdef BE_TEST @@ -65,14 +67,100 @@ inline const std::string& client_id(ExecEnv* env, const TNetworkAddress& addr) { } #endif -Status BrokerWriter::open() { +Status BrokerFileWriter::close() { + if (_closed) { + return Status::OK(); + } + _closed = true; + + TBrokerCloseWriterRequest request; + request.__set_version(TBrokerVersion::VERSION_ONE); + request.__set_fd(_fd); + VLOG_ROW << "debug: send broker close writer request: " + << apache::thrift::ThriftDebugString(request).c_str(); + + TBrokerOperationStatus response; + try { + Status status; + // use 20 second because close may take longer in remote storage, sometimes. + // TODO(cmy): optimize this if necessary. + BrokerServiceConnection client(client_cache(_env), _address, 20000, &status); + if (!status.ok()) { + LOG(WARNING) << "Create broker write client failed. broker=" << _address + << ", status=" << status; + return status; + } + + try { + client->closeWriter(response, request); + } catch (apache::thrift::transport::TTransportException& e) { + LOG(WARNING) << "Close broker writer failed. broker:" << _address + << " msg:" << e.what(); + status = client.reopen(); + if (!status.ok()) { + LOG(WARNING) << "Reopen broker writer failed. broker=" << _address + << ", status=" << status; + return status; + } + client->closeWriter(response, request); + } + } catch (apache::thrift::TException& e) { + std::stringstream ss; + ss << "Close broker writer failed, broker:" << _address << " msg:" << e.what(); + LOG(WARNING) << ss.str(); + return Status::InternalError(ss.str()); + } + + VLOG_ROW << "debug: send broker close writer response: " + << apache::thrift::ThriftDebugString(response).c_str(); + + if (response.statusCode != TBrokerOperationStatusCode::OK) { + std::stringstream ss; + ss << "Close broker writer failed, broker:" << _address << " msg:" << response.message; + LOG(WARNING) << ss.str(); + return Status::InternalError(ss.str()); + } + return Status::OK(); +} + +Status BrokerFileWriter::abort() { + // TODO: should remove file + return Status::OK(); +} + +Status BrokerFileWriter::appendv(const Slice* data, size_t data_cnt) { + DCHECK(!_closed); + if (!_opened) { + RETURN_IF_ERROR(_open()); + _opened = true; + } + + for (size_t i = 0; i < data_cnt; i++) { + const Slice& result = data[i]; + size_t left_bytes = result.size; + const char* p = result.data; + while (left_bytes > 0) { + size_t written_bytes = 0; + RETURN_IF_ERROR(_write((const uint8_t*)p, left_bytes, &written_bytes)); + left_bytes -= written_bytes; + p += written_bytes; + _bytes_appended += written_bytes; + } + } + return Status::OK(); +} + +Status BrokerFileWriter::finalize() { + return Status::OK(); +} + +Status BrokerFileWriter::_open() { TBrokerOpenWriterRequest request; - const TNetworkAddress& broker_addr = _addresses[_addr_idx]; request.__set_version(TBrokerVersion::VERSION_ONE); request.__set_path(_path); request.__set_openMode(TBrokerOpenMode::APPEND); - request.__set_clientId(client_id(_env, broker_addr)); + request.__set_clientId(client_id(_env, _address)); request.__set_properties(_properties); VLOG_ROW << "debug: send broker open writer request: " @@ -81,11 +169,11 @@ Status BrokerWriter::open() { TBrokerOpenWriterResponse response; try { Status status; - BrokerServiceConnection client(client_cache(_env), broker_addr, - config::thrift_rpc_timeout_ms, &status); + BrokerServiceConnection client(client_cache(_env), _address, config::thrift_rpc_timeout_ms, + &status); if (!status.ok()) { LOG(WARNING) << "Create broker writer client failed. " - << "broker=" << broker_addr << ", status=" << status; + << "broker=" << _address << ", status=" << status; return status; } @@ -97,7 +185,7 @@ Status BrokerWriter::open() { } } catch (apache::thrift::TException& e) { std::stringstream ss; - ss << "Open broker writer failed, broker:" << broker_addr << " failed:" << e.what(); + ss << "Open broker writer failed, broker:" << _address << " failed:" << e.what(); LOG(WARNING) << ss.str(); return Status::RpcError(ss.str()); } @@ -107,7 +195,7 @@ Status BrokerWriter::open() { if (response.opStatus.statusCode != TBrokerOperationStatusCode::OK) { std::stringstream ss; - ss << "Open broker writer failed, broker:" << broker_addr + ss << "Open broker writer failed, broker:" << _address << " failed:" << response.opStatus.message; LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); @@ -117,13 +205,12 @@ Status BrokerWriter::open() { return Status::OK(); } -Status BrokerWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_len) { +Status BrokerFileWriter::_write(const uint8_t* buf, size_t buf_len, size_t* written_bytes) { if (buf_len == 0) { - *written_len = 0; + *written_bytes = 0; return Status::OK(); } - const TNetworkAddress& broker_addr = _addresses[_addr_idx]; TBrokerPWriteRequest request; request.__set_version(TBrokerVersion::VERSION_ONE); request.__set_fd(_fd); @@ -136,11 +223,11 @@ Status BrokerWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_l TBrokerOperationStatus response; try { Status status; - BrokerServiceConnection client(client_cache(_env), broker_addr, - config::thrift_rpc_timeout_ms, &status); + BrokerServiceConnection client(client_cache(_env), _address, config::thrift_rpc_timeout_ms, + &status); if (!status.ok()) { LOG(WARNING) << "Create broker write client failed. " - << "broker=" << broker_addr << ", status=" << status; + << "broker=" << _address << ", status=" << status; return status; } @@ -153,7 +240,7 @@ Status BrokerWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_l } } catch (apache::thrift::TException& e) { std::stringstream ss; - ss << "Fail to write to broker, broker:" << broker_addr << " failed:" << e.what(); + ss << "Fail to write to broker, broker:" << _address << " failed:" << e.what(); LOG(WARNING) << ss.str(); return Status::RpcError(ss.str()); } @@ -163,74 +250,16 @@ Status BrokerWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_l if (response.statusCode != TBrokerOperationStatusCode::OK) { std::stringstream ss; - ss << "Fail to write to broker, broker:" << broker_addr << " msg:" << response.message; + ss << "Fail to write to broker, broker:" << _address << " msg:" << response.message; LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } - *written_len = buf_len; + *written_bytes = buf_len; _cur_offset += buf_len; return Status::OK(); } -Status BrokerWriter::close() { - if (_is_closed) { - return Status::OK(); - } - TBrokerCloseWriterRequest request; - - request.__set_version(TBrokerVersion::VERSION_ONE); - request.__set_fd(_fd); - - VLOG_ROW << "debug: send broker close writer request: " - << apache::thrift::ThriftDebugString(request).c_str(); - - const TNetworkAddress& broker_addr = _addresses[_addr_idx]; - TBrokerOperationStatus response; - try { - Status status; - // use 20 second because close may take longer in remote storage, sometimes. - // TODO(cmy): optimize this if necessary. - BrokerServiceConnection client(client_cache(_env), broker_addr, 20000, &status); - if (!status.ok()) { - LOG(WARNING) << "Create broker write client failed. broker=" << broker_addr - << ", status=" << status; - return status; - } - - try { - client->closeWriter(response, request); - } catch (apache::thrift::transport::TTransportException& e) { - LOG(WARNING) << "Close broker writer failed. broker:" << broker_addr - << " msg:" << e.what(); - status = client.reopen(); - if (!status.ok()) { - LOG(WARNING) << "Reopen broker writer failed. broker=" << broker_addr - << ", status=" << status; - return status; - } - client->closeWriter(response, request); - } - } catch (apache::thrift::TException& e) { - std::stringstream ss; - ss << "Close broker writer failed, broker:" << broker_addr << " msg:" << e.what(); - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - - VLOG_ROW << "debug: send broker close writer response: " - << apache::thrift::ThriftDebugString(response).c_str(); - - if (response.statusCode != TBrokerOperationStatusCode::OK) { - std::stringstream ss; - ss << "Close broker writer failed, broker:" << broker_addr << " msg:" << response.message; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - - _is_closed = true; - return Status::OK(); -} - +} // end namespace io } // end namespace doris diff --git a/be/src/io/broker_writer.h b/be/src/io/fs/broker_file_writer.h similarity index 61% rename from be/src/io/broker_writer.h rename to be/src/io/fs/broker_file_writer.h index 1155fd4e9e..5705cc18df 100644 --- a/be/src/io/broker_writer.h +++ b/be/src/io/fs/broker_file_writer.h @@ -25,7 +25,7 @@ #include "common/status.h" #include "gen_cpp/PaloBrokerService_types.h" #include "gen_cpp/Types_types.h" -#include "io/file_writer.h" +#include "io/fs/file_writer.h" namespace doris { @@ -33,32 +33,35 @@ class ExecEnv; class TBrokerRangeDesc; class TNetworkAddress; +namespace io { + // Reader of broker file -class BrokerWriter : public FileWriter { +class BrokerFileWriter : public FileWriter { public: - BrokerWriter(ExecEnv* env, const std::vector& broker_addresses, - const std::map& properties, const std::string& path, - int64_t start_offset); - virtual ~BrokerWriter(); + BrokerFileWriter(ExecEnv* env, const TNetworkAddress& broker_address, + const std::map& properties, const std::string& path, + int64_t start_offset, FileSystemSPtr fs); + virtual ~BrokerFileWriter(); - virtual Status open() override; + Status close() override; + Status abort() override; + Status appendv(const Slice* data, size_t data_cnt) override; + Status finalize() override; + Status write_at(size_t offset, const Slice& data) override { + return Status::NotSupported("not support"); + } - virtual Status write(const uint8_t* buf, size_t buf_len, size_t* written_len) override; - - virtual Status close() override; +private: + Status _open(); + Status _write(const uint8_t* buf, size_t buf_len, size_t* written_bytes); private: ExecEnv* _env; - const std::vector& _addresses; + const TNetworkAddress _address; const std::map& _properties; - std::string _path; int64_t _cur_offset; - - bool _is_closed; TBrokerFD _fd; - - // TODO: use for retry if one broker down - int _addr_idx; }; +} // end namespace io } // end namespace doris diff --git a/be/src/io/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp similarity index 97% rename from be/src/io/buffered_reader.cpp rename to be/src/io/fs/buffered_reader.cpp index dfad14c8c0..af88fb4b89 100644 --- a/be/src/io/buffered_reader.cpp +++ b/be/src/io/fs/buffered_reader.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "io/buffered_reader.h" +#include "io/fs/buffered_reader.h" #include #include @@ -26,6 +26,7 @@ #include "util/bit_util.h" namespace doris { +namespace io { BufferedFileStreamReader::BufferedFileStreamReader(io::FileReaderSPtr file, uint64_t offset, uint64_t length, size_t max_buf_size) @@ -67,8 +68,7 @@ Status BufferedFileStreamReader::read_bytes(const uint8_t** buf, uint64_t offset while (has_read < to_read) { size_t loop_read = 0; Slice resutl(_buf.get() + buf_remaining + has_read, to_read - has_read); - IOContext io_context; - RETURN_IF_ERROR(_file->read_at(_buf_end_offset + has_read, resutl, io_context, &loop_read)); + RETURN_IF_ERROR(_file->read_at(_buf_end_offset + has_read, resutl, &loop_read)); _statistics.read_calls++; if (loop_read == 0) { break; @@ -88,4 +88,5 @@ Status BufferedFileStreamReader::read_bytes(Slice& slice, uint64_t offset) { return read_bytes((const uint8_t**)&slice.data, offset, slice.size); } +} // namespace io } // namespace doris diff --git a/be/src/io/buffered_reader.h b/be/src/io/fs/buffered_reader.h similarity index 98% rename from be/src/io/buffered_reader.h rename to be/src/io/fs/buffered_reader.h index 8bab3b1a88..15f2f524d7 100644 --- a/be/src/io/buffered_reader.h +++ b/be/src/io/fs/buffered_reader.h @@ -27,6 +27,7 @@ #include "util/runtime_profile.h" namespace doris { +namespace io { /** * Load all the needed data in underlying buffer, so the caller does not need to prepare the data container. @@ -78,4 +79,5 @@ private: size_t _max_buf_size; }; +} // namespace io } // namespace doris diff --git a/be/src/io/fs/file_reader.cpp b/be/src/io/fs/file_reader.cpp new file mode 100644 index 0000000000..05de1ba499 --- /dev/null +++ b/be/src/io/fs/file_reader.cpp @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/file_reader.h" + +#include "io/fs/file_system.h" +#include "io/io_common.h" +#include "util/async_io.h" + +namespace doris { +namespace io { + +Status FileReader::read_at(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) { + if (bthread_self() == 0) { + return read_at_impl(offset, result, bytes_read, io_ctx); + } + Status s; + auto task = [&] { s = read_at_impl(offset, result, bytes_read, io_ctx); }; + AsyncIO::run_task(task, fs()->type()); + return s; +} + +} // namespace io +} // namespace doris diff --git a/be/src/io/fs/file_reader.h b/be/src/io/fs/file_reader.h index 542147bb90..c429b36ed3 100644 --- a/be/src/io/fs/file_reader.h +++ b/be/src/io/fs/file_reader.h @@ -19,16 +19,16 @@ #include "common/status.h" #include "gutil/macros.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/path.h" #include "util/slice.h" namespace doris { -struct IOContext; - namespace io { class FileSystem; +class IOContext; class FileReader { public: @@ -37,10 +37,12 @@ public: DISALLOW_COPY_AND_ASSIGN(FileReader); - virtual Status close() = 0; + /// If io_ctx is not null, + /// the caller must ensure that the IOContext exists during the left cycle of read_at() + Status read_at(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx = nullptr); - virtual Status read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) = 0; + virtual Status close() = 0; virtual const Path& path() const = 0; @@ -49,9 +51,11 @@ public: virtual bool closed() const = 0; virtual std::shared_ptr fs() const = 0; -}; -using FileReaderSPtr = std::shared_ptr; +protected: + virtual Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) = 0; +}; } // namespace io } // namespace doris diff --git a/be/src/io/fs/file_reader_options.cpp b/be/src/io/fs/file_reader_options.cpp index 0f8f74dbc2..f388f51322 100644 --- a/be/src/io/fs/file_reader_options.cpp +++ b/be/src/io/fs/file_reader_options.cpp @@ -20,6 +20,9 @@ namespace doris { namespace io { +FileReaderOptions FileReaderOptions::DEFAULT = + FileReaderOptions(FileCachePolicy::NO_CACHE, NoCachePathPolicy()); + FileCachePolicy cache_type_from_string(const std::string& type) { if (type == "sub_file_cache") { return FileCachePolicy::SUB_FILE_CACHE; diff --git a/be/src/io/fs/file_reader_options.h b/be/src/io/fs/file_reader_options.h index f5a6eaa8f3..cef4a54ea3 100644 --- a/be/src/io/fs/file_reader_options.h +++ b/be/src/io/fs/file_reader_options.h @@ -72,6 +72,8 @@ public: FileCachePolicy cache_type; const CachePathPolicy& path_policy; + + static FileReaderOptions DEFAULT; }; } // namespace io diff --git a/be/src/io/local_file_writer.h b/be/src/io/fs/file_reader_writer_fwd.h similarity index 60% rename from be/src/io/local_file_writer.h rename to be/src/io/fs/file_reader_writer_fwd.h index 48ba05764b..a0e3078095 100644 --- a/be/src/io/local_file_writer.h +++ b/be/src/io/fs/file_reader_writer_fwd.h @@ -14,35 +14,19 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +// This file is copied from +// https://github.com/ClickHouse/ClickHouse/blob/master/src/Interpreters/Cache/FileCache_fwd.h +// and modified by Doris #pragma once - -#include - -#include "io/file_writer.h" - namespace doris { +namespace io { -class RuntimeState; +class FileReader; +class FileWriter; -class LocalFileWriter : public FileWriter { -public: - LocalFileWriter(const std::string& path, int64_t start_offset); +using FileReaderSPtr = std::shared_ptr; +using FileWriterPtr = std::unique_ptr; - ~LocalFileWriter() override; - - Status open() override; - - virtual Status write(const uint8_t* buf, size_t buf_len, size_t* written_len) override; - - virtual Status close() override; - -private: - static Status _check_file_path(const std::string& file_path); - - std::string _path; - int64_t _start_offset; - FILE* _fp; -}; - -} // end namespace doris +} // namespace io +} // namespace doris diff --git a/be/src/io/fs/file_system.cpp b/be/src/io/fs/file_system.cpp new file mode 100644 index 0000000000..212020437a --- /dev/null +++ b/be/src/io/fs/file_system.cpp @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/file_system.h" + +#include "util/async_io.h" + +namespace doris { +namespace io { + +Status FileSystem::create_file(const Path& file, FileWriterPtr* writer) { + auto path = absolute_path(file); + if (bthread_self() == 0) { + return create_file_impl(path, writer); + } + Status s; + auto task = [&] { s = create_file_impl(path, writer); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status FileSystem::open_file(const Path& file, const FileReaderOptions& reader_options, + FileReaderSPtr* reader) { + auto path = absolute_path(file); + if (bthread_self() == 0) { + return open_file_impl(path, reader_options, reader); + } + Status s; + auto task = [&] { s = open_file_impl(path, reader_options, reader); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status FileSystem::create_directory(const Path& dir) { + auto path = absolute_path(dir); + if (bthread_self() == 0) { + return create_directory_impl(path); + } + Status s; + auto task = [&] { s = create_directory_impl(path); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status FileSystem::delete_file(const Path& file) { + auto path = absolute_path(file); + if (bthread_self() == 0) { + return delete_file_impl(path); + } + Status s; + auto task = [&] { s = delete_file_impl(path); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status FileSystem::delete_directory(const Path& dir) { + auto path = absolute_path(dir); + if (bthread_self() == 0) { + return delete_directory_impl(path); + } + Status s; + auto task = [&] { s = delete_directory_impl(path); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status FileSystem::batch_delete(const std::vector& files) { + std::vector abs_files; + for (auto& file : files) { + abs_files.push_back(absolute_path(file)); + } + if (bthread_self() == 0) { + return batch_delete_impl(abs_files); + } + Status s; + auto task = [&] { s = batch_delete_impl(abs_files); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status FileSystem::exists(const Path& path, bool* res) const { + auto fs_path = absolute_path(path); + if (bthread_self() == 0) { + return exists_impl(fs_path, res); + } + Status s; + auto task = [&] { s = exists_impl(fs_path, res); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status FileSystem::file_size(const Path& file, size_t* file_size) const { + auto path = absolute_path(file); + if (bthread_self() == 0) { + return file_size_impl(path, file_size); + } + Status s; + auto task = [&] { s = file_size_impl(path, file_size); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status FileSystem::list(const Path& dir, bool only_file, std::vector* files, + bool* exists) { + auto path = absolute_path(dir); + if (bthread_self() == 0) { + return list_impl(path, only_file, files, exists); + } + Status s; + auto task = [&] { s = list_impl(path, only_file, files, exists); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status FileSystem::rename(const Path& orig_name, const Path& new_name) { + auto orig_path = absolute_path(orig_name); + auto new_path = absolute_path(new_name); + if (bthread_self() == 0) { + return rename_impl(orig_path, new_path); + } + Status s; + auto task = [&] { s = rename_impl(orig_path, new_path); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status FileSystem::rename_dir(const Path& orig_name, const Path& new_name) { + auto orig_path = absolute_path(orig_name); + auto new_path = absolute_path(new_name); + if (bthread_self() == 0) { + return rename_dir_impl(orig_path, new_path); + } + Status s; + auto task = [&] { s = rename_dir_impl(orig_path, new_path); }; + AsyncIO::run_task(task, _type); + return s; +} + +} // namespace io +} // namespace doris diff --git a/be/src/io/fs/file_system.h b/be/src/io/fs/file_system.h index 642c842319..05c58b39a7 100644 --- a/be/src/io/fs/file_system.h +++ b/be/src/io/fs/file_system.h @@ -21,17 +21,14 @@ #include "common/status.h" #include "gutil/macros.h" -#include "io/fs/file_reader.h" #include "io/fs/file_reader_options.h" -#include "io/fs/file_writer.h" +#include "io/fs/file_reader_writer_fwd.h" #include "io/fs/path.h" +#include "io/io_common.h" namespace doris { namespace io { -class FileWriter; -class FileReader; - enum class FileSystemType : uint8_t { LOCAL, S3, @@ -39,40 +36,105 @@ enum class FileSystemType : uint8_t { BROKER, }; +struct FileInfo { + // only file name, no path + std::string file_name; + size_t file_size; + bool is_file; +}; + class FileSystem : public std::enable_shared_from_this { public: + // The following are public interface. + // And derived classes should implement all xxx_impl methods. + Status create_file(const Path& file, FileWriterPtr* writer); + Status open_file(const Path& file, FileReaderSPtr* reader) { + return open_file(file, FileReaderOptions::DEFAULT, reader); + } + Status open_file(const Path& file, const FileReaderOptions& reader_options, + FileReaderSPtr* reader); + Status create_directory(const Path& dir); + Status delete_file(const Path& file); + Status delete_directory(const Path& dir); + Status batch_delete(const std::vector& files); + Status exists(const Path& path, bool* res) const; + Status file_size(const Path& file, size_t* file_size) const; + Status list(const Path& dir, bool only_file, std::vector* files, bool* exists); + Status rename(const Path& orig_name, const Path& new_name); + Status rename_dir(const Path& orig_name, const Path& new_name); + + std::shared_ptr getSPtr() { return shared_from_this(); } + +public: + // the root path of this fs. + // if not empty, all given Path will be "_root_path/path" + const Path& root_path() const { return _root_path; } + // a unique id of this fs. + // used for cache or re-use. + // can be empty if not used + const std::string& id() const { return _id; } + // file system type + FileSystemType type() const { return _type; } + virtual ~FileSystem() = default; + // Each derived class should implement create method to create fs. DISALLOW_COPY_AND_ASSIGN(FileSystem); - virtual Status create_file(const Path& path, FileWriterPtr* writer) = 0; +protected: + /// create file and return a FileWriter + virtual Status create_file_impl(const Path& file, FileWriterPtr* writer) = 0; - virtual Status open_file(const Path& path, const FileReaderOptions& reader_options, - FileReaderSPtr* reader, IOContext* io_ctx) = 0; + /// open file and return a FileReader + virtual Status open_file_impl(const Path& file, const FileReaderOptions& reader_options, + FileReaderSPtr* reader) = 0; - virtual Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) = 0; + /// create directory recursively + virtual Status create_directory_impl(const Path& dir) = 0; - virtual Status delete_file(const Path& path) = 0; + /// delete file. + /// return OK if file does not exist + /// return ERR if not a regular file + virtual Status delete_file_impl(const Path& file) = 0; - // create directory recursively - virtual Status create_directory(const Path& path) = 0; + /// delete all files in "files" + virtual Status batch_delete_impl(const std::vector& files) = 0; - // remove all under directory recursively - virtual Status delete_directory(const Path& path) = 0; + /// remove all under directory recursively + /// return OK if dir does not exist + /// return ERR if not a dir + virtual Status delete_directory_impl(const Path& dir) = 0; - // hard link `src` to `dest` - // FIXME(cyx): Should we move this method to LocalFileSystem? - virtual Status link_file(const Path& src, const Path& dest) = 0; + /// check if path exist + /// return OK and res = 1 means exist, res = 0 means does not exist + /// return ERR otherwise + virtual Status exists_impl(const Path& path, bool* res) const = 0; - virtual Status exists(const Path& path, bool* res) const = 0; + /// return OK and get size of given file, save in "file_size". + /// return ERR otherwise + virtual Status file_size_impl(const Path& file, size_t* file_size) const = 0; - virtual Status file_size(const Path& path, size_t* file_size) const = 0; + /// return OK and list all objects in "dir", save in "files" + /// return ERR otherwise + /// will not traverse dir recursively. + /// if "only_file" is true, will only return regular files, otherwise, return files and subdirs. + /// the existence of dir will be saved in "exists" + /// if "dir" does not exist, it will return Status::OK, but "exists" will to false + virtual Status list_impl(const Path& dir, bool only_file, std::vector* files, + bool* exists) = 0; - virtual Status list(const Path& path, std::vector* files) = 0; + /// rename file from orig_name to new_name + virtual Status rename_impl(const Path& orig_name, const Path& new_name) = 0; - const Path& root_path() const { return _root_path; } - const std::string& id() const { return _id; } - FileSystemType type() const { return _type; } + /// rename dir from orig_name to new_name + virtual Status rename_dir_impl(const Path& orig_name, const Path& new_name) = 0; + + virtual Path absolute_path(const Path& path) const { + if (path.is_absolute()) { + return path; + } + return _root_path / path; + } protected: FileSystem(Path&& root_path, std::string&& id, FileSystemType type) diff --git a/be/src/io/fs/file_writer.h b/be/src/io/fs/file_writer.h index 9f2859a394..dfd8d2f094 100644 --- a/be/src/io/fs/file_writer.h +++ b/be/src/io/fs/file_writer.h @@ -21,16 +21,17 @@ #include "common/status.h" #include "gutil/macros.h" +#include "io/fs/file_reader_writer_fwd.h" +#include "io/fs/file_system.h" #include "io/fs/path.h" #include "util/slice.h" namespace doris { namespace io { -class FileSystem; class FileWriter { public: - FileWriter(Path&& path) : _path(std::move(path)) {} + FileWriter(Path&& path, FileSystemSPtr fs) : _path(std::move(path)), _fs(fs) {} virtual ~FileWriter() = default; DISALLOW_COPY_AND_ASSIGN(FileWriter); @@ -41,7 +42,7 @@ public: // Abnormal close and remove this file. virtual Status abort() = 0; - virtual Status append(const Slice& data) = 0; + Status append(const Slice& data) { return appendv(&data, 1); } virtual Status appendv(const Slice* data, size_t data_cnt) = 0; @@ -51,17 +52,19 @@ public: // FIXME(cyx): Does not seem to be an appropriate interface for file system? virtual Status finalize() = 0; - virtual size_t bytes_appended() const = 0; - - virtual std::shared_ptr fs() const = 0; - const Path& path() const { return _path; } + size_t bytes_appended() const { return _bytes_appended; } + + FileSystemSPtr fs() const { return _fs; } + protected: Path _path; + size_t _bytes_appended = 0; + FileSystemSPtr _fs; + bool _closed = false; + bool _opened = false; }; -using FileWriterPtr = std::unique_ptr; - } // namespace io } // namespace doris diff --git a/be/src/io/fs/fs_utils.cpp b/be/src/io/fs/fs_utils.cpp new file mode 100644 index 0000000000..fbd6a89fc6 --- /dev/null +++ b/be/src/io/fs/fs_utils.cpp @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/fs_utils.h" + +#include +#include + +#include + +namespace doris { +namespace io { + +std::string errno_to_str() { + char buf[1024]; + return fmt::format("({}), {}", errno, strerror_r(errno, buf, 1024)); +} + +std::string errcode_to_str(const std::error_code& ec) { + return fmt::format("({}), {}", ec.value(), ec.message()); +} + +std::string hdfs_error() { + std::stringstream ss; + char buf[1024]; + ss << "(" << errno << "), " << strerror_r(errno, buf, 1024); + ss << ", reason: " << hdfsGetLastError(); + return ss.str(); +} + +} // namespace io +} // namespace doris diff --git a/be/src/io/file_writer.h b/be/src/io/fs/fs_utils.h similarity index 66% rename from be/src/io/file_writer.h rename to be/src/io/fs/fs_utils.h index 5b08f09830..31ca702c32 100644 --- a/be/src/io/file_writer.h +++ b/be/src/io/fs/fs_utils.h @@ -17,23 +17,15 @@ #pragma once -#include - -#include "common/status.h" +#include +#include namespace doris { +namespace io { -class FileWriter { -public: - virtual ~FileWriter() {} +std::string errno_to_str(); +std::string errcode_to_str(const std::error_code& ec); +std::string hdfs_error(); - virtual Status open() = 0; - - // Writes up to count bytes from the buffer pointed buf to the file. - // NOTE: the number of bytes written may be less than count if. - virtual Status write(const uint8_t* buf, size_t buf_len, size_t* written_len) = 0; - - virtual Status close() = 0; -}; - -} // end namespace doris +} // namespace io +} // namespace doris diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index 738a6a3736..219410ac18 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -54,8 +54,8 @@ Status HdfsFileReader::close() { return Status::OK(); } -Status HdfsFileReader::read_at(size_t offset, Slice result, const IOContext& /*io_ctx*/, - size_t* bytes_read) { +Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* /*io_ctx*/) { DCHECK(!closed()); if (offset > _file_size) { return Status::IOError("offset exceeds file size(offset: {}, file size: {}, path: {})", diff --git a/be/src/io/fs/hdfs_file_reader.h b/be/src/io/fs/hdfs_file_reader.h index bd25b1d6ba..2b10baa602 100644 --- a/be/src/io/fs/hdfs_file_reader.h +++ b/be/src/io/fs/hdfs_file_reader.h @@ -19,6 +19,7 @@ #include "io/fs/file_reader.h" #include "io/fs/hdfs_file_system.h" + namespace doris { namespace io { @@ -31,9 +32,6 @@ public: Status close() override; - Status read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) override; - const Path& path() const override { return _path; } size_t size() const override { return _file_size; } @@ -42,6 +40,10 @@ public: FileSystemSPtr fs() const override { return _fs; } +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override; + private: Path _path; size_t _file_size; diff --git a/be/src/io/fs/hdfs_file_system.cpp b/be/src/io/fs/hdfs_file_system.cpp index 8f36600298..e893249eb2 100644 --- a/be/src/io/fs/hdfs_file_system.cpp +++ b/be/src/io/fs/hdfs_file_system.cpp @@ -19,17 +19,22 @@ #include "gutil/hash/hash.h" #include "io/cache/block/cached_remote_file_reader.h" +#include "io/fs/fs_utils.h" #include "io/fs/hdfs_file_reader.h" +#include "io/fs/hdfs_file_writer.h" +#include "io/fs/local_file_system.h" #include "io/hdfs_builder.h" #include "service/backend_options.h" +#include "util/hdfs_util.h" +#include "util/stack_util.h" namespace doris { namespace io { #ifndef CHECK_HDFS_HANDLE -#define CHECK_HDFS_HANDLE(handle) \ - if (!handle) { \ - return Status::InternalError("init Hdfs handle error"); \ +#define CHECK_HDFS_HANDLE(handle) \ + if (!handle) { \ + return Status::IOError("init Hdfs handle error"); \ } #endif @@ -61,9 +66,10 @@ private: void _clean_oldest(); }; -std::shared_ptr HdfsFileSystem::create(const THdfsParams& hdfs_params, - const std::string& path) { - return std::shared_ptr(new HdfsFileSystem(hdfs_params, path)); +Status HdfsFileSystem::create(const THdfsParams& hdfs_params, const std::string& path, + std::shared_ptr* fs) { + (*fs).reset(new HdfsFileSystem(hdfs_params, path)); + return (*fs)->connect(); } HdfsFileSystem::HdfsFileSystem(const THdfsParams& hdfs_params, const std::string& path) @@ -83,32 +89,25 @@ HdfsFileSystem::~HdfsFileSystem() { } } -Status HdfsFileSystem::connect() { +Status HdfsFileSystem::connect_impl() { RETURN_IF_ERROR(HdfsFileSystemCache::instance()->get_connection(_hdfs_params, &_fs_handle)); if (!_fs_handle) { - return Status::InternalError("failed to init Hdfs handle with, please check hdfs params."); + return Status::IOError("failed to init Hdfs handle with, please check hdfs params."); } return Status::OK(); } -Status HdfsFileSystem::create_file(const Path& /*path*/, FileWriterPtr* /*writer*/) { - // auto handle = get_handle(); - // CHECK_HDFS_HANDLE(handle); - // auto hdfs_file = hdfsOpenFile(handle->hdfs_fs, path.string().c_str(), O_WRONLY, 0, 0, 0); - // if (hdfs_file == nullptr) { - // return Status::InternalError("Failed to create file {}", path.string()); - // } - // hdfsCloseFile(handle->hdfs_fs, hdfs_file); - // return Status::OK(); - return Status::NotSupported("Currently not support to create file to HDFS"); +Status HdfsFileSystem::create_file_impl(const Path& file, FileWriterPtr* writer) { + *writer = std::make_unique(file, getSPtr()); + return Status::OK(); } -Status HdfsFileSystem::open_file(const Path& path, FileReaderSPtr* reader, IOContext* /*io_ctx*/) { +Status HdfsFileSystem::open_file_internal(const Path& file, FileReaderSPtr* reader) { CHECK_HDFS_HANDLE(_fs_handle); size_t file_len = 0; - RETURN_IF_ERROR(file_size(path, &file_len)); + RETURN_IF_ERROR(file_size_impl(file, &file_len)); - Path real_path = _covert_path(path); + Path real_path = convert_path(file, _namenode); auto hdfs_file = hdfsOpenFile(_fs_handle->hdfs_fs, real_path.string().c_str(), O_RDONLY, 0, 0, 0); if (hdfs_file == nullptr) { @@ -118,96 +117,244 @@ Status HdfsFileSystem::open_file(const Path& path, FileReaderSPtr* reader, IOCon _fs_handle->dec_ref(); // retry RETURN_IF_ERROR(connect()); - hdfs_file = hdfsOpenFile(_fs_handle->hdfs_fs, path.string().c_str(), O_RDONLY, 0, 0, 0); + hdfs_file = hdfsOpenFile(_fs_handle->hdfs_fs, file.string().c_str(), O_RDONLY, 0, 0, 0); if (hdfs_file == nullptr) { - return Status::InternalError( - "open file failed. (BE: {}) namenode:{}, path:{}, err: {}", - BackendOptions::get_localhost(), _namenode, path.string(), - hdfsGetLastError()); + return Status::IOError("failed to open {}: {}", file.native(), hdfs_error()); } } else { - return Status::InternalError("open file failed. (BE: {}) namenode:{}, path:{}, err: {}", - BackendOptions::get_localhost(), _namenode, path.string(), - hdfsGetLastError()); + return Status::IOError("failed to open {} from cache: {}", file.native(), hdfs_error()); } } *reader = std::make_shared( - path, file_len, _namenode, hdfs_file, + file, file_len, _namenode, hdfs_file, std::static_pointer_cast(shared_from_this())); return Status::OK(); } -Status HdfsFileSystem::delete_file(const Path& path) { +Status HdfsFileSystem::create_directory_impl(const Path& dir) { CHECK_HDFS_HANDLE(_fs_handle); - Path real_path = _covert_path(path); - // The recursive argument `is_recursive` is irrelevant if path is a file. - int is_recursive = 0; - int res = hdfsDelete(_fs_handle->hdfs_fs, real_path.string().c_str(), is_recursive); - if (res == -1) { - return Status::InternalError("Failed to delete file {}", path.string()); - } - return Status::OK(); -} - -Status HdfsFileSystem::create_directory(const Path& path) { - CHECK_HDFS_HANDLE(_fs_handle); - Path real_path = _covert_path(path); + Path real_path = convert_path(dir, _namenode); int res = hdfsCreateDirectory(_fs_handle->hdfs_fs, real_path.string().c_str()); if (res == -1) { - return Status::InternalError("Failed to create directory {}", path.string()); + return Status::IOError("failed to create directory {}: {}", dir.native(), hdfs_error()); } return Status::OK(); } -Status HdfsFileSystem::delete_directory(const Path& path) { +Status HdfsFileSystem::delete_file_impl(const Path& file) { + return delete_internal(file, 0); +} + +Status HdfsFileSystem::delete_directory_impl(const Path& dir) { + return delete_internal(dir, 1); +} + +Status HdfsFileSystem::batch_delete_impl(const std::vector& files) { + for (auto& file : files) { + RETURN_IF_ERROR(delete_file_impl(file)); + } + return Status::OK(); +} + +Status HdfsFileSystem::delete_internal(const Path& path, int is_recursive) { + bool exists = true; + RETURN_IF_ERROR(exists_impl(path, &exists)); + if (!exists) { + return Status::OK(); + } CHECK_HDFS_HANDLE(_fs_handle); - Path real_path = _covert_path(path); - // delete in recursive mode - int is_recursive = 1; + Path real_path = convert_path(path, _namenode); int res = hdfsDelete(_fs_handle->hdfs_fs, real_path.string().c_str(), is_recursive); if (res == -1) { - return Status::InternalError("Failed to delete directory {}", path.string()); + return Status::IOError("failed to delete directory {}: {}", path.native(), hdfs_error()); } return Status::OK(); } -Status HdfsFileSystem::exists(const Path& path, bool* res) const { +Status HdfsFileSystem::exists_impl(const Path& path, bool* res) const { CHECK_HDFS_HANDLE(_fs_handle); - Path real_path = _covert_path(path); + Path real_path = convert_path(path, _namenode); int is_exists = hdfsExists(_fs_handle->hdfs_fs, real_path.string().c_str()); - if (is_exists == 0) { - *res = true; - } else { - *res = false; - } + *res = (is_exists == 0); return Status::OK(); } -Status HdfsFileSystem::file_size(const Path& path, size_t* file_size) const { +Status HdfsFileSystem::file_size_impl(const Path& path, size_t* file_size) const { CHECK_HDFS_HANDLE(_fs_handle); - Path real_path = _covert_path(path); + Path real_path = convert_path(path, _namenode); hdfsFileInfo* file_info = hdfsGetPathInfo(_fs_handle->hdfs_fs, real_path.string().c_str()); if (file_info == nullptr) { - return Status::InternalError("Failed to get file size of {}", path.string()); + return Status::IOError("failed to get file size of {}: {}", path.native(), hdfs_error()); } *file_size = file_info->mSize; hdfsFreeFileInfo(file_info, 1); return Status::OK(); } -Status HdfsFileSystem::list(const Path& path, std::vector* files) { +Status HdfsFileSystem::list_impl(const Path& path, bool only_file, std::vector* files, + bool* exists) { + RETURN_IF_ERROR(exists_impl(path, exists)); + if (!(*exists)) { + return Status::OK(); + } + CHECK_HDFS_HANDLE(_fs_handle); - Path real_path = _covert_path(path); + Path real_path = convert_path(path, _namenode); int numEntries = 0; - hdfsFileInfo* file_info = - hdfsListDirectory(_fs_handle->hdfs_fs, real_path.string().c_str(), &numEntries); - if (file_info == nullptr) { - return Status::InternalError("Failed to list files/directors of {}", path.string()); + hdfsFileInfo* hdfs_file_info = + hdfsListDirectory(_fs_handle->hdfs_fs, real_path.c_str(), &numEntries); + if (hdfs_file_info == nullptr) { + return Status::IOError("failed to list files/directors {}: {}", path.native(), + hdfs_error()); } for (int idx = 0; idx < numEntries; ++idx) { - files->emplace_back(file_info[idx].mName); + auto& file = hdfs_file_info[idx]; + if (only_file && file.mKind == kObjectKindDirectory) { + continue; + } + FileInfo file_info; + file_info.file_name = file.mName; + file_info.file_size = file.mSize; + file_info.is_file = (file.mKind != kObjectKindDirectory); + files->emplace_back(std::move(file_info)); + } + hdfsFreeFileInfo(hdfs_file_info, numEntries); + return Status::OK(); +} + +Status HdfsFileSystem::rename_impl(const Path& orig_name, const Path& new_name) { + Path normal_orig_name = convert_path(orig_name, _namenode); + Path normal_new_name = convert_path(new_name, _namenode); + int ret = hdfsRename(_fs_handle->hdfs_fs, normal_orig_name.c_str(), normal_new_name.c_str()); + if (ret == 0) { + LOG(INFO) << "finished to rename file. orig: " << normal_orig_name + << ", new: " << normal_new_name; + return Status::OK(); + } else { + return Status::IOError("fail to rename from {} to {}: {}", normal_orig_name.native(), + normal_new_name.native(), hdfs_error()); + } + return Status::OK(); +} + +Status HdfsFileSystem::rename_dir_impl(const Path& orig_name, const Path& new_name) { + return rename_impl(orig_name, new_name); +} + +Status HdfsFileSystem::upload_impl(const Path& local_file, const Path& remote_file) { + // 1. open local file for read + FileSystemSPtr local_fs = global_local_filesystem(); + FileReaderSPtr local_reader = nullptr; + RETURN_IF_ERROR(local_fs->open_file(local_file, &local_reader)); + size_t file_len = local_reader->size(); + if (file_len == -1) { + return Status::IOError("failed to get size of file: {}", local_file.string()); + } + + // 2. open remote file for write + FileWriterPtr hdfs_writer = nullptr; + RETURN_IF_ERROR(create_file_impl(remote_file, &hdfs_writer)); + + constexpr size_t buf_sz = 1024 * 1024; + char read_buf[buf_sz]; + size_t left_len = file_len; + size_t read_offset = 0; + size_t bytes_read = 0; + while (left_len > 0) { + size_t read_len = left_len > buf_sz ? buf_sz : left_len; + RETURN_IF_ERROR(local_reader->read_at(read_offset, {read_buf, read_len}, &bytes_read)); + RETURN_IF_ERROR(hdfs_writer->append({read_buf, read_len})); + + read_offset += read_len; + left_len -= read_len; + } + + LOG(INFO) << "finished to write file: " << local_file << ", length: " << file_len; + return Status::OK(); +} + +Status HdfsFileSystem::batch_upload_impl(const std::vector& local_files, + const std::vector& remote_files) { + DCHECK(local_files.size() == remote_files.size()); + for (int i = 0; i < local_files.size(); ++i) { + RETURN_IF_ERROR(upload_impl(local_files[i], remote_files[i])); + } + return Status::OK(); +} + +Status HdfsFileSystem::direct_upload_impl(const Path& remote_file, const std::string& content) { + FileWriterPtr hdfs_writer = nullptr; + RETURN_IF_ERROR(create_file(remote_file, &hdfs_writer)); + RETURN_IF_ERROR(hdfs_writer->append({content})); + return Status::OK(); +} + +Status HdfsFileSystem::upload_with_checksum_impl(const Path& local, const Path& remote_file, + const std::string& checksum) { + std::string temp = remote_file.string() + ".part"; + std::string final_file = remote_file.string() + "." + checksum; + RETURN_IF_ERROR(upload_impl(local, temp)); + return rename_impl(temp, final_file); +} + +Status HdfsFileSystem::download_impl(const Path& remote_file, const Path& local_file) { + // 1. open remote file for read + FileReaderSPtr hdfs_reader = nullptr; + RETURN_IF_ERROR(open_file_internal(remote_file, &hdfs_reader)); + + // 2. remove the existing local file if exist + if (std::filesystem::remove(local_file)) { + LOG(INFO) << "remove the previously exist local file: " << local_file; + } + + // 3. open local file for write + FileSystemSPtr local_fs = global_local_filesystem(); + FileWriterPtr local_writer = nullptr; + RETURN_IF_ERROR(local_fs->create_file(local_file, &local_writer)); + + // 4. read remote and write to local + LOG(INFO) << "read remote file: " << remote_file << " to local: " << local_file; + constexpr size_t buf_sz = 1024 * 1024; + std::unique_ptr read_buf(new char[buf_sz]); + size_t write_offset = 0; + size_t cur_offset = 0; + while (true) { + size_t read_len = 0; + Slice file_slice(read_buf.get(), buf_sz); + RETURN_IF_ERROR(hdfs_reader->read_at(cur_offset, file_slice, &read_len)); + cur_offset += read_len; + if (read_len == 0) { + break; + } + + RETURN_IF_ERROR(local_writer->write_at(write_offset, {read_buf.get(), read_len})); + write_offset += read_len; + } + + return Status::OK(); +} + +Status HdfsFileSystem::direct_download_impl(const Path& remote_file, std::string* content) { + // 1. open remote file for read + FileReaderSPtr hdfs_reader = nullptr; + RETURN_IF_ERROR(open_file_internal(remote_file, &hdfs_reader)); + + constexpr size_t buf_sz = 1024 * 1024; + std::unique_ptr read_buf(new char[buf_sz]); + size_t write_offset = 0; + size_t cur_offset = 0; + while (true) { + size_t read_len = 0; + Slice file_slice(read_buf.get(), buf_sz); + RETURN_IF_ERROR(hdfs_reader->read_at(cur_offset, file_slice, &read_len)); + cur_offset += read_len; + if (read_len == 0) { + break; + } + + content->insert(write_offset, read_buf.get(), read_len); + write_offset += read_len; } - hdfsFreeFileInfo(file_info, numEntries); return Status::OK(); } @@ -215,17 +362,6 @@ HdfsFileSystemHandle* HdfsFileSystem::get_handle() { return _fs_handle; } -Path HdfsFileSystem::_covert_path(const Path& path) const { - // if the format of path is hdfs://ip:port/path, replace it to /path. - // path like hdfs://ip:port/path can't be used by libhdfs3. - Path real_path(path); - if (path.string().find(_namenode) != std::string::npos) { - std::string real_path_str = path.string().substr(_namenode.size()); - real_path = real_path_str; - } - return real_path; -} - // ************* HdfsFileSystemCache ****************** int HdfsFileSystemCache::MAX_CACHE_HANDLE = 64; @@ -234,7 +370,8 @@ Status HdfsFileSystemCache::_create_fs(const THdfsParams& hdfs_params, hdfsFS* f RETURN_IF_ERROR(createHDFSBuilder(hdfs_params, &builder)); hdfsFS hdfs_fs = hdfsBuilderConnect(builder.get()); if (hdfs_fs == nullptr) { - return Status::InternalError("connect to hdfs failed. error: {}", hdfsGetLastError()); + return Status::IOError("faield to connect to hdfs {}: {}", hdfs_params.fs_name, + hdfs_error()); } *fs = hdfs_fs; return Status::OK(); diff --git a/be/src/io/fs/hdfs_file_system.h b/be/src/io/fs/hdfs_file_system.h index 9e5edf6752..41ec7b7243 100644 --- a/be/src/io/fs/hdfs_file_system.h +++ b/be/src/io/fs/hdfs_file_system.h @@ -81,49 +81,43 @@ private: class HdfsFileSystem final : public RemoteFileSystem { public: - static std::shared_ptr create(const THdfsParams& hdfs_params, - const std::string& path); + static Status create(const THdfsParams& hdfs_params, const std::string& path, + std::shared_ptr* fs); ~HdfsFileSystem() override; - Status create_file(const Path& path, FileWriterPtr* writer) override; - - Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override; - - Status delete_file(const Path& path) override; - - Status create_directory(const Path& path) override; - - // Delete all files under path. - Status delete_directory(const Path& path) override; - - Status link_file(const Path& /*src*/, const Path& /*dest*/) override { - return Status::NotSupported("Not supported"); - } - - Status exists(const Path& path, bool* res) const override; - - Status file_size(const Path& path, size_t* file_size) const override; - - Status list(const Path& path, std::vector* files) override; - - Status upload(const Path& /*local_path*/, const Path& /*dest_path*/) override { - return Status::NotSupported("Currently not support to upload file to HDFS"); - } - - Status batch_upload(const std::vector& /*local_paths*/, - const std::vector& /*dest_paths*/) override { - return Status::NotSupported("Currently not support to batch upload file to HDFS"); - } - - Status connect() override; - HdfsFileSystemHandle* get_handle(); -private: - HdfsFileSystem(const THdfsParams& hdfs_params, const std::string& path); +protected: + Status connect_impl() override; + Status create_file_impl(const Path& file, FileWriterPtr* writer) override; + Status open_file_internal(const Path& file, FileReaderSPtr* reader) override; + Status create_directory_impl(const Path& dir) override; + Status delete_file_impl(const Path& file) override; + Status delete_directory_impl(const Path& dir) override; + Status batch_delete_impl(const std::vector& files) override; + Status exists_impl(const Path& path, bool* res) const override; + Status file_size_impl(const Path& file, size_t* file_size) const override; + Status list_impl(const Path& dir, bool only_file, std::vector* files, + bool* exists) override; + Status rename_impl(const Path& orig_name, const Path& new_name) override; + Status rename_dir_impl(const Path& orig_name, const Path& new_name) override; - Path _covert_path(const Path& path) const; + Status upload_impl(const Path& local_file, const Path& remote_file) override; + Status batch_upload_impl(const std::vector& local_files, + const std::vector& remote_files) override; + Status direct_upload_impl(const Path& remote_file, const std::string& content) override; + Status upload_with_checksum_impl(const Path& local_file, const Path& remote_file, + const std::string& checksum) override; + Status download_impl(const Path& remote_file, const Path& local_file) override; + Status direct_download_impl(const Path& remote_file, std::string* content) override; + +private: + Status delete_internal(const Path& path, int is_recursive); + +private: + friend class HdfsFileWriter; + HdfsFileSystem(const THdfsParams& hdfs_params, const std::string& path); const THdfsParams& _hdfs_params; std::string _namenode; // do not use std::shared_ptr or std::unique_ptr diff --git a/be/src/io/fs/hdfs_file_writer.cpp b/be/src/io/fs/hdfs_file_writer.cpp new file mode 100644 index 0000000000..71b3de77e5 --- /dev/null +++ b/be/src/io/fs/hdfs_file_writer.cpp @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/hdfs_file_writer.h" + +#include + +#include "common/logging.h" +#include "io/fs/hdfs_file_system.h" +#include "service/backend_options.h" +#include "util/hdfs_util.h" + +namespace doris { +namespace io { + +HdfsFileWriter::HdfsFileWriter(Path file, FileSystemSPtr fs) : FileWriter(std::move(file), fs) { + _hdfs_fs = (HdfsFileSystem*)_fs.get(); +} + +HdfsFileWriter::~HdfsFileWriter() { + if (_opened) { + close(); + } + CHECK(!_opened || _closed) << "open: " << _opened << ", closed: " << _closed; +} + +Status HdfsFileWriter::close() { + if (_closed) { + return Status::OK(); + } + _closed = true; + if (_hdfs_file == nullptr) { + return Status::OK(); + } + int result = hdfsFlush(_hdfs_fs->_fs_handle->hdfs_fs, _hdfs_file); + if (result == -1) { + std::stringstream ss; + ss << "failed to flush hdfs file. " + << "(BE: " << BackendOptions::get_localhost() << ")" + << "namenode:" << _hdfs_fs->_namenode << " path:" << _path + << ", err: " << hdfsGetLastError(); + LOG(WARNING) << ss.str(); + return Status::InternalError(ss.str()); + } + hdfsCloseFile(_hdfs_fs->_fs_handle->hdfs_fs, _hdfs_file); + _hdfs_file = nullptr; + return Status::OK(); +} + +Status HdfsFileWriter::abort() { + // TODO: should delete remote file + return Status::OK(); +} + +Status HdfsFileWriter::appendv(const Slice* data, size_t data_cnt) { + DCHECK(!_closed); + if (!_opened) { + RETURN_IF_ERROR(_open()); + _opened = true; + } + + for (size_t i = 0; i < data_cnt; i++) { + const Slice& result = data[i]; + size_t left_bytes = result.size; + const char* p = result.data; + while (left_bytes > 0) { + int64_t written_bytes = + hdfsWrite(_hdfs_fs->_fs_handle->hdfs_fs, _hdfs_file, p, left_bytes); + if (written_bytes < 0) { + return Status::InternalError("write hdfs failed. namenode: {}, path: {}, error: {}", + _hdfs_fs->_namenode, _path.native(), + hdfsGetLastError()); + } + left_bytes -= written_bytes; + p += written_bytes; + _bytes_appended += written_bytes; + } + } + return Status::OK(); +} + +// Call this method when there is no more data to write. +// FIXME(cyx): Does not seem to be an appropriate interface for file system? +Status HdfsFileWriter::finalize() { + DCHECK(!_closed); + if (_opened) { + RETURN_IF_ERROR(close()); + } + return Status::OK(); +} + +Status HdfsFileWriter::_open() { + _path = convert_path(_path, _hdfs_fs->_namenode); + std::string hdfs_dir = _path.parent_path().string(); + int exists = hdfsExists(_hdfs_fs->_fs_handle->hdfs_fs, hdfs_dir.c_str()); + if (exists != 0) { + VLOG_NOTICE << "hdfs dir doesn't exist, create it: " << hdfs_dir; + int ret = hdfsCreateDirectory(_hdfs_fs->_fs_handle->hdfs_fs, hdfs_dir.c_str()); + if (ret != 0) { + std::stringstream ss; + ss << "create dir failed. " + << "(BE: " << BackendOptions::get_localhost() << ")" + << " namenode: " << _hdfs_fs->_namenode << " path: " << hdfs_dir + << ", err: " << hdfsGetLastError(); + LOG(WARNING) << ss.str(); + return Status::InternalError(ss.str()); + } + } + // open file + _hdfs_file = hdfsOpenFile(_hdfs_fs->_fs_handle->hdfs_fs, _path.c_str(), O_WRONLY, 0, 0, 0); + if (_hdfs_file == nullptr) { + std::stringstream ss; + ss << "open file failed. " + << "(BE: " << BackendOptions::get_localhost() << ")" + << " namenode:" << _hdfs_fs->_namenode << " path:" << _path + << ", err: " << hdfsGetLastError(); + LOG(WARNING) << ss.str(); + return Status::InternalError(ss.str()); + } + VLOG_NOTICE << "open file. namenode:" << _hdfs_fs->_namenode << ", path:" << _path; + return Status::OK(); +} + +} // end namespace io +} // end namespace doris diff --git a/be/src/io/hdfs_writer.h b/be/src/io/fs/hdfs_file_writer.h similarity index 60% rename from be/src/io/hdfs_writer.h rename to be/src/io/fs/hdfs_file_writer.h index f5ef32a238..92130a93c2 100644 --- a/be/src/io/hdfs_writer.h +++ b/be/src/io/fs/hdfs_file_writer.h @@ -21,33 +21,35 @@ #include #include "gen_cpp/PlanNodes_types.h" -#include "io/file_writer.h" +#include "io/fs/file_writer.h" +#include "io/fs/path.h" #include "io/hdfs_builder.h" namespace doris { -class HDFSWriter : public FileWriter { -public: - HDFSWriter(const std::map& properties, const std::string& path); - ~HDFSWriter(); - Status open() override; +namespace io { - // Writes up to count bytes from the buffer pointed buf to the file. - // NOTE: the number of bytes written may be less than count if. - Status write(const uint8_t* buf, size_t buf_len, size_t* written_len) override; +class HdfsFileSystem; +class HdfsFileWriter : public FileWriter { +public: + HdfsFileWriter(Path file, FileSystemSPtr fs); + ~HdfsFileWriter(); Status close() override; + Status abort() override; + Status appendv(const Slice* data, size_t data_cnt) override; + Status finalize() override; + Status write_at(size_t offset, const Slice& data) override { + return Status::NotSupported("not support"); + } private: - Status _connect(); - void _parse_properties(const std::map& prop); + Status _open(); private: - std::map _properties; - std::string _namenode = ""; - std::string _path = ""; - hdfsFS _hdfs_fs = nullptr; hdfsFile _hdfs_file = nullptr; - bool _closed = false; + // A convenient pointer to _fs + HdfsFileSystem* _hdfs_fs; }; +} // namespace io } // namespace doris diff --git a/be/src/io/fs/local_file_reader.cpp b/be/src/io/fs/local_file_reader.cpp index afff78bea3..7139950a09 100644 --- a/be/src/io/fs/local_file_reader.cpp +++ b/be/src/io/fs/local_file_reader.cpp @@ -54,19 +54,9 @@ Status LocalFileReader::close() { } return Status::OK(); } -Status LocalFileReader::read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) { - if (bthread_self() == 0) { - return read_at_impl(offset, result, io_ctx, bytes_read); - } - Status s; - auto task = [&] { s = read_at_impl(offset, result, io_ctx, bytes_read); }; - AsyncIO::run_task(task, io::FileSystemType::LOCAL); - return s; -} -Status LocalFileReader::read_at_impl(size_t offset, Slice result, const IOContext& /*io_ctx*/, - size_t* bytes_read) { +Status LocalFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* /*io_ctx*/) { DCHECK(!closed()); if (offset > _file_size) { return Status::IOError("offset exceeds file size(offset: {}, file size: {}, path: {})", diff --git a/be/src/io/fs/local_file_reader.h b/be/src/io/fs/local_file_reader.h index e2e4903cc7..1f74e7b4be 100644 --- a/be/src/io/fs/local_file_reader.h +++ b/be/src/io/fs/local_file_reader.h @@ -34,11 +34,6 @@ public: Status close() override; - Status read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) override; - - Status read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, size_t* bytes_read); - const Path& path() const override { return _path; } size_t size() const override { return _file_size; } @@ -47,6 +42,10 @@ public: FileSystemSPtr fs() const override { return _fs; } +private: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override; + private: int _fd = -1; // owned Path _path; diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index 1a12b7375a..b88f8fbc47 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -18,6 +18,7 @@ #include "io/fs/local_file_system.h" #include "io/fs/file_system.h" +#include "io/fs/fs_utils.h" #include "io/fs/local_file_reader.h" #include "io/fs/local_file_writer.h" #include "util/async_io.h" @@ -34,218 +35,155 @@ LocalFileSystem::LocalFileSystem(Path&& root_path, std::string&& id) LocalFileSystem::~LocalFileSystem() = default; -Path LocalFileSystem::absolute_path(const Path& path) const { - if (path.is_absolute()) { - return path; - } - return _root_path / path; -} - -Status LocalFileSystem::create_file(const Path& path, FileWriterPtr* writer) { - if (bthread_self() == 0) { - return create_file_impl(path, writer); - } - Status s; - auto task = [&] { s = create_file_impl(path, writer); }; - AsyncIO::run_task(task, io::FileSystemType::LOCAL); - return s; -} - -Status LocalFileSystem::create_file_impl(const Path& path, FileWriterPtr* writer) { - auto fs_path = absolute_path(path); - int fd = ::open(fs_path.c_str(), O_TRUNC | O_WRONLY | O_CREAT | O_CLOEXEC, 0666); +Status LocalFileSystem::create_file_impl(const Path& file, FileWriterPtr* writer) { + int fd = ::open(file.c_str(), O_TRUNC | O_WRONLY | O_CREAT | O_CLOEXEC, 0666); if (-1 == fd) { - return Status::IOError("cannot open {}: {}", fs_path.native(), std::strerror(errno)); + return Status::IOError("failed to open {}: {}", file.native(), errno_to_str()); } *writer = std::make_unique( - std::move(fs_path), fd, std::static_pointer_cast(shared_from_this())); + std::move(file), fd, std::static_pointer_cast(shared_from_this())); return Status::OK(); } -Status LocalFileSystem::open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) { - if (bthread_self() == 0) { - return open_file_impl(path, reader, io_ctx); - } - Status s; - auto task = [&] { s = open_file_impl(path, reader, io_ctx); }; - AsyncIO::run_task(task, io::FileSystemType::LOCAL); - return s; -} - -Status LocalFileSystem::open_file_impl(const Path& path, FileReaderSPtr* reader, - IOContext* /*io_ctx*/) { - auto fs_path = absolute_path(path); +Status LocalFileSystem::open_file_impl(const Path& file, + const FileReaderOptions& /*reader_options*/, + FileReaderSPtr* reader) { size_t fsize = 0; - RETURN_IF_ERROR(file_size(fs_path, &fsize)); + RETURN_IF_ERROR(file_size_impl(file, &fsize)); int fd = -1; - RETRY_ON_EINTR(fd, open(fs_path.c_str(), O_RDONLY)); + RETRY_ON_EINTR(fd, open(file.c_str(), O_RDONLY)); if (fd < 0) { - return Status::IOError("cannot open {}: {}", fs_path.native(), std::strerror(errno)); + return Status::IOError("failed to open {}: {}", file.native(), errno_to_str()); } *reader = std::make_shared( - std::move(fs_path), fsize, fd, + std::move(file), fsize, fd, std::static_pointer_cast(shared_from_this())); return Status::OK(); } -Status LocalFileSystem::delete_file(const Path& path) { - if (bthread_self() == 0) { - return delete_file_impl(path); +Status LocalFileSystem::create_directory_impl(const Path& dir) { + if (std::filesystem::exists(dir)) { + return Status::IOError("failed to create {}, already exists", dir.native()); } - Status s; - auto task = [&] { s = delete_file_impl(path); }; - AsyncIO::run_task(task, io::FileSystemType::LOCAL); - return s; + std::error_code ec; + std::filesystem::create_directories(dir, ec); + if (ec) { + return Status::IOError("failed to create {}: {}", dir.native(), errcode_to_str(ec)); + } + return Status::OK(); } -Status LocalFileSystem::delete_file_impl(const Path& path) { - auto fs_path = absolute_path(path); - if (!std::filesystem::exists(fs_path)) { +Status LocalFileSystem::delete_file_impl(const Path& file) { + if (!std::filesystem::exists(file)) { return Status::OK(); } - if (!std::filesystem::is_regular_file(fs_path)) { - return Status::IOError("{} is not a file", fs_path.native()); + if (!std::filesystem::is_regular_file(file)) { + return Status::IOError("failed to delete {}, not a file", file.native()); } std::error_code ec; - std::filesystem::remove(fs_path, ec); + std::filesystem::remove(file, ec); if (ec) { - return Status::IOError("cannot delete {}: {}", fs_path.native(), std::strerror(ec.value())); + return Status::IOError("failed to delete {}: {}", file.native(), errcode_to_str(ec)); } return Status::OK(); } -Status LocalFileSystem::create_directory(const Path& path) { - if (bthread_self() == 0) { - return create_directory_impl(path); - } - Status s; - auto task = [&] { s = create_directory_impl(path); }; - AsyncIO::run_task(task, io::FileSystemType::LOCAL); - return s; -} - -Status LocalFileSystem::create_directory_impl(const Path& path) { - auto fs_path = absolute_path(path); - if (std::filesystem::exists(fs_path)) { - return Status::IOError("{} exists", fs_path.native()); - } - std::error_code ec; - std::filesystem::create_directories(fs_path, ec); - if (ec) { - return Status::IOError("cannot create {}: {}", fs_path.native(), std::strerror(ec.value())); - } - return Status::OK(); -} - -Status LocalFileSystem::delete_directory(const Path& path) { - if (bthread_self() == 0) { - return delete_directory_impl(path); - } - Status s; - auto task = [&] { s = delete_directory_impl(path); }; - AsyncIO::run_task(task, io::FileSystemType::LOCAL); - return s; -} - -Status LocalFileSystem::delete_directory_impl(const Path& path) { - auto fs_path = absolute_path(path); - if (!std::filesystem::exists(fs_path)) { +Status LocalFileSystem::delete_directory_impl(const Path& dir) { + if (!std::filesystem::exists(dir)) { return Status::OK(); } - if (!std::filesystem::is_directory(fs_path)) { - return Status::IOError("{} is not a directory", fs_path.native()); + if (!std::filesystem::is_directory(dir)) { + return Status::IOError("failed to delete {}, not a directory", dir.native()); } std::error_code ec; - std::filesystem::remove_all(fs_path, ec); + std::filesystem::remove_all(dir, ec); if (ec) { - return Status::IOError("cannot delete {}: {}", fs_path.native(), std::strerror(ec.value())); + return Status::IOError("failed to delete {}: {}", dir.native(), errcode_to_str(ec)); } return Status::OK(); } +Status LocalFileSystem::batch_delete_impl(const std::vector& files) { + for (auto& file : files) { + RETURN_IF_ERROR(delete_file_impl(file)); + } + return Status::OK(); +} + +Status LocalFileSystem::exists_impl(const Path& path, bool* res) const { + *res = std::filesystem::exists(path); + return Status::OK(); +} + +Status LocalFileSystem::file_size_impl(const Path& file, size_t* file_size) const { + std::error_code ec; + *file_size = std::filesystem::file_size(file, ec); + if (ec) { + return Status::IOError("failed to get file size {}: {}", file.native(), errcode_to_str(ec)); + } + return Status::OK(); +} + +Status LocalFileSystem::list_impl(const Path& dir, bool only_file, std::vector* files, + bool* exists) { + if (!std::filesystem::exists(dir)) { + *exists = false; + return Status::OK(); + } + std::error_code ec; + for (const auto& entry : std::filesystem::directory_iterator(dir, ec)) { + if (only_file && !entry.is_regular_file()) { + continue; + } + FileInfo file_info; + file_info.file_name = entry.path().filename(); + file_info.file_size = entry.file_size(); + file_info.is_file = entry.is_regular_file(); + files->push_back(std::move(file_info)); + } + if (ec) { + return Status::IOError("failed to list {}: {}", dir.native(), errcode_to_str(ec)); + } + return Status::OK(); +} + +Status LocalFileSystem::rename_impl(const Path& orig_name, const Path& new_name) { + std::error_code ec; + std::filesystem::rename(orig_name, new_name, ec); + if (ec) { + return Status::IOError("failed to rename {} to {}: {}", orig_name.native(), + new_name.native(), errcode_to_str(ec)); + } + return Status::OK(); +} + +Status LocalFileSystem::rename_dir_impl(const Path& orig_name, const Path& new_name) { + return rename_impl(orig_name, new_name); +} + Status LocalFileSystem::link_file(const Path& src, const Path& dest) { + auto src_file = absolute_path(src); + auto dest_file = absolute_path(dest); if (bthread_self() == 0) { - return link_file_impl(src, dest); + return link_file_impl(src_file, dest_file); } Status s; - auto task = [&] { s = link_file_impl(src, dest); }; - AsyncIO::run_task(task, io::FileSystemType::LOCAL); + auto task = [&] { s = link_file_impl(src_file, dest_file); }; + AsyncIO::run_task(task, _type); return s; } Status LocalFileSystem::link_file_impl(const Path& src, const Path& dest) { if (::link(src.c_str(), dest.c_str()) != 0) { - return Status::IOError("fail to create hard link: {}. from {} to {}", std::strerror(errno), - src.native(), dest.native()); + return Status::IOError("failed to create hard link from {} to {}: {}", src.native(), + dest.native(), errno_to_str()); } return Status::OK(); } -Status LocalFileSystem::exists(const Path& path, bool* res) const { - if (bthread_self() == 0) { - return exists_impl(path, res); - } - Status s; - auto task = [&] { s = exists_impl(path, res); }; - AsyncIO::run_task(task, io::FileSystemType::LOCAL); - return s; -} +static std::shared_ptr local_fs = io::LocalFileSystem::create(""); -Status LocalFileSystem::exists_impl(const Path& path, bool* res) const { - auto fs_path = absolute_path(path); - *res = std::filesystem::exists(fs_path); - return Status::OK(); -} - -Status LocalFileSystem::file_size(const Path& path, size_t* file_size) const { - if (bthread_self() == 0) { - return file_size_impl(path, file_size); - } - - Status s; - auto task = [&] { s = file_size_impl(path, file_size); }; - AsyncIO::run_task(task, io::FileSystemType::LOCAL); - return s; -} - -Status LocalFileSystem::file_size_impl(const Path& path, size_t* file_size) const { - auto fs_path = absolute_path(path); - std::error_code ec; - *file_size = std::filesystem::file_size(fs_path, ec); - if (ec) { - return Status::IOError("cannot get file size {}: {}", fs_path.native(), - std::strerror(ec.value())); - } - return Status::OK(); -} - -Status LocalFileSystem::list(const Path& path, std::vector* files) { - if (bthread_self() == 0) { - return list_impl(path, files); - } - - Status s; - auto task = [&] { s = list_impl(path, files); }; - AsyncIO::run_task(task, io::FileSystemType::LOCAL); - return s; -} - -Status LocalFileSystem::list_impl(const Path& path, std::vector* files) { - files->clear(); - auto fs_path = absolute_path(path); - std::error_code ec; - for (const auto& entry : std::filesystem::directory_iterator(fs_path, ec)) { - files->push_back(entry.path().filename()); - } - if (ec) { - return Status::IOError("cannot list {}: {}", fs_path.native(), std::strerror(ec.value())); - } - return Status::OK(); -} - -static FileSystemSPtr local_fs = io::LocalFileSystem::create(""); - -const FileSystemSPtr& global_local_filesystem() { +const std::shared_ptr& global_local_filesystem() { return local_fs; } diff --git a/be/src/io/fs/local_file_system.h b/be/src/io/fs/local_file_system.h index 2251904616..44202c8995 100644 --- a/be/src/io/fs/local_file_system.h +++ b/be/src/io/fs/local_file_system.h @@ -25,57 +25,32 @@ namespace io { class LocalFileSystem final : public FileSystem { public: static std::shared_ptr create(Path path, std::string id = ""); - ~LocalFileSystem() override; - Status create_file(const Path& path, FileWriterPtr* writer) override; - - Status create_file_impl(const Path& path, FileWriterPtr* writer); - - Status open_file(const Path& path, const FileReaderOptions& reader_options, - FileReaderSPtr* reader, IOContext* io_ctx) override { - return open_file(path, reader, io_ctx); - } - - Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override; - - Status open_file_impl(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx); - - Status delete_file(const Path& path) override; - - Status delete_file_impl(const Path& path); - - Status create_directory(const Path& path) override; - - Status create_directory_impl(const Path& path); - - Status delete_directory(const Path& path) override; - - Status delete_directory_impl(const Path& path); - - Status link_file(const Path& src, const Path& dest) override; + /// hard link dest file to src file + Status link_file(const Path& src, const Path& dest); +protected: + Status create_file_impl(const Path& file, FileWriterPtr* writer) override; + Status open_file_impl(const Path& file, const FileReaderOptions& reader_options, + FileReaderSPtr* reader) override; + Status create_directory_impl(const Path& dir) override; + Status delete_file_impl(const Path& file) override; + Status delete_directory_impl(const Path& dir) override; + Status batch_delete_impl(const std::vector& files) override; + Status exists_impl(const Path& path, bool* res) const override; + Status file_size_impl(const Path& file, size_t* file_size) const override; + Status list_impl(const Path& dir, bool only_file, std::vector* files, + bool* exists) override; + Status rename_impl(const Path& orig_name, const Path& new_name) override; + Status rename_dir_impl(const Path& orig_name, const Path& new_name) override; Status link_file_impl(const Path& src, const Path& dest); - Status exists(const Path& path, bool* res) const override; - - Status exists_impl(const Path& path, bool* res) const; - - Status file_size(const Path& path, size_t* file_size) const override; - - Status file_size_impl(const Path& path, size_t* file_size) const; - - Status list(const Path& path, std::vector* files) override; - - Status list_impl(const Path& path, std::vector* files); - private: LocalFileSystem(Path&& root_path, std::string&& id = ""); - - Path absolute_path(const Path& path) const; }; -const FileSystemSPtr& global_local_filesystem(); +const std::shared_ptr& global_local_filesystem(); } // namespace io } // namespace doris diff --git a/be/src/io/fs/local_file_writer.cpp b/be/src/io/fs/local_file_writer.cpp index 85a577b958..c141471d98 100644 --- a/be/src/io/fs/local_file_writer.cpp +++ b/be/src/io/fs/local_file_writer.cpp @@ -56,21 +56,21 @@ Status sync_dir(const io::Path& dirname) { namespace io { -LocalFileWriter::LocalFileWriter(Path path, int fd, std::shared_ptr fs) - : FileWriter(std::move(path)), _fd(fd), _fs(std::move(fs)) { +LocalFileWriter::LocalFileWriter(Path path, int fd, FileSystemSPtr fs) + : FileWriter(std::move(path), fs), _fd(fd) { + _opened = true; DorisMetrics::instance()->local_file_open_writing->increment(1); DorisMetrics::instance()->local_file_writer_total->increment(1); } -LocalFileWriter::LocalFileWriter(Path path, int fd) : FileWriter(std::move(path)), _fd(fd) { - DorisMetrics::instance()->local_file_open_writing->increment(1); - DorisMetrics::instance()->local_file_writer_total->increment(1); -} +LocalFileWriter::LocalFileWriter(Path path, int fd) + : LocalFileWriter(path, fd, global_local_filesystem()) {} LocalFileWriter::~LocalFileWriter() { - if (!_closed) { - WARN_IF_ERROR(abort(), fmt::format("Cannot abort {}", _path.native())); + if (_opened) { + close(); } + CHECK(!_opened || _closed) << "open: " << _opened << ", closed: " << _closed; } Status LocalFileWriter::close() { @@ -82,14 +82,6 @@ Status LocalFileWriter::abort() { return io::global_local_filesystem()->delete_file(_path); } -Status LocalFileWriter::append(const Slice& data) { - Status st = appendv(&data, 1); - if (st.ok()) { - DorisMetrics::instance()->local_bytes_written_total->increment(data.size); - } - return st; -} - Status LocalFileWriter::appendv(const Slice* data, size_t data_cnt) { DCHECK(!_closed); _dirty = true; @@ -142,48 +134,6 @@ Status LocalFileWriter::appendv(const Slice* data, size_t data_cnt) { return Status::OK(); } -Status LocalFileWriter::finalize() { - DCHECK(!_closed); - if (_dirty) { -#if defined(__linux__) - int flags = SYNC_FILE_RANGE_WRITE; - if (sync_file_range(_fd, 0, 0, flags) < 0) { - return Status::IOError("cannot sync {}: {}", _path.native(), std::strerror(errno)); - } -#endif - } - return Status::OK(); -} - -Status LocalFileWriter::_close(bool sync) { - if (_closed) { - return Status::OK(); - } - if (sync && _dirty) { -#ifdef __APPLE__ - if (fcntl(_fd, F_FULLFSYNC) < 0) { - return Status::IOError("cannot sync {}: {}", _path.native(), std::strerror(errno)); - } -#else - if (0 != ::fdatasync(_fd)) { - return Status::IOError("cannot fdatasync {}: {}", _path.native(), std::strerror(errno)); - } -#endif - RETURN_IF_ERROR(detail::sync_dir(_path.parent_path())); - _dirty = false; - } - _closed = true; - - DorisMetrics::instance()->local_file_open_writing->increment(-1); - DorisMetrics::instance()->file_created_total->increment(1); - DorisMetrics::instance()->local_bytes_written_total->increment(_bytes_appended); - - if (0 != ::close(_fd)) { - return Status::IOError("cannot close {}: {}", _path.native(), std::strerror(errno)); - } - return Status::OK(); -} - Status LocalFileWriter::write_at(size_t offset, const Slice& data) { DCHECK(!_closed); _dirty = true; @@ -204,5 +154,47 @@ Status LocalFileWriter::write_at(size_t offset, const Slice& data) { return Status::OK(); } +Status LocalFileWriter::finalize() { + DCHECK(!_closed); + if (_dirty) { +#if defined(__linux__) + int flags = SYNC_FILE_RANGE_WRITE; + if (sync_file_range(_fd, 0, 0, flags) < 0) { + return Status::IOError("cannot sync {}: {}", _path.native(), std::strerror(errno)); + } +#endif + } + return Status::OK(); +} + +Status LocalFileWriter::_close(bool sync) { + if (_closed) { + return Status::OK(); + } + _closed = true; + if (sync && _dirty) { +#ifdef __APPLE__ + if (fcntl(_fd, F_FULLFSYNC) < 0) { + return Status::IOError("cannot sync {}: {}", _path.native(), std::strerror(errno)); + } +#else + if (0 != ::fdatasync(_fd)) { + return Status::IOError("cannot fdatasync {}: {}", _path.native(), std::strerror(errno)); + } +#endif + RETURN_IF_ERROR(detail::sync_dir(_path.parent_path())); + _dirty = false; + } + + DorisMetrics::instance()->local_file_open_writing->increment(-1); + DorisMetrics::instance()->file_created_total->increment(1); + DorisMetrics::instance()->local_bytes_written_total->increment(_bytes_appended); + + if (0 != ::close(_fd)) { + return Status::IOError("cannot close {}: {}", _path.native(), std::strerror(errno)); + } + return Status::OK(); +} + } // namespace io } // namespace doris diff --git a/be/src/io/fs/local_file_writer.h b/be/src/io/fs/local_file_writer.h index 4cce72b130..e4745893dd 100644 --- a/be/src/io/fs/local_file_writer.h +++ b/be/src/io/fs/local_file_writer.h @@ -28,37 +28,22 @@ namespace io { class LocalFileWriter final : public FileWriter { public: - LocalFileWriter(Path path, int fd, std::shared_ptr fs); - + LocalFileWriter(Path path, int fd, FileSystemSPtr fs); LocalFileWriter(Path path, int fd); - ~LocalFileWriter() override; Status close() override; - Status abort() override; - - Status append(const Slice& data) override; - Status appendv(const Slice* data, size_t data_cnt) override; - Status write_at(size_t offset, const Slice& data) override; - Status finalize() override; - size_t bytes_appended() const override { return _bytes_appended; } - - FileSystemSPtr fs() const override { return _fs; } - private: Status _close(bool sync); private: int _fd; // owned - std::shared_ptr _fs; - size_t _bytes_appended = 0; bool _dirty = false; - bool _closed = false; }; } // namespace io diff --git a/be/src/io/fs/remote_file_system.cpp b/be/src/io/fs/remote_file_system.cpp index 0222152e65..755de9a263 100644 --- a/be/src/io/fs/remote_file_system.cpp +++ b/be/src/io/fs/remote_file_system.cpp @@ -26,21 +26,91 @@ namespace doris { namespace io { -Status RemoteFileSystem::open_file(const Path& path, const FileReaderOptions& reader_options, - FileReaderSPtr* reader, IOContext* io_ctx) { +Status RemoteFileSystem::upload(const Path& local_file, const Path& dest_file) { + auto dest_path = absolute_path(dest_file); if (bthread_self() == 0) { - return open_file_impl(path, reader_options, reader, io_ctx); + return upload_impl(local_file, dest_path); } Status s; - auto task = [&] { s = open_file_impl(path, reader_options, reader, io_ctx); }; - AsyncIO::run_task(task, io::FileSystemType::S3); + auto task = [&] { s = upload_impl(local_file, dest_path); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status RemoteFileSystem::batch_upload(const std::vector& local_files, + const std::vector& remote_files) { + std::vector remote_paths; + for (auto& path : remote_files) { + remote_paths.push_back(absolute_path(path)); + } + if (bthread_self() == 0) { + return batch_upload_impl(local_files, remote_paths); + } + Status s; + auto task = [&] { s = batch_upload_impl(local_files, remote_paths); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status RemoteFileSystem::direct_upload(const Path& remote_file, const std::string& content) { + auto remote_path = absolute_path(remote_file); + if (bthread_self() == 0) { + return direct_upload_impl(remote_path, content); + } + Status s; + auto task = [&] { s = direct_upload_impl(remote_path, content); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status RemoteFileSystem::upload_with_checksum(const Path& local_file, const Path& remote, + const std::string& checksum) { + auto remote_path = absolute_path(remote); + if (bthread_self() == 0) { + return upload_with_checksum_impl(local_file, remote_path, checksum); + } + Status s; + auto task = [&] { s = upload_with_checksum_impl(local_file, remote_path, checksum); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status RemoteFileSystem::download(const Path& remote_file, const Path& local) { + auto remote_path = absolute_path(remote_file); + if (bthread_self() == 0) { + return download_impl(remote_path, local); + } + Status s; + auto task = [&] { s = download_impl(remote_path, local); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status RemoteFileSystem::direct_download(const Path& remote_file, std::string* content) { + auto remote_path = absolute_path(remote_file); + if (bthread_self() == 0) { + return direct_download_impl(remote_path, content); + } + Status s; + auto task = [&] { s = direct_download_impl(remote_path, content); }; + AsyncIO::run_task(task, _type); + return s; +} + +Status RemoteFileSystem::connect() { + if (bthread_self() == 0) { + return connect_impl(); + } + Status s; + auto task = [&] { s = connect_impl(); }; + AsyncIO::run_task(task, _type); return s; } Status RemoteFileSystem::open_file_impl(const Path& path, const FileReaderOptions& reader_options, - FileReaderSPtr* reader, IOContext* io_ctx) { + FileReaderSPtr* reader) { FileReaderSPtr raw_reader; - RETURN_IF_ERROR(open_file(path, &raw_reader, io_ctx)); + RETURN_IF_ERROR(open_file_internal(path, &raw_reader)); switch (reader_options.cache_type) { case io::FileCachePolicy::NO_CACHE: { *reader = raw_reader; @@ -57,11 +127,9 @@ Status RemoteFileSystem::open_file_impl(const Path& path, const FileReaderOption break; } case io::FileCachePolicy::FILE_BLOCK_CACHE: { - DCHECK(io_ctx); StringPiece str(raw_reader->path().native()); std::string cache_path = reader_options.path_policy.get_cache_path(path.native()); - *reader = - std::make_shared(std::move(raw_reader), cache_path, io_ctx); + *reader = std::make_shared(std::move(raw_reader), cache_path); break; } default: { diff --git a/be/src/io/fs/remote_file_system.h b/be/src/io/fs/remote_file_system.h index ff1d2f6961..a62746afd5 100644 --- a/be/src/io/fs/remote_file_system.h +++ b/be/src/io/fs/remote_file_system.h @@ -28,28 +28,53 @@ public: : FileSystem(std::move(root_path), std::move(id), type) {} ~RemoteFileSystem() override = default; - // `local_path` should be an absolute path on local filesystem. - virtual Status upload(const Path& local_path, const Path& dest_path) = 0; + Status upload(const Path& local_file, const Path& dest_file); + Status batch_upload(const std::vector& local_files, + const std::vector& remote_files); + Status direct_upload(const Path& remote_file, const std::string& content); + Status upload_with_checksum(const Path& local_file, const Path& remote, + const std::string& checksum); + Status download(const Path& remote_file, const Path& local); + Status direct_download(const Path& remote_file, std::string* content); - virtual Status batch_upload(const std::vector& local_paths, - const std::vector& dest_paths) = 0; + Status connect(); - virtual Status batch_delete(const std::vector& paths) { - return Status::NotSupported("batch_delete"); - } +protected: + /// connect to remote file system + virtual Status connect_impl() = 0; - virtual Status connect() = 0; + virtual Status open_file_impl(const Path& file, const FileReaderOptions& reader_options, + FileReaderSPtr* reader) override; + /// upload load_file to remote remote_file + /// local_file should be an absolute path on local filesystem. + virtual Status upload_impl(const Path& local_file, const Path& remote_file) = 0; - Status open_file(const Path& path, const FileReaderOptions& reader_options, - FileReaderSPtr* reader, IOContext* io_ctx) override; + /// upload all files in load_files to remote_files + /// path in local_files should be an absolute path on local filesystem. + /// the size of local_files and remote_files must be equal. + virtual Status batch_upload_impl(const std::vector& local_files, + const std::vector& remote_files) = 0; - Status open_file_impl(const Path& path, const FileReaderOptions& reader_options, - FileReaderSPtr* reader, IOContext* io_ctx); + /// save the content in "content" directly to remote file + virtual Status direct_upload_impl(const Path& remote_file, const std::string& content) = 0; - Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override { - return Status::NotSupported("implemented in derived classes"); - } + /// upload local_file to remote_file, + /// and the final remote file name is "remote_file.checksum" + virtual Status upload_with_checksum_impl(const Path& local_file, const Path& remote_file, + const std::string& checksum) = 0; + + /// download remote_file to local_file + /// local_file should be an absolute path on local filesystem. + virtual Status download_impl(const Path& remote_file, const Path& local_file) = 0; + + /// save of content of remote_file directly into "content" + virtual Status direct_download_impl(const Path& remote_file, std::string* content) = 0; + + // The derived class should implement this method. + virtual Status open_file_internal(const Path& file, FileReaderSPtr* reader) = 0; }; +using RemoteFileSystemSPtr = std::shared_ptr; + } // namespace io } // namespace doris diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 3b6ce94034..b60e038d39 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -50,19 +50,8 @@ Status S3FileReader::close() { return Status::OK(); } -Status S3FileReader::read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) { - if (bthread_self() == 0) { - return read_at_impl(offset, result, io_ctx, bytes_read); - } - Status s; - auto task = [&] { s = read_at_impl(offset, result, io_ctx, bytes_read); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - -Status S3FileReader::read_at_impl(size_t offset, Slice result, const IOContext& /*io_ctx*/, - size_t* bytes_read) { +Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* /*io_ctx*/) { DCHECK(!closed()); if (offset > _file_size) { return Status::IOError("offset exceeds file size(offset: {}, file size: {}, path: {})", diff --git a/be/src/io/fs/s3_file_reader.h b/be/src/io/fs/s3_file_reader.h index 9ab21c9f48..c21084ae8d 100644 --- a/be/src/io/fs/s3_file_reader.h +++ b/be/src/io/fs/s3_file_reader.h @@ -34,11 +34,6 @@ public: Status close() override; - Status read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) override; - - Status read_at_impl(size_t offset, Slice result, const IOContext& io_ctx, size_t* bytes_read); - const Path& path() const override { return _path; } size_t size() const override { return _file_size; } @@ -47,6 +42,10 @@ public: FileSystemSPtr fs() const override { return _fs; } +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override; + private: Path _path; size_t _file_size; diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index ee2161975f..d970c9dc4e 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -39,20 +39,34 @@ #include "io/fs/remote_file_system.h" #include "io/fs/s3_file_reader.h" #include "io/fs/s3_file_writer.h" -#include "util/async_io.h" +#include "util/s3_uri.h" +#include "util/s3_util.h" namespace doris { namespace io { #ifndef CHECK_S3_CLIENT -#define CHECK_S3_CLIENT(client) \ - if (!client) { \ - return Status::InternalError("init s3 client error"); \ +#define CHECK_S3_CLIENT(client) \ + if (!client) { \ + return Status::IOError("init s3 client error"); \ } #endif -std::shared_ptr S3FileSystem::create(S3Conf s3_conf, std::string id) { - return std::shared_ptr(new S3FileSystem(std::move(s3_conf), std::move(id))); +#ifndef CHECK_S3_PATH +#define CHECK_S3_PATH(uri, path) \ + S3URI uri(path.string()); \ + RETURN_IF_ERROR(uri.parse()); +#endif + +#ifndef GET_KEY +#define GET_KEY(key, path) \ + std::string key; \ + RETURN_IF_ERROR(get_key(path, &key)); +#endif + +Status S3FileSystem::create(S3Conf s3_conf, std::string id, std::shared_ptr* fs) { + (*fs).reset(new S3FileSystem(std::move(s3_conf), std::move(id))); + return (*fs)->connect(); } S3FileSystem::S3FileSystem(S3Conf&& s3_conf, std::string&& id) @@ -60,11 +74,14 @@ S3FileSystem::S3FileSystem(S3Conf&& s3_conf, std::string&& id) fmt::format("{}/{}/{}", s3_conf.endpoint, s3_conf.bucket, s3_conf.prefix), std::move(id), FileSystemType::S3), _s3_conf(std::move(s3_conf)) { - if (_s3_conf.prefix.size() > 0 && _s3_conf.prefix[0] == '/') { - _s3_conf.prefix = _s3_conf.prefix.substr(1); - } - if (!_s3_conf.prefix.empty() && _s3_conf.prefix.back() == '/') { - _s3_conf.prefix.pop_back(); + // remove the first and last '/' + if (!_s3_conf.prefix.empty()) { + if (_s3_conf.prefix[0] == '/') { + _s3_conf.prefix = _s3_conf.prefix.substr(1); + } + if (_s3_conf.prefix.back() == '/') { + _s3_conf.prefix.pop_back(); + } } _executor = Aws::MakeShared( id.c_str(), config::s3_transfer_executor_pool_size); @@ -72,142 +89,25 @@ S3FileSystem::S3FileSystem(S3Conf&& s3_conf, std::string&& id) S3FileSystem::~S3FileSystem() = default; -Status S3FileSystem::connect() { - if (bthread_self() == 0) { - return connect_impl(); - } - Status s; - auto task = [&] { s = connect_impl(); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - Status S3FileSystem::connect_impl() { std::lock_guard lock(_client_mu); - _client = ClientFactory::instance().create(_s3_conf); + _client = S3ClientFactory::instance().create(_s3_conf); if (!_client) { - return Status::InternalError("failed to init s3 client with {}", _s3_conf.to_string()); + return Status::IOError("failed to init s3 client with {}", _s3_conf.to_string()); } return Status::OK(); } -Status S3FileSystem::upload(const Path& local_path, const Path& dest_path) { - if (bthread_self() == 0) { - return upload_impl(local_path, dest_path); - } - Status s; - auto task = [&] { s = upload_impl(local_path, dest_path); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - -Status S3FileSystem::upload_impl(const Path& local_path, const Path& dest_path) { - auto client = get_client(); - CHECK_S3_CLIENT(client); - - Aws::Transfer::TransferManagerConfiguration transfer_config(_executor.get()); - transfer_config.s3Client = client; - auto transfer_manager = Aws::Transfer::TransferManager::Create(transfer_config); - - auto start = std::chrono::steady_clock::now(); - - auto key = get_key(dest_path); - auto handle = transfer_manager->UploadFile(local_path.native(), _s3_conf.bucket, key, - "text/plain", Aws::Map()); - handle->WaitUntilFinished(); - - auto duration = std::chrono::duration(std::chrono::steady_clock::now() - start); - - if (handle->GetStatus() != Aws::Transfer::TransferStatus::COMPLETED) { - return Status::IOError("failed to upload(endpoint={}, bucket={}, key={}): {}", - _s3_conf.endpoint, _s3_conf.bucket, key, - handle->GetLastError().GetMessage()); - } - - auto file_size = std::filesystem::file_size(local_path); - LOG(INFO) << "Upload " << local_path.native() << " to s3, endpoint=" << _s3_conf.endpoint - << ", bucket=" << _s3_conf.bucket << ", key=" << key - << ", duration=" << duration.count() << ", capacity=" << file_size - << ", tp=" << (file_size) / duration.count(); - +Status S3FileSystem::create_file_impl(const Path& file, FileWriterPtr* writer) { + GET_KEY(key, file); + *writer = std::make_unique(key, get_client(), _s3_conf, getSPtr()); return Status::OK(); } -Status S3FileSystem::batch_upload(const std::vector& local_paths, - const std::vector& dest_paths) { - if (bthread_self() == 0) { - return batch_upload_impl(local_paths, dest_paths); - } - Status s; - auto task = [&] { s = batch_upload_impl(local_paths, dest_paths); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - -Status S3FileSystem::batch_upload_impl(const std::vector& local_paths, - const std::vector& dest_paths) { - auto client = get_client(); - CHECK_S3_CLIENT(client); - - if (local_paths.size() != dest_paths.size()) { - return Status::InvalidArgument("local_paths.size() != dest_paths.size()"); - } - - Aws::Transfer::TransferManagerConfiguration transfer_config(_executor.get()); - transfer_config.s3Client = client; - auto transfer_manager = Aws::Transfer::TransferManager::Create(transfer_config); - - std::vector> handles; - for (int i = 0; i < local_paths.size(); ++i) { - auto key = get_key(dest_paths[i]); - LOG(INFO) << "Start to upload " << local_paths[i].native() - << " to s3, endpoint=" << _s3_conf.endpoint << ", bucket=" << _s3_conf.bucket - << ", key=" << key; - auto handle = - transfer_manager->UploadFile(local_paths[i].native(), _s3_conf.bucket, key, - "text/plain", Aws::Map()); - handles.push_back(std::move(handle)); - } - for (auto& handle : handles) { - handle->WaitUntilFinished(); - if (handle->GetStatus() != Aws::Transfer::TransferStatus::COMPLETED) { - // TODO(cyx): Maybe we can cancel remaining handles. - return Status::IOError(handle->GetLastError().GetMessage()); - } - } - return Status::OK(); -} - -Status S3FileSystem::create_file(const Path& path, FileWriterPtr* writer) { - if (bthread_self() == 0) { - return create_file_impl(path, writer); - } - Status s; - auto task = [&] { s = create_file_impl(path, writer); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - -Status S3FileSystem::create_file_impl(const Path& path, FileWriterPtr* writer) { - *writer = std::make_unique(Path(get_key(path)), get_client(), _s3_conf); - return Status::OK(); -} - -Status S3FileSystem::open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) { - if (bthread_self() == 0) { - return open_file_impl(path, reader, io_ctx); - } - Status s; - auto task = [&] { s = open_file_impl(path, reader, io_ctx); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - -Status S3FileSystem::open_file_impl(const Path& path, FileReaderSPtr* reader, - IOContext* /*io_ctx*/) { +Status S3FileSystem::open_file_internal(const Path& file, FileReaderSPtr* reader) { size_t fsize = 0; - RETURN_IF_ERROR(file_size(path, &fsize)); - auto key = get_key(path); + RETURN_IF_ERROR(file_size_impl(file, &fsize)); + GET_KEY(key, file); auto fs_path = Path(_s3_conf.endpoint) / _s3_conf.bucket / key; *reader = std::make_shared( std::move(fs_path), fsize, std::move(key), _s3_conf.bucket, @@ -215,22 +115,16 @@ Status S3FileSystem::open_file_impl(const Path& path, FileReaderSPtr* reader, return Status::OK(); } -Status S3FileSystem::delete_file(const Path& path) { - if (bthread_self() == 0) { - return delete_file_impl(path); - } - Status s; - auto task = [&] { s = delete_file_impl(path); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; +Status S3FileSystem::create_directory_impl(const Path& dir) { + return Status::OK(); } -Status S3FileSystem::delete_file_impl(const Path& path) { +Status S3FileSystem::delete_file_impl(const Path& file) { auto client = get_client(); CHECK_S3_CLIENT(client); Aws::S3::Model::DeleteObjectRequest request; - auto key = get_key(path); + GET_KEY(key, file); request.WithBucket(_s3_conf.bucket).WithKey(key); auto outcome = client->DeleteObject(request); @@ -238,31 +132,15 @@ Status S3FileSystem::delete_file_impl(const Path& path) { outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { return Status::OK(); } - return Status::IOError("failed to delete object(endpoint={}, bucket={}, key={}): {}", - _s3_conf.endpoint, _s3_conf.bucket, key, - outcome.GetError().GetMessage()); + return Status::IOError("failed to delete file {}: {}", file.native(), error_msg(key, outcome)); } -Status S3FileSystem::create_directory(const Path& path) { - return Status::OK(); -} - -Status S3FileSystem::delete_directory(const Path& path) { - if (bthread_self() == 0) { - return delete_directory_impl(path); - } - Status s; - auto task = [&] { s = delete_directory_impl(path); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - -Status S3FileSystem::delete_directory_impl(const Path& path) { +Status S3FileSystem::delete_directory_impl(const Path& dir) { auto client = get_client(); CHECK_S3_CLIENT(client); Aws::S3::Model::ListObjectsV2Request request; - auto prefix = get_key(path); + GET_KEY(prefix, dir); if (!prefix.empty() && prefix.back() != '/') { prefix.push_back('/'); } @@ -274,9 +152,8 @@ Status S3FileSystem::delete_directory_impl(const Path& path) { do { auto outcome = client->ListObjectsV2(request); if (!outcome.IsSuccess()) { - return Status::IOError("failed to list objects(endpoint={}, bucket={}, prefix={}): {}", - _s3_conf.endpoint, _s3_conf.bucket, prefix, - outcome.GetError().GetMessage()); + return Status::IOError("failed to list objects when delete dir {}: {}", dir.native(), + error_msg(prefix, outcome)); } const auto& result = outcome.GetResult(); Aws::Vector objects; @@ -290,16 +167,13 @@ Status S3FileSystem::delete_directory_impl(const Path& path) { delete_request.SetDelete(std::move(del)); auto delete_outcome = client->DeleteObjects(delete_request); if (!delete_outcome.IsSuccess()) { - return Status::IOError( - "failed to delete objects(endpoint={}, bucket={}, prefix={}): {}", - _s3_conf.endpoint, _s3_conf.bucket, prefix, - delete_outcome.GetError().GetMessage()); + return Status::IOError("failed to delete dir {}: {}", dir.native(), + error_msg(prefix, delete_outcome)); } if (!delete_outcome.GetResult().GetErrors().empty()) { const auto& e = delete_outcome.GetResult().GetErrors().front(); - return Status::IOError("fail to delete object(endpoint={}, bucket={}, key={}): {}", - _s3_conf.endpoint, _s3_conf.bucket, e.GetKey(), - e.GetMessage()); + return Status::IOError("fail to delete object: {}", + error_msg(e.GetKey(), e.GetMessage())); } VLOG_TRACE << "delete " << objects.size() << " s3 objects, endpoint: " << _s3_conf.endpoint @@ -311,104 +185,13 @@ Status S3FileSystem::delete_directory_impl(const Path& path) { return Status::OK(); } -Status S3FileSystem::link_file(const Path& src, const Path& dest) { - return Status::NotSupported("not support"); -} - -Status S3FileSystem::exists(const Path& path, bool* res) const { - if (bthread_self() == 0) { - return exists_impl(path, res); - } - Status s; - auto task = [&] { s = exists_impl(path, res); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - -Status S3FileSystem::exists_impl(const Path& path, bool* res) const { - auto client = get_client(); - CHECK_S3_CLIENT(client); - - Aws::S3::Model::HeadObjectRequest request; - auto key = get_key(path); - request.WithBucket(_s3_conf.bucket).WithKey(key); - - auto outcome = client->HeadObject(request); - if (outcome.IsSuccess()) { - *res = true; - } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - *res = false; - } else { - return Status::IOError("failed to get object head(endpoint={}, bucket={}, key={}): {}", - _s3_conf.endpoint, _s3_conf.bucket, key, - outcome.GetError().GetMessage()); - } - return Status::OK(); -} - -Status S3FileSystem::file_size(const Path& path, size_t* file_size) const { - if (bthread_self() == 0) { - return file_size_impl(path, file_size); - } - Status s; - auto task = [&] { s = file_size_impl(path, file_size); }; - AsyncIO::run_task(task, io::FileSystemType::S3); - return s; -} - -Status S3FileSystem::file_size_impl(const Path& path, size_t* file_size) const { - auto client = get_client(); - CHECK_S3_CLIENT(client); - - Aws::S3::Model::HeadObjectRequest request; - auto key = get_key(path); - request.WithBucket(_s3_conf.bucket).WithKey(key); - - auto outcome = client->HeadObject(request); - if (outcome.IsSuccess()) { - *file_size = outcome.GetResult().GetContentLength(); - } else { - return Status::IOError("failed to get object size(endpoint={}, bucket={}, key={}): {}", - _s3_conf.endpoint, _s3_conf.bucket, key, - outcome.GetError().GetMessage()); - } - return Status::OK(); -} - -Status S3FileSystem::list(const Path& path, std::vector* files) { - auto client = get_client(); - CHECK_S3_CLIENT(client); - - Aws::S3::Model::ListObjectsV2Request request; - auto prefix = get_key(path); - if (!prefix.empty() && prefix.back() != '/') { - prefix.push_back('/'); - } - request.WithBucket(_s3_conf.bucket).WithPrefix(prefix); - bool is_trucated = false; - do { - auto outcome = client->ListObjectsV2(request); - if (!outcome.IsSuccess()) { - return Status::IOError("failed to list objects(endpoint={}, bucket={}, prefix={}): {}", - _s3_conf.endpoint, _s3_conf.bucket, prefix, - outcome.GetError().GetMessage()); - } - for (const auto& obj : outcome.GetResult().GetContents()) { - files->push_back(obj.GetKey().substr(prefix.size())); - } - is_trucated = outcome.GetResult().GetIsTruncated(); - request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); - } while (is_trucated); - return Status::OK(); -} - -Status S3FileSystem::batch_delete(const std::vector& paths) { +Status S3FileSystem::batch_delete_impl(const std::vector& remote_files) { auto client = get_client(); CHECK_S3_CLIENT(client); // `DeleteObjectsRequest` can only contain 1000 keys at most. constexpr size_t max_delete_batch = 1000; - auto path_iter = paths.begin(); + auto path_iter = remote_files.begin(); Aws::S3::Model::DeleteObjectsRequest delete_request; delete_request.SetBucket(_s3_conf.bucket); @@ -416,9 +199,10 @@ Status S3FileSystem::batch_delete(const std::vector& paths) { Aws::S3::Model::Delete del; Aws::Vector objects; auto path_begin = path_iter; - for (; path_iter != paths.end() && (path_iter - path_begin < max_delete_batch); + for (; path_iter != remote_files.end() && (path_iter - path_begin < max_delete_batch); ++path_iter) { - objects.emplace_back().SetKey(get_key(*path_iter)); + GET_KEY(key, *path_iter); + objects.emplace_back().SetKey(key); } if (objects.empty()) { return Status::OK(); @@ -427,29 +211,295 @@ Status S3FileSystem::batch_delete(const std::vector& paths) { delete_request.SetDelete(std::move(del)); auto delete_outcome = client->DeleteObjects(delete_request); if (UNLIKELY(!delete_outcome.IsSuccess())) { - return Status::IOError( - "failed to delete objects(endpoint={}, bucket={}, key[0]={}): {}", - _s3_conf.endpoint, _s3_conf.bucket, objects.front().GetKey(), - delete_outcome.GetError().GetMessage()); + return Status::IOError("failed to delete objects: {}", + error_msg(objects.front().GetKey(), delete_outcome)); } if (UNLIKELY(!delete_outcome.GetResult().GetErrors().empty())) { const auto& e = delete_outcome.GetResult().GetErrors().front(); - return Status::IOError("failed to delete objects(endpoint={}, bucket={}, key={}): {}", - _s3_conf.endpoint, _s3_conf.bucket, e.GetKey(), - delete_outcome.GetError().GetMessage()); + return Status::IOError("failed to delete objects: {}", + error_msg(e.GetKey(), delete_outcome)); } - } while (path_iter != paths.end()); + } while (path_iter != remote_files.end()); return Status::OK(); } -std::string S3FileSystem::get_key(const Path& path) const { - StringPiece str(path.native()); - if (str.starts_with(_root_path.native())) { - return fmt::format("{}/{}", _s3_conf.prefix, str.data() + _root_path.native().size()); +Status S3FileSystem::exists_impl(const Path& path, bool* res) const { + auto client = get_client(); + CHECK_S3_CLIENT(client); + GET_KEY(key, path); + + Aws::S3::Model::HeadObjectRequest request; + request.WithBucket(_s3_conf.bucket).WithKey(key); + + auto outcome = client->HeadObject(request); + if (outcome.IsSuccess()) { + *res = true; + } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { + *res = false; + } else { + return Status::IOError("failed to check exists {}: {}", path.native(), + error_msg(key, outcome)); } - // We consider it as a relative path. - return fmt::format("{}/{}", _s3_conf.prefix, path.native()); + return Status::OK(); +} + +Status S3FileSystem::file_size_impl(const Path& file, size_t* file_size) const { + auto client = get_client(); + CHECK_S3_CLIENT(client); + + Aws::S3::Model::HeadObjectRequest request; + GET_KEY(key, file); + request.WithBucket(_s3_conf.bucket).WithKey(key); + + auto outcome = client->HeadObject(request); + if (outcome.IsSuccess()) { + *file_size = outcome.GetResult().GetContentLength(); + } else { + return Status::IOError("failed to get file size {}, {}", file.native(), + error_msg(key, outcome)); + } + return Status::OK(); +} + +Status S3FileSystem::list_impl(const Path& dir, bool only_file, std::vector* files, + bool* exists) { + // For object storage, this path is always not exist. + // So we ignore this property and set exists to true. + *exists = true; + auto client = get_client(); + CHECK_S3_CLIENT(client); + GET_KEY(prefix, dir); + if (!prefix.empty() && prefix.back() != '/') { + prefix.push_back('/'); + } + + Aws::S3::Model::ListObjectsV2Request request; + request.WithBucket(_s3_conf.bucket).WithPrefix(prefix); + bool is_trucated = false; + do { + auto outcome = client->ListObjectsV2(request); + if (!outcome.IsSuccess()) { + return Status::IOError("failed to list {}: {}", dir.native(), + error_msg(prefix, outcome)); + } + for (const auto& obj : outcome.GetResult().GetContents()) { + std::string key = obj.GetKey(); + bool is_dir = (key.at(key.size() - 1) == '/'); + if (only_file && is_dir) { + continue; + } + FileInfo file_info; + // note: if full path is s3://bucket/path/to/file.txt + // obj.GetKey() will be /path/to/file.txt + file_info.file_name = obj.GetKey().substr(prefix.size()); + file_info.file_size = obj.GetSize(); + file_info.is_file = !is_dir; + files->push_back(std::move(file_info)); + } + is_trucated = outcome.GetResult().GetIsTruncated(); + request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); + } while (is_trucated); + return Status::OK(); +} + +Status S3FileSystem::rename_impl(const Path& orig_name, const Path& new_name) { + RETURN_IF_ERROR(copy(orig_name, new_name)); + return delete_file_impl(orig_name); +} + +Status S3FileSystem::rename_dir_impl(const Path& orig_name, const Path& new_name) { + RETURN_IF_ERROR(copy_dir(orig_name, new_name)); + return delete_directory_impl(orig_name); +} + +Status S3FileSystem::upload_impl(const Path& local_file, const Path& remote_file) { + auto client = get_client(); + CHECK_S3_CLIENT(client); + + Aws::Transfer::TransferManagerConfiguration transfer_config(_executor.get()); + transfer_config.s3Client = client; + auto transfer_manager = Aws::Transfer::TransferManager::Create(transfer_config); + + auto start = std::chrono::steady_clock::now(); + + GET_KEY(key, remote_file); + auto handle = transfer_manager->UploadFile(local_file.native(), _s3_conf.bucket, key, + "text/plain", Aws::Map()); + handle->WaitUntilFinished(); + + auto duration = std::chrono::duration(std::chrono::steady_clock::now() - start); + + if (handle->GetStatus() != Aws::Transfer::TransferStatus::COMPLETED) { + return Status::IOError("failed to upload {}: {}", remote_file.native(), + error_msg(key, handle->GetLastError().GetMessage())); + } + + auto file_size = std::filesystem::file_size(local_file); + LOG(INFO) << "Upload " << local_file.native() << " to s3, endpoint=" << _s3_conf.endpoint + << ", bucket=" << _s3_conf.bucket << ", key=" << key + << ", duration=" << duration.count() << ", capacity=" << file_size + << ", tp=" << (file_size) / duration.count(); + + return Status::OK(); +} + +Status S3FileSystem::batch_upload_impl(const std::vector& local_files, + const std::vector& remote_files) { + auto client = get_client(); + CHECK_S3_CLIENT(client); + + if (local_files.size() != remote_files.size()) { + return Status::InvalidArgument("local_files.size({}) != remote_files.size({})", + local_files.size(), remote_files.size()); + } + + Aws::Transfer::TransferManagerConfiguration transfer_config(_executor.get()); + transfer_config.s3Client = client; + auto transfer_manager = Aws::Transfer::TransferManager::Create(transfer_config); + + std::vector> handles; + for (int i = 0; i < local_files.size(); ++i) { + GET_KEY(key, remote_files[i]); + LOG(INFO) << "Start to upload " << local_files[i].native() + << " to s3, endpoint=" << _s3_conf.endpoint << ", bucket=" << _s3_conf.bucket + << ", key=" << key; + auto handle = + transfer_manager->UploadFile(local_files[i].native(), _s3_conf.bucket, key, + "text/plain", Aws::Map()); + handles.push_back(std::move(handle)); + } + for (auto& handle : handles) { + handle->WaitUntilFinished(); + if (handle->GetStatus() != Aws::Transfer::TransferStatus::COMPLETED) { + // TODO(cyx): Maybe we can cancel remaining handles. + return Status::IOError("failed to upload: {}", + error_msg("", handle->GetLastError().GetMessage())); + } + } + return Status::OK(); +} + +Status S3FileSystem::direct_upload_impl(const Path& remote_file, const std::string& content) { + CHECK_S3_CLIENT(_client); + Aws::S3::Model::PutObjectRequest request; + GET_KEY(key, remote_file); + request.WithBucket(_s3_conf.bucket).WithKey(key); + const std::shared_ptr input_data = + Aws::MakeShared("upload_directly"); + *input_data << content.c_str(); + if (input_data->good()) { + request.SetBody(input_data); + } + if (!input_data->good()) { + return Status::IOError("failed to direct upload {}: failed to read from string", + remote_file.native()); + } + Aws::S3::Model::PutObjectOutcome response = _client->PutObject(request); + if (response.IsSuccess()) { + return Status::OK(); + } else { + return Status::IOError("failed to direct upload {}: {}", remote_file.native(), + error_msg(key, response)); + } +} + +Status S3FileSystem::upload_with_checksum_impl(const Path& local_file, const Path& remote_file, + const std::string& checksum) { + return upload_impl(local_file, remote_file.string() + "." + checksum); +} + +Status S3FileSystem::download_impl(const Path& remote_file, const Path& local_file) { + auto client = get_client(); + CHECK_S3_CLIENT(client); + GET_KEY(key, remote_file); + Aws::S3::Model::GetObjectRequest request; + request.WithBucket(_s3_conf.bucket).WithKey(key); + Aws::S3::Model::GetObjectOutcome response = _client->GetObject(request); + if (response.IsSuccess()) { + Aws::OFStream local_file_s; + local_file_s.open(local_file, std::ios::out | std::ios::binary); + if (local_file_s.good()) { + local_file_s << response.GetResult().GetBody().rdbuf(); + } + if (!local_file_s.good()) { + return Status::IOError("failed to download {}: failed to write file: {}", + remote_file.native(), local_file.native()); + } + } else { + return Status::IOError("failed to download {}: {}", remote_file.native(), + error_msg(key, response)); + } + return Status::OK(); +} + +Status S3FileSystem::direct_download_impl(const Path& remote, std::string* content) { + CHECK_S3_CLIENT(_client); + Aws::S3::Model::GetObjectRequest request; + GET_KEY(key, remote); + request.WithBucket(_s3_conf.bucket).WithKey(key); + Aws::S3::Model::GetObjectOutcome response = _client->GetObject(request); + if (response.IsSuccess()) { + std::stringstream ss; + ss << response.GetResult().GetBody().rdbuf(); + *content = ss.str(); + } else { + return Status::IOError("failed to direct download {}: {}", remote.native(), + error_msg(key, response)); + } + return Status::OK(); +} + +Status S3FileSystem::copy(const Path& src, const Path& dst) { + CHECK_S3_CLIENT(_client); + Aws::S3::Model::CopyObjectRequest request; + GET_KEY(src_key, src); + GET_KEY(dst_key, dst); + request.WithCopySource(_s3_conf.bucket + "/" + src_key) + .WithKey(dst_key) + .WithBucket(_s3_conf.bucket); + Aws::S3::Model::CopyObjectOutcome response = _client->CopyObject(request); + if (response.IsSuccess()) { + return Status::OK(); + } else { + return Status::IOError("failed to copy from {} to {}: {}", src.native(), dst.native(), + error_msg(src_key, response)); + } +} + +Status S3FileSystem::copy_dir(const Path& src, const Path& dst) { + std::vector files; + bool exists = false; + RETURN_IF_ERROR(list_impl(src, true, &files, &exists)); + if (!exists) { + return Status::IOError("path not found: {}", src.native()); + } + if (files.empty()) { + LOG(WARNING) << "Nothing need to copy: " << src << " -> " << dst; + return Status::OK(); + } + for (auto& file : files) { + RETURN_IF_ERROR(copy(src / file.file_name, dst / file.file_name)); + } + return Status::OK(); +} + +Status S3FileSystem::get_key(const Path& path, std::string* key) const { + CHECK_S3_PATH(uri, path); + *key = uri.get_key(); + return Status::OK(); +} + +template +std::string S3FileSystem::error_msg(const std::string& key, const AwsOutcome& outcome) const { + return fmt::format("(endpoint: {}, bucket: {}, key:{}, {}), {}", _s3_conf.endpoint, + _s3_conf.bucket, key, outcome.GetError().GetExceptionName(), + outcome.GetError().GetMessage()); +} + +std::string S3FileSystem::error_msg(const std::string& key, const std::string& err) const { + return fmt::format("(endpoint: {}, bucket: {}, key:{}), {}", _s3_conf.endpoint, _s3_conf.bucket, + key, err); } } // namespace io diff --git a/be/src/io/fs/s3_file_system.h b/be/src/io/fs/s3_file_system.h index 0576421e53..f97e9137cc 100644 --- a/be/src/io/fs/s3_file_system.h +++ b/be/src/io/fs/s3_file_system.h @@ -35,77 +35,62 @@ namespace io { // This class is thread-safe.(Except `set_xxx` method) class S3FileSystem final : public RemoteFileSystem { public: - static std::shared_ptr create(S3Conf s3_conf, std::string id); - + static Status create(S3Conf s3_conf, std::string id, std::shared_ptr* fs); ~S3FileSystem() override; - - Status create_file(const Path& path, FileWriterPtr* writer) override; - - Status create_file_impl(const Path& path, FileWriterPtr* writer); - - Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override; - - Status open_file_impl(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx); - - Status delete_file(const Path& path) override; - - Status delete_file_impl(const Path& path); - - Status create_directory(const Path& path) override; - - // Delete all objects start with path. - Status delete_directory(const Path& path) override; - - Status delete_directory_impl(const Path& path); - - Status link_file(const Path& src, const Path& dest) override; - - Status exists(const Path& path, bool* res) const override; - - Status exists_impl(const Path& path, bool* res) const; - - Status file_size(const Path& path, size_t* file_size) const override; - - Status file_size_impl(const Path& path, size_t* file_size) const; - - Status list(const Path& path, std::vector* files) override; - - Status upload(const Path& local_path, const Path& dest_path) override; - - Status upload_impl(const Path& local_path, const Path& dest_path); - - Status batch_upload(const std::vector& local_paths, - const std::vector& dest_paths) override; - - Status batch_upload_impl(const std::vector& local_paths, - const std::vector& dest_paths); - - Status batch_delete(const std::vector& paths) override; - - Status connect() override; - - Status connect_impl(); + // Guarded by external lock. + void set_conf(S3Conf s3_conf) { _s3_conf = std::move(s3_conf); } std::shared_ptr get_client() const { std::lock_guard lock(_client_mu); return _client; } - // Guarded by external lock. - void set_conf(S3Conf s3_conf) { _s3_conf = std::move(s3_conf); } +protected: + Status connect_impl() override; + Status create_file_impl(const Path& file, FileWriterPtr* writer) override; + Status open_file_internal(const Path& file, FileReaderSPtr* reader) override; + Status create_directory_impl(const Path& dir) override; + Status delete_file_impl(const Path& file) override; + Status delete_directory_impl(const Path& dir) override; + Status batch_delete_impl(const std::vector& files) override; + Status exists_impl(const Path& path, bool* res) const override; + Status file_size_impl(const Path& file, size_t* file_size) const override; + Status list_impl(const Path& dir, bool only_file, std::vector* files, + bool* exists) override; + Status rename_impl(const Path& orig_name, const Path& new_name) override; + Status rename_dir_impl(const Path& orig_name, const Path& new_name) override; - std::string get_key(const Path& path) const; + Status upload_impl(const Path& local_file, const Path& remote_file) override; + Status batch_upload_impl(const std::vector& local_files, + const std::vector& remote_files) override; + Status direct_upload_impl(const Path& remote_file, const std::string& content) override; + Status upload_with_checksum_impl(const Path& local_file, const Path& remote_file, + const std::string& checksum) override; + Status download_impl(const Path& remote_file, const Path& local_file) override; + Status direct_download_impl(const Path& remote_file, std::string* content) override; + + Path absolute_path(const Path& path) const override { + // do nothing + return path; + } private: S3FileSystem(S3Conf&& s3_conf, std::string&& id); + template + std::string error_msg(const std::string& key, const AwsOutcome& outcome) const; + std::string error_msg(const std::string& key, const std::string& err) const; + /// copy file from src to dst + Status copy(const Path& src, const Path& dst); + /// copy dir from src to dst + Status copy_dir(const Path& src, const Path& dst); + Status get_key(const Path& path, std::string* key) const; + private: S3Conf _s3_conf; - - // FIXME(cyx): We can use std::atomic since c++20. - std::shared_ptr _client; + // TODO(cyx): We can use std::atomic since c++20. mutable std::mutex _client_mu; - + std::shared_ptr _client; std::shared_ptr _executor; }; diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 9142a18faf..f613b14e17 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -57,16 +57,17 @@ static const int MAX_SIZE_EACH_PART = 5 * 1024 * 1024; static const char* STREAM_TAG = "S3FileWriter"; S3FileWriter::S3FileWriter(Path path, std::shared_ptr client, - const S3Conf& s3_conf) - : FileWriter(std::move(path)), _client(client), _s3_conf(s3_conf) { + const S3Conf& s3_conf, FileSystemSPtr fs) + : FileWriter(std::move(path), fs), _client(client), _s3_conf(s3_conf) { DorisMetrics::instance()->s3_file_open_writing->increment(1); DorisMetrics::instance()->s3_file_writer_total->increment(1); } S3FileWriter::~S3FileWriter() { - if (!_closed) { - WARN_IF_ERROR(abort(), fmt::format("Cannot abort {}", _path.native())); + if (_opened) { + close(); } + CHECK(!_opened || _closed) << "open: " << _opened << ", closed: " << _closed; } Status S3FileWriter::close() { @@ -91,34 +92,6 @@ Status S3FileWriter::abort() { outcome.GetError().GetMessage()); } -Status S3FileWriter::_open() { - CreateMultipartUploadRequest create_request; - create_request.WithBucket(_s3_conf.bucket).WithKey(_path.native()); - create_request.SetContentType("text/plain"); - - _reset_stream(); - auto outcome = _client->CreateMultipartUpload(create_request); - - if (outcome.IsSuccess()) { - _upload_id = outcome.GetResult().GetUploadId(); - LOG(INFO) << "create multi part upload successfully (endpoint=" << _s3_conf.endpoint - << ", bucket=" << _s3_conf.bucket << ", key=" << _path.native() - << ") upload_id: " << _upload_id; - return Status::OK(); - } - return Status::IOError( - "failed to create multi part upload (endpoint={}, bucket={}, key={}): {}", - _s3_conf.endpoint, _s3_conf.bucket, _path.native(), outcome.GetError().GetMessage()); -} - -Status S3FileWriter::append(const Slice& data) { - Status st = appendv(&data, 1); - if (st.ok()) { - DorisMetrics::instance()->s3_bytes_written_total->increment(data.size); - } - return st; -} - Status S3FileWriter::appendv(const Slice* data, size_t data_cnt) { DCHECK(!_closed); if (!_is_open) { @@ -140,6 +113,34 @@ Status S3FileWriter::appendv(const Slice* data, size_t data_cnt) { return Status::OK(); } +Status S3FileWriter::finalize() { + DCHECK(!_closed); + if (_opened) { + _close(); + } + return Status::OK(); +} + +Status S3FileWriter::_open() { + CreateMultipartUploadRequest create_request; + create_request.WithBucket(_s3_conf.bucket).WithKey(_path.native()); + create_request.SetContentType("text/plain"); + + _reset_stream(); + auto outcome = _client->CreateMultipartUpload(create_request); + + if (outcome.IsSuccess()) { + _upload_id = outcome.GetResult().GetUploadId(); + LOG(INFO) << "create multi part upload successfully (endpoint=" << _s3_conf.endpoint + << ", bucket=" << _s3_conf.bucket << ", key=" << _path.native() + << ") upload_id: " << _upload_id; + return Status::OK(); + } + return Status::IOError( + "failed to create multi part upload when open (endpoint={}, bucket={}, key={}): {}", + _s3_conf.endpoint, _s3_conf.bucket, _path.native(), outcome.GetError().GetMessage()); +} + Status S3FileWriter::_upload_part() { if (_stream_ptr->str().size() == 0) { return Status::OK(); @@ -192,14 +193,6 @@ void S3FileWriter::_reset_stream() { _stream_ptr = Aws::MakeShared(STREAM_TAG, ""); } -Status S3FileWriter::finalize() { - DCHECK(!_closed); - if (_is_open) { - _close(); - } - return Status::OK(); -} - Status S3FileWriter::_close() { if (_closed) { return Status::OK(); @@ -223,7 +216,8 @@ Status S3FileWriter::_close() { if (!compute_outcome.IsSuccess()) { return Status::IOError( - "failed to create multi part upload (endpoint={}, bucket={}, key={}): {}", + "failed to create multi part upload when close (endpoint={}, bucket={}, " + "key={}): {}", _s3_conf.endpoint, _s3_conf.bucket, _path.native(), compute_outcome.GetError().GetMessage()); } @@ -241,9 +235,5 @@ Status S3FileWriter::_close() { return Status::OK(); } -Status S3FileWriter::write_at(size_t offset, const Slice& data) { - return Status::NotSupported("not support"); -} - } // namespace io } // namespace doris diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index 12dd32de9f..87aa58680f 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -36,35 +36,22 @@ namespace io { class S3FileWriter final : public FileWriter { public: - S3FileWriter(Path path, std::shared_ptr client, const S3Conf& s3_conf); + S3FileWriter(Path path, std::shared_ptr client, const S3Conf& s3_conf, + FileSystemSPtr fs); ~S3FileWriter() override; Status close() override; - Status abort() override; - - Status append(const Slice& data) override; - Status appendv(const Slice* data, size_t data_cnt) override; - - Status write_at(size_t offset, const Slice& data) override; - Status finalize() override; - - size_t bytes_appended() const override { return _bytes_appended; } - - FileSystemSPtr fs() const override { return _fs; } - -private: - std::shared_ptr _fs; + Status write_at(size_t offset, const Slice& data) override { + return Status::NotSupported("not support"); + } private: Status _close(); - Status _open(); - Status _upload_part(); - void _reset_stream(); private: @@ -73,7 +60,6 @@ private: std::string _upload_id; bool _is_open = false; bool _closed = false; - size_t _bytes_appended = 0; std::shared_ptr _stream_ptr; // Current Part Num for CompletedPart diff --git a/be/src/io/fs/stream_load_pipe.cpp b/be/src/io/fs/stream_load_pipe.cpp index 9d33556bd2..f0e1be2c6e 100644 --- a/be/src/io/fs/stream_load_pipe.cpp +++ b/be/src/io/fs/stream_load_pipe.cpp @@ -39,8 +39,8 @@ StreamLoadPipe::~StreamLoadPipe() { } } -Status StreamLoadPipe::read_at(size_t /*offset*/, Slice result, const IOContext& /*io_ctx*/, - size_t* bytes_read) { +Status StreamLoadPipe::read_at_impl(size_t /*offset*/, Slice result, size_t* bytes_read, + const IOContext* /*io_ctx*/) { SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->orphan_mem_tracker()); *bytes_read = 0; size_t bytes_req = result.size; @@ -98,8 +98,7 @@ Status StreamLoadPipe::read_one_message(std::unique_ptr* data, size_t // _total_length > 0, read the entire data data->reset(new uint8_t[_total_length]); Slice result(data->get(), _total_length); - IOContext io_ctx; - Status st = read_at(0, result, io_ctx, length); + Status st = read_at(0, result, length); return st; } diff --git a/be/src/io/fs/stream_load_pipe.h b/be/src/io/fs/stream_load_pipe.h index 75401b8c20..33e72c57a1 100644 --- a/be/src/io/fs/stream_load_pipe.h +++ b/be/src/io/fs/stream_load_pipe.h @@ -47,9 +47,6 @@ public: Status append(const ByteBufferPtr& buf) override; - Status read_at(size_t offset, Slice result, const IOContext& io_ctx, - size_t* bytes_read) override; - const Path& path() const override { return _path; } size_t size() const override { return 0; } @@ -72,6 +69,10 @@ public: FileSystemSPtr fs() const override { return nullptr; } +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx) override; + private: // read the next buffer from _buf_queue Status _read_next_buffer(std::unique_ptr* data, size_t* length); diff --git a/be/src/io/hdfs_writer.cpp b/be/src/io/hdfs_writer.cpp deleted file mode 100644 index e6814438f1..0000000000 --- a/be/src/io/hdfs_writer.cpp +++ /dev/null @@ -1,154 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "io/hdfs_writer.h" - -#include - -#include "common/logging.h" -#include "service/backend_options.h" - -namespace doris { - -HDFSWriter::HDFSWriter(const std::map& properties, - const std::string& path) - : _properties(properties), _path(path), _hdfs_fs(nullptr) { - _parse_properties(_properties); -} - -HDFSWriter::~HDFSWriter() { - close(); -} - -Status HDFSWriter::open() { - if (_namenode.empty()) { - LOG(WARNING) << "hdfs properties is incorrect."; - return Status::InternalError("hdfs properties is incorrect"); - } - // if the format of _path is hdfs://ip:port/path, replace it to /path. - // path like hdfs://ip:port/path can't be used by libhdfs3. - if (_path.find(_namenode) != _path.npos) { - _path = _path.substr(_namenode.size()); - } - RETURN_IF_ERROR(_connect()); - if (_hdfs_fs == nullptr) { - return Status::InternalError("HDFS writer open without client"); - } - int exists = hdfsExists(_hdfs_fs, _path.c_str()); - if (exists == 0) { - // the path already exists - return Status::AlreadyExist("{} already exists.", _path); - } - - std::filesystem::path hdfs_path(_path); - std::string hdfs_dir = hdfs_path.parent_path().string(); - exists = hdfsExists(_hdfs_fs, hdfs_dir.c_str()); - if (exists != 0) { - VLOG_NOTICE << "hdfs dir doesn't exist, create it: " << hdfs_dir; - int ret = hdfsCreateDirectory(_hdfs_fs, hdfs_dir.c_str()); - if (ret != 0) { - std::stringstream ss; - ss << "create dir failed. " - << "(BE: " << BackendOptions::get_localhost() << ")" - << " namenode: " << _namenode << " path: " << hdfs_dir - << ", err: " << hdfsGetLastError(); - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - } - // open file - _hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_WRONLY, 0, 0, 0); - if (_hdfs_file == nullptr) { - std::stringstream ss; - ss << "open file failed. " - << "(BE: " << BackendOptions::get_localhost() << ")" - << " namenode:" << _namenode << " path:" << _path << ", err: " << hdfsGetLastError(); - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - VLOG_NOTICE << "open file. namenode:" << _namenode << ", path:" << _path; - return Status::OK(); -} - -Status HDFSWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_len) { - if (buf_len == 0) { - *written_len = 0; - return Status::OK(); - } - int32_t result = hdfsWrite(_hdfs_fs, _hdfs_file, buf, buf_len); - if (result < 0) { - std::stringstream ss; - ss << "write file failed. " - << "(BE: " << BackendOptions::get_localhost() << ")" - << "namenode:" << _namenode << " path:" << _path << ", err: " << hdfsGetLastError(); - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - - *written_len = (unsigned int)result; - return Status::OK(); -} - -Status HDFSWriter::close() { - if (_closed) { - return Status::OK(); - } - _closed = true; - if (_hdfs_fs == nullptr) { - return Status::OK(); - } - if (_hdfs_file == nullptr) { - // Even if there is an error, the resources associated with the hdfsFS will be freed. - hdfsDisconnect(_hdfs_fs); - return Status::OK(); - } - int result = hdfsFlush(_hdfs_fs, _hdfs_file); - if (result == -1) { - std::stringstream ss; - ss << "failed to flush hdfs file. " - << "(BE: " << BackendOptions::get_localhost() << ")" - << "namenode:" << _namenode << " path:" << _path << ", err: " << hdfsGetLastError(); - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - hdfsCloseFile(_hdfs_fs, _hdfs_file); - hdfsDisconnect(_hdfs_fs); - - _hdfs_file = nullptr; - _hdfs_fs = nullptr; - return Status::OK(); -} - -Status HDFSWriter::_connect() { - HDFSCommonBuilder builder; - RETURN_IF_ERROR(createHDFSBuilder(_properties, &builder)); - _hdfs_fs = hdfsBuilderConnect(builder.get()); - if (_hdfs_fs == nullptr) { - return Status::InternalError("connect to hdfs failed. namenode address:{}, error {}", - _namenode, hdfsGetLastError()); - } - return Status::OK(); -} - -void HDFSWriter::_parse_properties(const std::map& prop) { - auto iter = prop.find(FS_KEY); - if (iter != prop.end()) { - _namenode = iter->second; - } -} - -} // end namespace doris diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h new file mode 100644 index 0000000000..3e5bce3fc0 --- /dev/null +++ b/be/src/io/io_common.h @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "gen_cpp/Types_types.h" + +namespace doris { + +enum ReaderType { + READER_QUERY = 0, + READER_ALTER_TABLE = 1, + READER_BASE_COMPACTION = 2, + READER_CUMULATIVE_COMPACTION = 3, + READER_CHECKSUM = 4, + READER_COLD_DATA_COMPACTION = 5, + READER_SEGMENT_COMPACTION = 6, +}; + +namespace io { + +struct FileCacheStatistics { + int64_t num_io_total = 0; + int64_t num_io_hit_cache = 0; + int64_t num_io_bytes_read_total = 0; + int64_t num_io_bytes_read_from_file_cache = 0; + int64_t num_io_bytes_read_from_write_cache = 0; + int64_t num_io_written_in_file_cache = 0; + int64_t num_io_bytes_written_in_file_cache = 0; + int64_t num_io_bytes_skip_cache = 0; +}; + +class IOContext { +public: + IOContext() = default; + + IOContext(const TUniqueId* query_id_, FileCacheStatistics* stats_, bool is_presistent_, + bool use_disposable_cache_, bool read_segment_index_, bool enable_file_cache) + : query_id(query_id_), + is_persistent(is_presistent_), + use_disposable_cache(use_disposable_cache_), + read_segment_index(read_segment_index_), + file_cache_stats(stats_), + enable_file_cache(enable_file_cache) {} + ReaderType reader_type; + const TUniqueId* query_id = nullptr; + bool is_persistent = false; + bool use_disposable_cache = false; + bool read_segment_index = false; + FileCacheStatistics* file_cache_stats = nullptr; + bool enable_file_cache = true; +}; + +} // namespace io +} // namespace doris diff --git a/be/src/io/local_file_writer.cpp b/be/src/io/local_file_writer.cpp deleted file mode 100644 index b6c6dfcefa..0000000000 --- a/be/src/io/local_file_writer.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "io/local_file_writer.h" - -#include "service/backend_options.h" -#include "util/error_util.h" -#include "util/file_utils.h" - -namespace doris { - -LocalFileWriter::LocalFileWriter(const std::string& path, int64_t start_offset) - : _path(path), _start_offset(start_offset), _fp(nullptr) {} - -LocalFileWriter::~LocalFileWriter() { - close(); -} - -Status LocalFileWriter::open() { - RETURN_IF_ERROR(_check_file_path(_path)); - - _fp = fopen(_path.c_str(), "w+"); - if (_fp == nullptr) { - return Status::InternalError("Open file failed. path={}, errno= {}, description={}", _path, - errno, get_str_err_msg()); - } - - if (_start_offset != 0) { - int success = fseek(_fp, _start_offset, SEEK_SET); - if (success != 0) { - return Status::InternalError( - "Seek to start_offset failed. offset={}, errno= {}, description={}", - _start_offset, errno, get_str_err_msg()); - } - } - - return Status::OK(); -} - -Status LocalFileWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_len) { - size_t bytes_written = fwrite(buf, 1, buf_len, _fp); - if (bytes_written < buf_len) { - return Status::InternalError( - "fail to write to file. len={}, path={}, failed with errno={}, description={}", - buf_len, _path, errno, get_str_err_msg()); - } - - *written_len = bytes_written; - return Status::OK(); -} - -Status LocalFileWriter::close() { - if (_fp != nullptr) { - fclose(_fp); - _fp = nullptr; - } - return Status::OK(); -} - -Status LocalFileWriter::_check_file_path(const std::string& file_path) { - // For local file writer, the file_path is a local dir. - // Here we do a simple security verification by checking whether the file exists. - // Because the file path is currently arbitrarily specified by the user, - // Doris is not responsible for ensuring the correctness of the path. - // This is just to prevent overwriting the existing file. - if (FileUtils::check_exist(file_path)) { - return Status::InternalError("File already exists: {}. Host: {}", file_path, - BackendOptions::get_localhost()); - } - - return Status::OK(); -} - -} // end namespace doris diff --git a/be/src/io/s3_writer.cpp b/be/src/io/s3_writer.cpp deleted file mode 100644 index ac9bbf997e..0000000000 --- a/be/src/io/s3_writer.cpp +++ /dev/null @@ -1,138 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "io/s3_writer.h" - -#include -#include -#include -#include - -#include "runtime/exec_env.h" -#include "runtime/tmp_file_mgr.h" -#include "service/backend_options.h" -#include "util/s3_uri.h" -#include "util/s3_util.h" - -namespace doris { - -#ifndef CHECK_S3_CLIENT -#define CHECK_S3_CLIENT(client) \ - if (!client) { \ - return Status::InternalError("init aws s3 client error."); \ - } -#endif - -S3Writer::S3Writer(const std::map& properties, const std::string& path, - int64_t start_offset) - : _properties(properties), - _path(path), - _uri(path), - _client(ClientFactory::instance().create(_properties)) { - std::string tmp_path = ExecEnv::GetInstance()->tmp_file_mgr()->get_tmp_dir_path(); - LOG(INFO) << "init aws s3 client with tmp path " << tmp_path; - if (tmp_path.at(tmp_path.size() - 1) != '/') { - tmp_path.append("/"); - } - _temp_file = std::make_shared( - tmp_path.c_str(), ".doris_tmp", - std::ios_base::binary | std::ios_base::trunc | std::ios_base::in | std::ios_base::out); - DCHECK(_client) << "init aws s3 client error."; -} - -S3Writer::~S3Writer() { - close(); -} - -Status S3Writer::open() { - CHECK_S3_CLIENT(_client); - if (!_uri.parse()) { - return Status::InvalidArgument("s3 uri is invalid: {}", _path); - } - Aws::S3::Model::HeadObjectRequest request; - request.WithBucket(_uri.get_bucket()).WithKey(_uri.get_key()); - Aws::S3::Model::HeadObjectOutcome response = _client->HeadObject(request); - if (response.IsSuccess()) { - return Status::AlreadyExist("{} already exists.", _path); - } else if (response.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return Status::OK(); - } else { - return Status::InternalError("Error: [{}:{}] at {}", response.GetError().GetExceptionName(), - response.GetError().GetMessage(), - BackendOptions::get_localhost()); - } -} - -Status S3Writer::write(const uint8_t* buf, size_t buf_len, size_t* written_len) { - if (buf_len == 0) { - *written_len = 0; - return Status::OK(); - } - if (!_temp_file) { - RETURN_NOT_OK_STATUS_WITH_WARN( - Status::BufferAllocFailed( - "The internal temporary file is not writable for {}. at {}", - strerror(errno), BackendOptions::get_localhost()), - "write temp file error"); - } - _temp_file->write(reinterpret_cast(buf), buf_len); - if (!_temp_file->good()) { - RETURN_NOT_OK_STATUS_WITH_WARN( - Status::BufferAllocFailed( - "Could not append to the internal temporary file for {}. at {}", - strerror(errno), BackendOptions::get_localhost()), - "write temp file error"); - } - *written_len = buf_len; - return Status::OK(); -} - -Status S3Writer::close() { - if (_temp_file) { - RETURN_IF_ERROR(_sync()); - _temp_file.reset(); - } - return Status::OK(); -} - -Status S3Writer::_sync() { - if (!_temp_file) { - RETURN_NOT_OK_STATUS_WITH_WARN( - Status::BufferAllocFailed( - "The internal temporary file is not writable for {}. at {}", - strerror(errno), BackendOptions::get_localhost()), - "write temp file error"); - } - CHECK_S3_CLIENT(_client); - Aws::S3::Model::PutObjectRequest request; - request.WithBucket(_uri.get_bucket()).WithKey(_uri.get_key()); - long offset = _temp_file->tellp(); - _temp_file->seekg(0); - request.SetBody(_temp_file); - request.SetContentLength(offset); - auto response = _client->PutObject(request); - _temp_file->clear(); - _temp_file->seekp(offset); - if (response.IsSuccess()) { - return Status::OK(); - } else { - return Status::InternalError("Error: [{}:{}] at {}", response.GetError().GetExceptionName(), - response.GetError().GetMessage(), - BackendOptions::get_localhost()); - } -} -} // end namespace doris diff --git a/be/src/io/s3_writer.h b/be/src/io/s3_writer.h deleted file mode 100644 index a0ad9954a9..0000000000 --- a/be/src/io/s3_writer.h +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "io/file_writer.h" -#include "util/s3_uri.h" - -namespace Aws { -namespace Utils { -class TempFile; -} -namespace S3 { -class S3Client; -} -} // namespace Aws - -namespace doris { -class S3Writer : public FileWriter { -public: - S3Writer(const std::map& properties, const std::string& path, - int64_t start_offset); - ~S3Writer(); - Status open() override; - - // Writes up to count bytes from the buffer pointed buf to the file. - // NOTE: the number of bytes written may be less than count if. - Status write(const uint8_t* buf, size_t buf_len, size_t* written_len) override; - - Status close() override; - -private: - Status _sync(); - - const std::map& _properties; - std::string _path; - S3URI _uri; - std::shared_ptr _client; - std::shared_ptr _temp_file; -}; - -} // end namespace doris diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt index c37bf8938f..067cab7bc9 100644 --- a/be/src/olap/CMakeLists.txt +++ b/be/src/olap/CMakeLists.txt @@ -35,7 +35,6 @@ add_library(Olap STATIC cumulative_compaction_policy.cpp delete_handler.cpp delta_writer.cpp - file_helper.cpp hll.cpp inverted_index_parser.cpp itoken_extractor.cpp diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index 6bd647a3be..8fd7d357ef 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -44,7 +44,6 @@ #include "gutil/strings/substitute.h" #include "io/fs/local_file_system.h" #include "io/fs/path.h" -#include "olap/file_helper.h" #include "olap/olap_define.h" #include "olap/rowset/beta_rowset.h" #include "olap/rowset/rowset_meta_manager.h" diff --git a/be/src/olap/file_header.h b/be/src/olap/file_header.h new file mode 100644 index 0000000000..1732331a0d --- /dev/null +++ b/be/src/olap/file_header.h @@ -0,0 +1,240 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include + +#include "io/fs/file_reader.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" +#include "olap/lru_cache.h" +#include "olap/olap_common.h" +#include "olap/olap_define.h" +#include "olap/utils.h" +#include "util/debug_util.h" + +namespace doris { + +typedef struct _FixedFileHeader { + // the length of the entire file + uint32_t file_length; + // Checksum of the file's contents except the FileHeader + uint32_t checksum; + // Protobuf length of section + uint32_t protobuf_length; + // Checksum of Protobuf part + uint32_t protobuf_checksum; +} __attribute__((packed)) FixedFileHeader; + +typedef struct _FixedFileHeaderV2 { + uint64_t magic_number; + uint32_t version; + // the length of the entire file + uint64_t file_length; + // Checksum of the file's contents except the FileHeader + uint32_t checksum; + // Protobuf length of section + uint64_t protobuf_length; + // Checksum of Protobuf part + uint32_t protobuf_checksum; +} __attribute__((packed)) FixedFileHeaderV2; + +template +class FileHeader { +public: + FileHeader(const std::string& file_path) : _file_path(file_path) { + memset(&_fixed_file_header, 0, sizeof(_fixed_file_header)); + memset(&_extra_fixed_header, 0, sizeof(_extra_fixed_header)); + _fixed_file_header_size = sizeof(_fixed_file_header); + } + ~FileHeader() {} + + // To calculate the length of the proto part, it needs to be called after the proto is operated, + // and prepare must be called before calling serialize + Status prepare(); + + // call prepare() first, serialize() will write fixed header and protobuffer. + // Write the header to the starting position of the incoming file handle + Status serialize(); + + // read from file, validate file length, signature and alder32 of protobuffer. + // Read the header from the beginning of the incoming file handle + Status deserialize(); + + // Check the validity of Header + // it is actually call deserialize(). + Status validate(); + + uint64_t file_length() const { return _fixed_file_header.file_length; } + uint32_t checksum() const { return _fixed_file_header.checksum; } + const ExtraType& extra() const { return _extra_fixed_header; } + ExtraType* mutable_extra() { return &_extra_fixed_header; } + const MessageType& message() const { return _proto; } + MessageType* mutable_message() { return &_proto; } + uint64_t size() const { + return _fixed_file_header_size + sizeof(_extra_fixed_header) + + _fixed_file_header.protobuf_length; + } + + void set_file_length(uint64_t file_length) { _fixed_file_header.file_length = file_length; } + void set_checksum(uint32_t checksum) { _fixed_file_header.checksum = checksum; } + +private: + std::string _file_path; + FixedFileHeaderV2 _fixed_file_header; + uint32_t _fixed_file_header_size; + + std::string _proto_string; + ExtraType _extra_fixed_header; + MessageType _proto; +}; + +// FileHeader implementation +template +Status FileHeader::prepare() { + try { + if (!_proto.SerializeToString(&_proto_string)) { + LOG(WARNING) << "serialize file header to string error. [path='" << _file_path << "']"; + return Status::Error(); + } + } catch (...) { + LOG(WARNING) << "serialize file header to string error. [path='" << _file_path << "']"; + return Status::Error(); + } + + _fixed_file_header.protobuf_checksum = + olap_adler32(olap_adler32_init(), _proto_string.c_str(), _proto_string.size()); + + _fixed_file_header.checksum = 0; + _fixed_file_header.protobuf_length = _proto_string.size(); + _fixed_file_header.file_length = size(); + _fixed_file_header.version = OLAP_DATA_VERSION_APPLIED; + _fixed_file_header.magic_number = OLAP_FIX_HEADER_MAGIC_NUMBER; + + return Status::OK(); +} + +template +Status FileHeader::serialize() { + // write to file + io::FileWriterPtr file_writer; + RETURN_IF_ERROR(io::global_local_filesystem()->create_file(_file_path, &file_writer)); + RETURN_IF_ERROR(file_writer->write_at( + 0, {(const uint8_t*)&_fixed_file_header, _fixed_file_header_size})); + RETURN_IF_ERROR(file_writer->write_at( + _fixed_file_header_size, + {(const uint8_t*)&_extra_fixed_header, sizeof(_extra_fixed_header)})); + RETURN_IF_ERROR(file_writer->write_at(_fixed_file_header_size + sizeof(_extra_fixed_header), + {_proto_string})); + return file_writer->close(); +} + +template +Status FileHeader::deserialize() { + io::FileReaderSPtr file_reader; + RETURN_IF_ERROR(io::global_local_filesystem()->open_file(_file_path, &file_reader)); + off_t real_file_length = 0; + uint32_t real_protobuf_checksum = 0; + size_t bytes_read = 0; + RETURN_IF_ERROR(file_reader->read_at( + 0, {(const uint8_t*)&_fixed_file_header, _fixed_file_header_size}, &bytes_read)); + + //Status read_at(size_t offset, Slice result, size_t* bytes_read, + // const IOContext* io_ctx = nullptr); + + if (_fixed_file_header.magic_number != OLAP_FIX_HEADER_MAGIC_NUMBER) { + VLOG_TRACE << "old fix header found, magic num=" << _fixed_file_header.magic_number; + FixedFileHeader tmp_header; + RETURN_IF_ERROR(file_reader->read_at(0, {(const uint8_t*)&tmp_header, sizeof(tmp_header)}, + &bytes_read)); + _fixed_file_header.file_length = tmp_header.file_length; + _fixed_file_header.checksum = tmp_header.checksum; + _fixed_file_header.protobuf_length = tmp_header.protobuf_length; + _fixed_file_header.protobuf_checksum = tmp_header.protobuf_checksum; + _fixed_file_header.magic_number = OLAP_FIX_HEADER_MAGIC_NUMBER; + _fixed_file_header.version = OLAP_DATA_VERSION_APPLIED; + _fixed_file_header_size = sizeof(tmp_header); + } + + VLOG_NOTICE << "fix head loaded. file_length=" << _fixed_file_header.file_length + << ", checksum=" << _fixed_file_header.checksum + << ", protobuf_length=" << _fixed_file_header.protobuf_length + << ", magic_number=" << _fixed_file_header.magic_number + << ", version=" << _fixed_file_header.version; + + RETURN_IF_ERROR(file_reader->read_at( + _fixed_file_header_size, + {(const uint8_t*)&_extra_fixed_header, sizeof(_extra_fixed_header)}, &bytes_read)); + + std::unique_ptr buf(new (std::nothrow) char[_fixed_file_header.protobuf_length]); + if (nullptr == buf.get()) { + char errmsg[64]; + LOG(WARNING) << "malloc protobuf buf error. file=" << file_reader->path().native() + << ", error=" << strerror_r(errno, errmsg, 64); + return Status::Error(); + } + RETURN_IF_ERROR(file_reader->read_at(_fixed_file_header_size + sizeof(_extra_fixed_header), + {buf.get(), _fixed_file_header.protobuf_length}, + &bytes_read)); + real_file_length = file_reader->size(); + + if (file_length() != static_cast(real_file_length)) { + LOG(WARNING) << "file length is not match. file=" << file_reader->path().native() + << ", file_length=" << file_length() + << ", real_file_length=" << real_file_length; + return Status::Error(); + } + + // check proto checksum + real_protobuf_checksum = + olap_adler32(olap_adler32_init(), buf.get(), _fixed_file_header.protobuf_length); + + if (real_protobuf_checksum != _fixed_file_header.protobuf_checksum) { + LOG(WARNING) << "checksum is not match. file=" << file_reader->path().native() + << ", expect=" << _fixed_file_header.protobuf_checksum + << ", actual=" << real_protobuf_checksum; + return Status::Error(); + } + + try { + std::string protobuf_str(buf.get(), _fixed_file_header.protobuf_length); + + if (!_proto.ParseFromString(protobuf_str)) { + LOG(WARNING) << "fail to parse file content to protobuf object. file=" + << file_reader->path().native(); + return Status::Error(); + } + } catch (...) { + LOG(WARNING) << "fail to load protobuf. file='" << file_reader->path().native(); + return Status::Error(); + } + + return Status::OK(); +} + +template +Status FileHeader::validate() { + return deserialize(); +} + +} // namespace doris diff --git a/be/src/olap/file_helper.cpp b/be/src/olap/file_helper.cpp deleted file mode 100644 index a1779484f8..0000000000 --- a/be/src/olap/file_helper.cpp +++ /dev/null @@ -1,377 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "olap/file_helper.h" - -#include -#include -#include - -#include -#include - -#include "common/config.h" -#include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/storage_engine.h" -#include "olap/utils.h" -#include "util/debug_util.h" - -using std::string; - -namespace doris { -using namespace ErrorCode; - -FileHandler::FileHandler() : _fd(-1), _wr_length(0), _file_name("") {} - -FileHandler::~FileHandler() { - this->close(); -} - -Status FileHandler::open(const string& file_name, int flag) { - if (_fd != -1 && _file_name == file_name) { - return Status::OK(); - } - - RETURN_IF_ERROR(this->close()); - - _fd = ::open(file_name.c_str(), flag); - - if (_fd < 0) { - char errmsg[64]; - LOG(WARNING) << "failed to open file. [err=" << strerror_r(errno, errmsg, 64) - << ", file_name='" << file_name << "' flag=" << flag << "]"; - if (errno == EEXIST) { - return Status::Error(); - } - return Status::Error(); - } - - VLOG_NOTICE << "success to open file. file_name=" << file_name << ", mode=" << flag - << " fd=" << _fd; - _file_name = file_name; - return Status::OK(); -} - -Status FileHandler::open_with_mode(const string& file_name, int flag, int mode) { - if (_fd != -1 && _file_name == file_name) { - return Status::OK(); - } - - RETURN_IF_ERROR(this->close()); - - _fd = ::open(file_name.c_str(), flag, mode); - - if (_fd < 0) { - char err_buf[64]; - LOG(WARNING) << "failed to open file. [err=" << strerror_r(errno, err_buf, 64) - << " file_name='" << file_name << "' flag=" << flag << " mode=" << mode << "]"; - if (errno == EEXIST) { - return Status::Error(); - } - return Status::Error(); - } - - VLOG_NOTICE << "success to open file. file_name=" << file_name << ", mode=" << mode - << ", fd=" << _fd; - _file_name = file_name; - return Status::OK(); -} - -Status FileHandler::_release() { - return Status::OK(); -} - -Status FileHandler::close() { - if (_fd < 0) { - return Status::OK(); - } - -#ifndef __APPLE__ - // try to sync page cache if have written some bytes - if (_wr_length > 0) { - posix_fadvise(_fd, 0, 0, POSIX_FADV_DONTNEED); - // Clean dirty pages and wait for io queue empty and return - sync_file_range(_fd, 0, 0, SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER); - _wr_length = 0; - } -#endif - - // In some extreme cases (fd is available, but fsync fails) can cause handle leaks - if (::close(_fd) < 0) { - char errmsg[64]; - LOG(WARNING) << "failed to close file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' fd=" << _fd << "]"; - return Status::Error(); - } - - VLOG_NOTICE << "finished to close file. " - << "file_name=" << _file_name << ", fd=" << _fd; - _fd = -1; - _file_name = ""; - _wr_length = 0; - return Status::OK(); -} - -Status FileHandler::pread(void* buf, size_t size, size_t offset) { - char* ptr = reinterpret_cast(buf); - - while (size > 0) { - ssize_t rd_size = ::pread(_fd, ptr, size, offset); - - if (rd_size < 0) { - char errmsg[64]; - LOG(WARNING) << "failed to pread from file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' fd=" << _fd << " size=" << size - << " offset=" << offset << "]"; - return Status::Error(); - } else if (0 == rd_size) { - char errmsg[64]; - LOG(WARNING) << "read unenough from file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' fd=" << _fd << " size=" << size - << " offset=" << offset << "]"; - return Status::Error(); - } - - size -= rd_size; - offset += rd_size; - ptr += rd_size; - } - - return Status::OK(); -} - -Status FileHandler::write(const void* buf, size_t buf_size) { - size_t org_buf_size = buf_size; - const char* ptr = reinterpret_cast(buf); - while (buf_size > 0) { - ssize_t wr_size = ::write(_fd, ptr, buf_size); - - if (wr_size < 0) { - char errmsg[64]; - LOG(WARNING) << "failed to write to file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' fd=" << _fd << " size=" << buf_size - << "]"; - return Status::Error(); - } else if (0 == wr_size) { - char errmsg[64]; - LOG(WARNING) << "write unenough to file. [err=" << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' fd=" << _fd << " size=" << buf_size - << "]"; - return Status::Error(); - } - - buf_size -= wr_size; - ptr += wr_size; - } - _wr_length += org_buf_size; - -#ifndef __APPLE__ - // try to sync page cache if cache size is bigger than threshold - if (_wr_length >= _cache_threshold) { - posix_fadvise(_fd, 0, 0, POSIX_FADV_DONTNEED); - // Clean dirty pages and wait for io queue empty and return - sync_file_range(_fd, 0, 0, SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER); - _wr_length = 0; - } -#endif - - return Status::OK(); -} - -Status FileHandler::pwrite(const void* buf, size_t buf_size, size_t offset) { - const char* ptr = reinterpret_cast(buf); - - size_t org_buf_size = buf_size; - while (buf_size > 0) { - ssize_t wr_size = ::pwrite(_fd, ptr, buf_size, offset); - - if (wr_size < 0) { - char errmsg[64]; - LOG(WARNING) << "failed to pwrite to file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' fd=" << _fd << " size=" << buf_size - << " offset=" << offset << "]"; - return Status::Error(); - } else if (0 == wr_size) { - char errmsg[64]; - LOG(WARNING) << "pwrite unenough to file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' fd=" << _fd << " size=" << buf_size - << " offset=" << offset << "]"; - return Status::Error(); - } - - buf_size -= wr_size; - ptr += wr_size; - offset += wr_size; - } - _wr_length += org_buf_size; - - return Status::OK(); -} - -off_t FileHandler::length() const { - struct stat stat_data; - - if (fstat(_fd, &stat_data) < 0) { - LOG(WARNING) << "fstat error. [fd=" << _fd << "]"; - return -1; - } - - return stat_data.st_size; -} - -FileHandlerWithBuf::FileHandlerWithBuf() : _fp(nullptr), _file_name("") {} - -FileHandlerWithBuf::~FileHandlerWithBuf() { - this->close(); -} - -Status FileHandlerWithBuf::open(const string& file_name, const char* mode) { - if (_fp != nullptr && _file_name == file_name) { - return Status::OK(); - } - - RETURN_IF_ERROR(this->close()); - - _fp = ::fopen(file_name.c_str(), mode); - - if (nullptr == _fp) { - char errmsg[64]; - LOG(WARNING) << "failed to open file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << file_name << "' flag='" << mode << "']"; - if (errno == EEXIST) { - return Status::Error(); - } - return Status::Error(); - } - - VLOG_NOTICE << "success to open file. " - << "file_name=" << file_name << ", mode=" << mode; - _file_name = file_name; - return Status::OK(); -} - -Status FileHandlerWithBuf::open_with_mode(const string& file_name, const char* mode) { - return this->open(file_name, mode); -} - -Status FileHandlerWithBuf::close() { - if (nullptr == _fp) { - return Status::OK(); - } - - // In some cases (fd is available, but fsync fails) can cause handle leaks - if (::fclose(_fp) != 0) { - char errmsg[64]; - LOG(WARNING) << "failed to close file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "']"; - return Status::Error(); - } - - _fp = nullptr; - _file_name = ""; - return Status::OK(); -} - -Status FileHandlerWithBuf::read(void* buf, size_t size) { - if (OLAP_UNLIKELY(nullptr == _fp)) { - LOG(WARNING) << "Fail to write, fp is nullptr!"; - return Status::Error(); - } - - size_t rd_size = ::fread(buf, 1, size, _fp); - - if (rd_size == size) { - return Status::OK(); - } else if (::feof(_fp)) { - char errmsg[64]; - LOG(WARNING) << "read unenough from file. [err=" << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' size=" << size << " rd_size=" << rd_size - << "]"; - return Status::Error(); - } else { - char errmsg[64]; - LOG(WARNING) << "failed to read from file. [err=" << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' size=" << size << "]"; - return Status::Error(); - } -} - -Status FileHandlerWithBuf::pread(void* buf, size_t size, size_t offset) { - if (OLAP_UNLIKELY(nullptr == _fp)) { - LOG(WARNING) << "Fail to write, fp is nullptr!"; - return Status::Error(); - } - - if (0 != ::fseek(_fp, offset, SEEK_SET)) { - char errmsg[64]; - LOG(WARNING) << "failed to seek file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' size=" << size << " offset=" << offset - << "]"; - return Status::Error(); - } - - return this->read(buf, size); -} - -Status FileHandlerWithBuf::write(const void* buf, size_t buf_size) { - if (OLAP_UNLIKELY(nullptr == _fp)) { - LOG(WARNING) << "Fail to write, fp is nullptr!"; - return Status::Error(); - } - - size_t wr_size = ::fwrite(buf, 1, buf_size, _fp); - - if (wr_size != buf_size) { - char errmsg[64]; - LOG(WARNING) << "failed to write to file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' size=" << buf_size << "]"; - return Status::Error(); - } - - return Status::OK(); -} - -Status FileHandlerWithBuf::pwrite(const void* buf, size_t buf_size, size_t offset) { - if (OLAP_UNLIKELY(nullptr == _fp)) { - LOG(WARNING) << "Fail to write, fp is nullptr!"; - return Status::Error(); - } - - if (0 != ::fseek(_fp, offset, SEEK_SET)) { - char errmsg[64]; - LOG(WARNING) << "failed to seek file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' size=" << buf_size - << " offset=" << offset << "]"; - return Status::Error(); - } - - return this->write(buf, buf_size); -} - -off_t FileHandlerWithBuf::length() const { - struct stat stat_data; - - if (stat(_file_name.c_str(), &stat_data) < 0) { - LOG(WARNING) << "fstat error. [file_name='" << _file_name << "']"; - return -1; - } - - return stat_data.st_size; -} - -} // namespace doris diff --git a/be/src/olap/file_helper.h b/be/src/olap/file_helper.h deleted file mode 100644 index 5b5a292968..0000000000 --- a/be/src/olap/file_helper.h +++ /dev/null @@ -1,430 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include -#include -#include - -#include "olap/lru_cache.h" -#include "olap/olap_common.h" -#include "olap/olap_define.h" -#include "olap/utils.h" -#include "util/debug_util.h" - -namespace doris { - -typedef struct FileDescriptor { - int fd; - FileDescriptor(int fd) : fd(fd) {} - ~FileDescriptor() { ::close(fd); } -} FileDescriptor; - -class FileHandler { -public: - FileHandler(); - ~FileHandler(); - - Status open(const std::string& file_name, int flag); - // The argument mode specifies the permissions to use in case a new file is created. - Status open_with_mode(const std::string& file_name, int flag, int mode); - Status close(); - - Status pread(void* buf, size_t size, size_t offset); - Status write(const void* buf, size_t buf_size); - Status pwrite(const void* buf, size_t buf_size, size_t offset); - - int32_t sync() { return 0; } - - off_t tell() const { - off_t res = -1; - - if (-1 == (res = lseek(_fd, 0, SEEK_CUR))) { - char errmsg[64]; - LOG(WARNING) << "fail to tell file. [err=" << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' fd=" << _fd << "]"; - } - - return res; - } - - off_t length() const; - - off_t seek(off_t offset, int whence) { - off_t res = -1; - - if (-1 == (res = lseek(_fd, offset, whence))) { - char errmsg[64]; - LOG(WARNING) << "fail to seek file. [err=" << strerror_r(errno, errmsg, 64) - << "file_name='" << _file_name << "' fd=" << _fd << " offset=" << offset - << " whence=" << whence << "]"; - } - - return res; - } - - const std::string& file_name() { return _file_name; } - - int fd() { return _fd; } - - static void _delete_cache_file_descriptor(const CacheKey& key, void* value) { - FileDescriptor* file_desc = reinterpret_cast(value); - SAFE_DELETE(file_desc); - } - -private: - Status _release(); - - int _fd; - off_t _wr_length; - std::string _file_name; - -#ifndef __APPLE__ - const int64_t _cache_threshold = 1 << 19; -#endif -}; - -class FileHandlerWithBuf { -public: - FileHandlerWithBuf(); - ~FileHandlerWithBuf(); - - Status open(const std::string& file_name, const char* mode); - // The argument mode specifies the permissions to use in case a new file is created. - Status open_with_mode(const std::string& file_name, const char* mode); - Status close(); - - Status read(void* buf, size_t size); - Status pread(void* buf, size_t size, size_t offset); - Status write(const void* buf, size_t buf_size); - Status pwrite(const void* buf, size_t buf_size, size_t offset); - - int32_t sync() { - int32_t res = -1; - if (0 != (res = ::fflush(_fp))) { - char errmsg[64]; - LOG(WARNING) << "fail to fsync file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "']"; - } - return res; - } - - off_t tell() const { - off_t res = -1; - if (-1 == (res = ::ftell(_fp))) { - char errmsg[64]; - LOG(WARNING) << "fail to tell file. [err= " << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "']"; - } - return res; - } - - off_t length() const; - - off_t seek(off_t offset, int whence) { - off_t res = -1; - if (-1 == (res = ::fseek(_fp, offset, whence))) { - char errmsg[64]; - LOG(WARNING) << "fail to seek file. [err=" << strerror_r(errno, errmsg, 64) - << " file_name='" << _file_name << "' offset=" << offset - << " whence=" << whence << "]"; - } - return res; - } - - const std::string& file_name() { return _file_name; } - - int fd() { return ::fileno(_fp); } - -private: - FILE* _fp; - std::string _file_name; -}; - -typedef struct _FixedFileHeader { - // the length of the entire file - uint32_t file_length; - // Checksum of the file's contents except the FileHeader - uint32_t checksum; - // Protobuf length of section - uint32_t protobuf_length; - // Checksum of Protobuf part - uint32_t protobuf_checksum; -} __attribute__((packed)) FixedFileHeader; - -typedef struct _FixedFileHeaderV2 { - uint64_t magic_number; - uint32_t version; - // the length of the entire file - uint64_t file_length; - // Checksum of the file's contents except the FileHeader - uint32_t checksum; - // Protobuf length of section - uint64_t protobuf_length; - // Checksum of Protobuf part - uint32_t protobuf_checksum; -} __attribute__((packed)) FixedFileHeaderV2; - -template -class FileHeader { -public: - FileHeader() { - memset(&_fixed_file_header, 0, sizeof(_fixed_file_header)); - memset(&_extra_fixed_header, 0, sizeof(_extra_fixed_header)); - _fixed_file_header_size = sizeof(_fixed_file_header); - } - ~FileHeader() {} - - // To calculate the length of the proto part, it needs to be called after the proto is operated, and prepare must be called before calling serialize - Status prepare(FileHandlerType* file_handler); - - // call prepare() first, serialize() will write fixed header and protobuffer. - // Write the header to the starting position of the incoming file handle - Status serialize(FileHandlerType* file_handler); - - // read from file, validate file length, signature and alder32 of protobuffer. - // Read the header from the beginning of the incoming file handle - Status unserialize(FileHandlerType* file_handler); - - // Check the validity of Header - // it is actually call unserialize(). - Status validate(const std::string& filename); - - uint64_t file_length() const { return _fixed_file_header.file_length; } - uint32_t checksum() const { return _fixed_file_header.checksum; } - const ExtraType& extra() const { return _extra_fixed_header; } - ExtraType* mutable_extra() { return &_extra_fixed_header; } - const MessageType& message() const { return _proto; } - MessageType* mutable_message() { return &_proto; } - uint64_t size() const { - return _fixed_file_header_size + sizeof(_extra_fixed_header) + - _fixed_file_header.protobuf_length; - } - - void set_file_length(uint64_t file_length) { _fixed_file_header.file_length = file_length; } - void set_checksum(uint32_t checksum) { _fixed_file_header.checksum = checksum; } - -private: - FixedFileHeaderV2 _fixed_file_header; - uint32_t _fixed_file_header_size; - - std::string _proto_string; - ExtraType _extra_fixed_header; - MessageType _proto; -}; - -// FileHandler implementation -template -Status FileHeader::prepare(FileHandlerType* file_handler) { - if (nullptr == file_handler) { - return Status::Error(); - } - - // Use the file name as Signature to prevent problems caused by some misoperations - // _proto.set_signature(basename(file_handler->file_name().c_str())); - - try { - if (!_proto.SerializeToString(&_proto_string)) { - LOG(WARNING) << "serialize file header to string error. [path='" - << file_handler->file_name() << "']"; - return Status::Error(); - } - } catch (...) { - LOG(WARNING) << "serialize file header to string error. [path='" - << file_handler->file_name() << "']"; - return Status::Error(); - } - - _fixed_file_header.protobuf_checksum = - olap_adler32(olap_adler32_init(), _proto_string.c_str(), _proto_string.size()); - - _fixed_file_header.checksum = 0; - _fixed_file_header.protobuf_length = _proto_string.size(); - _fixed_file_header.file_length = size(); - _fixed_file_header.version = OLAP_DATA_VERSION_APPLIED; - _fixed_file_header.magic_number = OLAP_FIX_HEADER_MAGIC_NUMBER; - - return Status::OK(); -} - -template -Status FileHeader::serialize( - FileHandlerType* file_handler) { - if (nullptr == file_handler) { - return Status::Error(); - } - - // write to file - if (!file_handler->pwrite(&_fixed_file_header, _fixed_file_header_size, 0)) { - char errmsg[64]; - LOG(WARNING) << "fail to write fixed header to file. [file='" << file_handler->file_name() - << "' err=" << strerror_r(errno, errmsg, 64) << "]"; - return Status::Error(); - } - - if (!file_handler->pwrite(&_extra_fixed_header, sizeof(_extra_fixed_header), - _fixed_file_header_size)) { - char errmsg[64]; - LOG(WARNING) << "fail to write extra fixed header to file. [file='" - << file_handler->file_name() << "' err=" << strerror_r(errno, errmsg, 64) - << "]"; - return Status::Error(); - } - - if (!file_handler->pwrite(_proto_string.c_str(), _proto_string.size(), - _fixed_file_header_size + sizeof(_extra_fixed_header))) { - char errmsg[64]; - LOG(WARNING) << "fail to write proto header to file. [file='" << file_handler->file_name() - << "' err='" << strerror_r(errno, errmsg, 64) << "']"; - return Status::Error(); - } - - return Status::OK(); -} - -template -Status FileHeader::unserialize( - FileHandlerType* file_handler) { - if (nullptr == file_handler) { - return Status::Error(); - } - - off_t real_file_length = 0; - uint32_t real_protobuf_checksum = 0; - - if (!file_handler->pread(&_fixed_file_header, _fixed_file_header_size, 0)) { - char errmsg[64]; - LOG(WARNING) << "fail to load header structure from file. file=" - << file_handler->file_name() << ", error=" << strerror_r(errno, errmsg, 64); - return Status::Error(); - } - - if (_fixed_file_header.magic_number != OLAP_FIX_HEADER_MAGIC_NUMBER) { - VLOG_TRACE << "old fix header found, magic num=" << _fixed_file_header.magic_number; - FixedFileHeader tmp_header; - - if (!file_handler->pread(&tmp_header, sizeof(tmp_header), 0)) { - char errmsg[64]; - LOG(WARNING) << "fail to load header structure from file. file=" - << file_handler->file_name() - << ", error=" << strerror_r(errno, errmsg, 64); - return Status::Error(); - } - - _fixed_file_header.file_length = tmp_header.file_length; - _fixed_file_header.checksum = tmp_header.checksum; - _fixed_file_header.protobuf_length = tmp_header.protobuf_length; - _fixed_file_header.protobuf_checksum = tmp_header.protobuf_checksum; - _fixed_file_header.magic_number = OLAP_FIX_HEADER_MAGIC_NUMBER; - _fixed_file_header.version = OLAP_DATA_VERSION_APPLIED; - - _fixed_file_header_size = sizeof(tmp_header); - } - - VLOG_NOTICE << "fix head loaded. file_length=" << _fixed_file_header.file_length - << ", checksum=" << _fixed_file_header.checksum - << ", protobuf_length=" << _fixed_file_header.protobuf_length - << ", magic_number=" << _fixed_file_header.magic_number - << ", version=" << _fixed_file_header.version; - - if (!file_handler->pread(&_extra_fixed_header, sizeof(_extra_fixed_header), - _fixed_file_header_size)) { - char errmsg[64]; - LOG(WARNING) << "fail to load extra fixed header from file. file=" - << file_handler->file_name() << ", error=" << strerror_r(errno, errmsg, 64); - return Status::Error(); - } - - std::unique_ptr buf(new (std::nothrow) char[_fixed_file_header.protobuf_length]); - - if (nullptr == buf.get()) { - char errmsg[64]; - LOG(WARNING) << "malloc protobuf buf error. file=" << file_handler->file_name() - << ", error=" << strerror_r(errno, errmsg, 64); - return Status::Error(); - } - - if (!file_handler->pread(buf.get(), _fixed_file_header.protobuf_length, - _fixed_file_header_size + sizeof(_extra_fixed_header))) { - char errmsg[64]; - LOG(WARNING) << "fail to load protobuf from file. file=" << file_handler->file_name() - << ", error=" << strerror_r(errno, errmsg, 64); - return Status::Error(); - } - - real_file_length = file_handler->length(); - - if (file_length() != static_cast(real_file_length)) { - LOG(WARNING) << "file length is not match. file=" << file_handler->file_name() - << ", file_length=" << file_length() - << ", real_file_length=" << real_file_length; - return Status::Error(); - } - - // check proto checksum - real_protobuf_checksum = - olap_adler32(olap_adler32_init(), buf.get(), _fixed_file_header.protobuf_length); - - if (real_protobuf_checksum != _fixed_file_header.protobuf_checksum) { - LOG(WARNING) << "checksum is not match. file=" << file_handler->file_name() - << ", expect=" << _fixed_file_header.protobuf_checksum - << ", actual=" << real_protobuf_checksum; - return Status::Error(); - } - - try { - std::string protobuf_str(buf.get(), _fixed_file_header.protobuf_length); - - if (!_proto.ParseFromString(protobuf_str)) { - LOG(WARNING) << "fail to parse file content to protobuf object. file=" - << file_handler->file_name(); - return Status::Error(); - } - } catch (...) { - LOG(WARNING) << "fail to load protobuf. file='" << file_handler->file_name(); - return Status::Error(); - } - - return Status::OK(); -} - -template -Status FileHeader::validate(const std::string& filename) { - FileHandler file_handler; - Status res = Status::OK(); - - if (!file_handler.open(filename.c_str(), O_RDONLY)) { - char errmsg[64]; - LOG(WARNING) << "fail to open file. [file='" << filename - << "' err=" << strerror_r(errno, errmsg, 64) << "]"; - return Status::Error(); - } - - if (!(res = unserialize(&file_handler))) { - LOG(WARNING) << "unserialize file header error. [file='" << filename << "']"; - return res; - } - - return Status::OK(); -} - -} // namespace doris diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h index 62422f1410..adb1a5feb0 100644 --- a/be/src/olap/iterators.h +++ b/be/src/olap/iterators.h @@ -20,6 +20,7 @@ #include #include "common/status.h" +#include "io/io_common.h" #include "olap/block_column_predicate.h" #include "olap/column_predicate.h" #include "olap/olap_common.h" @@ -34,25 +35,6 @@ class RowCursor; class Schema; class ColumnPredicate; -struct IOContext { - IOContext() = default; - - IOContext(const TUniqueId* query_id_, FileCacheStatistics* stats_, bool is_presistent_, - bool use_disposable_cache_, bool read_segment_index_, bool enable_file_cache) - : query_id(query_id_), - is_persistent(is_presistent_), - use_disposable_cache(use_disposable_cache_), - read_segment_index(read_segment_index_), - file_cache_stats(stats_), - enable_file_cache(enable_file_cache) {} - ReaderType reader_type; - const TUniqueId* query_id = nullptr; - bool is_persistent = false; - bool use_disposable_cache = false; - bool read_segment_index = false; - FileCacheStatistics* file_cache_stats = nullptr; - bool enable_file_cache = true; -}; namespace vectorized { struct IteratorRowRef; }; @@ -117,7 +99,7 @@ public: bool read_orderby_key_reverse = false; // columns for orderby keys std::vector* read_orderby_key_columns = nullptr; - IOContext io_ctx; + io::IOContext io_ctx; vectorized::VExpr* remaining_vconjunct_root = nullptr; vectorized::VExprContext* common_vexpr_ctxs_pushdown = nullptr; const std::set* output_columns = nullptr; diff --git a/be/src/olap/merger.h b/be/src/olap/merger.h index a3d9b54e1c..467c126304 100644 --- a/be/src/olap/merger.h +++ b/be/src/olap/merger.h @@ -17,6 +17,7 @@ #pragma once +#include "io/io_common.h" #include "olap/olap_define.h" #include "olap/rowid_conversion.h" #include "olap/rowset/rowset_writer.h" diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 5dfbb96a8b..e45761fe58 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -33,6 +33,7 @@ #include "env/env.h" #include "gen_cpp/Types_types.h" +#include "io/io_common.h" #include "olap/olap_define.h" #include "util/hash_util.hpp" #include "util/uid_util.h" @@ -169,16 +170,6 @@ enum PushType { PUSH_NORMAL_V2 = 4, // for spark load }; -enum ReaderType { - READER_QUERY = 0, - READER_ALTER_TABLE = 1, - READER_BASE_COMPACTION = 2, - READER_CUMULATIVE_COMPACTION = 3, - READER_CHECKSUM = 4, - READER_COLD_DATA_COMPACTION = 5, - READER_SEGMENT_COMPACTION = 6, -}; - constexpr bool field_is_slice_type(const FieldType& field_type) { return field_type == OLAP_FIELD_TYPE_VARCHAR || field_type == OLAP_FIELD_TYPE_CHAR || field_type == OLAP_FIELD_TYPE_STRING; @@ -264,17 +255,6 @@ using KeyRange = std::pair; static const int GENERAL_DEBUG_COUNT = 0; -struct FileCacheStatistics { - int64_t num_io_total = 0; - int64_t num_io_hit_cache = 0; - int64_t num_io_bytes_read_total = 0; - int64_t num_io_bytes_read_from_file_cache = 0; - int64_t num_io_bytes_read_from_write_cache = 0; - int64_t num_io_written_in_file_cache = 0; - int64_t num_io_bytes_written_in_file_cache = 0; - int64_t num_io_bytes_skip_cache = 0; -}; - // ReaderStatistics used to collect statistics when scan data from storage struct OlapReaderStatistics { int64_t io_ns = 0; @@ -373,7 +353,7 @@ struct OlapReaderStatistics { // SCOPED_RAW_TIMER(&_stats->general_debug_ns[1]); int64_t general_debug_ns[GENERAL_DEBUG_COUNT] = {}; - FileCacheStatistics file_cache_stats; + io::FileCacheStatistics file_cache_stats; int64_t load_segments_timer = 0; }; diff --git a/be/src/olap/push_handler.h b/be/src/olap/push_handler.h index b852e68bb2..8546fbf749 100644 --- a/be/src/olap/push_handler.h +++ b/be/src/olap/push_handler.h @@ -25,7 +25,6 @@ #include "gen_cpp/AgentService_types.h" #include "gen_cpp/MasterService_types.h" #include "gen_cpp/PaloInternalService_types.h" -#include "olap/file_helper.h" #include "olap/merger.h" #include "olap/olap_common.h" #include "olap/row_cursor.h" diff --git a/be/src/olap/reader.h b/be/src/olap/reader.h index d286bf42a6..cfee9ee10a 100644 --- a/be/src/olap/reader.h +++ b/be/src/olap/reader.h @@ -22,6 +22,7 @@ #include "exprs/bitmapfilter_predicate.h" #include "exprs/function_filter.h" #include "exprs/hybrid_set.h" +#include "io/io_common.h" #include "olap/delete_handler.h" #include "olap/row_cursor.h" #include "olap/rowset/rowset_reader.h" diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 4f459d8365..a8cfb8e022 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -217,6 +217,10 @@ Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id, if (!fs) { return Status::Error(); } + if (fs->type() != io::FileSystemType::LOCAL) { + return Status::InternalError("should be local file system"); + } + io::LocalFileSystem* local_fs = (io::LocalFileSystem*)fs.get(); for (int i = 0; i < num_segments(); ++i) { auto dst_path = segment_file_path(dir, new_rowset_id, i + new_rowset_start_seg_id); // TODO(lingbin): use Env API? or EnvUtil? @@ -228,7 +232,7 @@ Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id, auto src_path = segment_file_path(i); // TODO(lingbin): how external storage support link? // use copy? or keep refcount to avoid being delete? - if (!fs->link_file(src_path, dst_path).ok()) { + if (!local_fs->link_file(src_path, dst_path).ok()) { LOG(WARNING) << "fail to create hard link. from=" << src_path << ", " << "to=" << dst_path << ", errno=" << Errno::no(); return Status::Error(); @@ -244,7 +248,7 @@ Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id, InvertedIndexDescriptor::get_index_file_name(dst_path, index_meta->index_id()); - if (!fs->link_file(inverted_index_src_file_path, inverted_index_dst_file_path) + if (!local_fs->link_file(inverted_index_src_file_path, inverted_index_dst_file_path) .ok()) { LOG(WARNING) << "fail to create hard link. from=" << inverted_index_src_file_path << ", " diff --git a/be/src/olap/rowset/rowset_reader_context.h b/be/src/olap/rowset/rowset_reader_context.h index 7671a75d43..825c6dbcd1 100644 --- a/be/src/olap/rowset/rowset_reader_context.h +++ b/be/src/olap/rowset/rowset_reader_context.h @@ -18,6 +18,7 @@ #ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H #define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_READER_CONTEXT_H +#include "io/io_common.h" #include "olap/column_predicate.h" #include "olap/olap_common.h" #include "runtime/runtime_state.h" diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index ac064de0a3..62601a1e6e 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -26,6 +26,7 @@ #include "common/status.h" // for Status #include "gen_cpp/segment_v2.pb.h" // for ColumnMetaPB #include "io/fs/file_reader.h" +#include "io/io_common.h" #include "olap/block_column_predicate.h" #include "olap/column_predicate.h" #include "olap/iterators.h" @@ -76,7 +77,7 @@ struct ColumnIteratorOptions { // page types are divided into DATA_PAGE & INDEX_PAGE // INDEX_PAGE including index_page, dict_page and short_key_page PageTypePB type; - IOContext io_ctx; + io::IOContext io_ctx; void sanity_check() const { CHECK_NOTNULL(file_reader); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp index 4f966bf44d..26b509ff7f 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp @@ -19,6 +19,7 @@ #include "CLucene/SharedHeader.h" #include "CLucene/StdHeader.h" +#include "io/fs/file_writer.h" #include "olap/iterators.h" #include "util/md5.h" @@ -237,7 +238,7 @@ bool DorisCompoundDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, co } SharedHandle* h = _CLNEW SharedHandle(path); - if (!fs->open_file(path, &h->_reader, nullptr).ok()) { + if (!fs->open_file(path, &h->_reader).ok()) { error.set(CL_ERR_IO, "open file error"); } @@ -335,8 +336,7 @@ void DorisCompoundDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_ Slice result {b, (size_t)len}; size_t bytes_read = 0; - IOContext io_ctx; - if (!_handle->_reader->read_at(_pos, result, io_ctx, &bytes_read).ok()) { + if (!_handle->_reader->read_at(_pos, result, &bytes_read).ok()) { _CLTHROWA(CL_ERR_IO, "read past EOF"); } bufferLength = len; @@ -512,10 +512,11 @@ bool DorisCompoundDirectory::list(std::vector* names) const { CND_PRECONDITION(!directory.empty(), "directory is not open"); char fl[CL_MAX_DIR]; priv_getFN(fl, ""); - std::vector paths; - RETURN_IF_ERROR(fs->list(fl, &paths)); - for (auto path : paths) { - names->push_back(path.string()); + std::vector files; + bool exists; + RETURN_IF_ERROR(fs->list(fl, true, &files, &exists)); + for (auto& file : files) { + names->push_back(file.file_name); } return true; } diff --git a/be/src/olap/rowset/segment_v2/page_io.cpp b/be/src/olap/rowset/segment_v2/page_io.cpp index 697775dcdd..7eb3dd0549 100644 --- a/be/src/olap/rowset/segment_v2/page_io.cpp +++ b/be/src/olap/rowset/segment_v2/page_io.cpp @@ -137,8 +137,8 @@ Status PageIO::read_and_decompress_page(const PageReadOptions& opts, PageHandle* { SCOPED_RAW_TIMER(&opts.stats->io_ns); size_t bytes_read = 0; - RETURN_IF_ERROR(opts.file_reader->read_at(opts.page_pointer.offset, page_slice, opts.io_ctx, - &bytes_read)); + RETURN_IF_ERROR(opts.file_reader->read_at(opts.page_pointer.offset, page_slice, &bytes_read, + &opts.io_ctx)); DCHECK_EQ(bytes_read, page_size); opts.stats->compressed_bytes_read += page_size; } diff --git a/be/src/olap/rowset/segment_v2/page_io.h b/be/src/olap/rowset/segment_v2/page_io.h index 14f93021bf..06e345c27a 100644 --- a/be/src/olap/rowset/segment_v2/page_io.h +++ b/be/src/olap/rowset/segment_v2/page_io.h @@ -71,7 +71,7 @@ struct PageReadOptions { // index_page should not be pre-decoded bool pre_decode = true; - IOContext io_ctx; + io::IOContext io_ctx; void sanity_check() const { CHECK_NOTNULL(file_reader); diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index ea0e6b44cd..f59b716361 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -50,14 +50,14 @@ Status Segment::open(io::FileSystemSPtr fs, const std::string& path, uint32_t se std::shared_ptr* output) { io::FileReaderSPtr file_reader; #ifndef BE_TEST - RETURN_IF_ERROR(fs->open_file(path, reader_options, &file_reader, nullptr)); + RETURN_IF_ERROR(fs->open_file(path, reader_options, &file_reader)); #else // be ut use local file reader instead of remote file reader while use remote cache if (!config::file_cache_type.empty()) { - RETURN_IF_ERROR(io::global_local_filesystem()->open_file(path, reader_options, &file_reader, - nullptr)); + RETURN_IF_ERROR( + io::global_local_filesystem()->open_file(path, reader_options, &file_reader)); } else { - RETURN_IF_ERROR(fs->open_file(path, reader_options, &file_reader, nullptr)); + RETURN_IF_ERROR(fs->open_file(path, reader_options, &file_reader)); } #endif @@ -148,9 +148,7 @@ Status Segment::_parse_footer() { uint8_t fixed_buf[12]; size_t bytes_read = 0; - IOContext io_ctx; - RETURN_IF_ERROR( - _file_reader->read_at(file_size - 12, Slice(fixed_buf, 12), io_ctx, &bytes_read)); + RETURN_IF_ERROR(_file_reader->read_at(file_size - 12, Slice(fixed_buf, 12), &bytes_read)); DCHECK_EQ(bytes_read, 12); // validate magic number @@ -170,8 +168,7 @@ Status Segment::_parse_footer() { std::string footer_buf; footer_buf.resize(footer_length); - RETURN_IF_ERROR( - _file_reader->read_at(file_size - 12 - footer_length, footer_buf, io_ctx, &bytes_read)); + RETURN_IF_ERROR(_file_reader->read_at(file_size - 12 - footer_length, footer_buf, &bytes_read)); DCHECK_EQ(bytes_read, footer_length); // validate footer PB's checksum diff --git a/be/src/olap/storage_policy.h b/be/src/olap/storage_policy.h index 02d86603fa..04f05540ae 100644 --- a/be/src/olap/storage_policy.h +++ b/be/src/olap/storage_policy.h @@ -52,7 +52,7 @@ void delete_storage_policy(int64_t id); std::vector> get_storage_policy_ids(); struct StorageResource { - io::FileSystemSPtr fs; + io::RemoteFileSystemSPtr fs; int64_t version = -1; }; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index f32e0b077b..d30b5b0ba4 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1819,13 +1819,12 @@ Status Tablet::_read_cooldown_meta(const std::shared_ptr& TabletMetaPB* tablet_meta_pb) { std::string remote_meta_path = remote_tablet_meta_path(tablet_id(), _cooldown_replica_id, _cooldown_term); - IOContext io_ctx; io::FileReaderSPtr tablet_meta_reader; - RETURN_IF_ERROR(fs->open_file(remote_meta_path, &tablet_meta_reader, &io_ctx)); + RETURN_IF_ERROR(fs->open_file(remote_meta_path, &tablet_meta_reader)); auto file_size = tablet_meta_reader->size(); size_t bytes_read; auto buf = std::unique_ptr(new uint8_t[file_size]); - RETURN_IF_ERROR(tablet_meta_reader->read_at(0, {buf.get(), file_size}, io_ctx, &bytes_read)); + RETURN_IF_ERROR(tablet_meta_reader->read_at(0, {buf.get(), file_size}, &bytes_read)); tablet_meta_reader->close(); if (!tablet_meta_pb->ParseFromArray(buf.get(), file_size)) { return Status::InternalError("malformed tablet meta"); @@ -2131,7 +2130,7 @@ void Tablet::remove_unused_remote_files() { req.__isset.confirm_list = true; // tablet_id -> [fs, unused_remote_files] using unused_remote_files_buffer_t = std::unordered_map< - int64_t, std::pair, std::vector>>; + int64_t, std::pair, std::vector>>; unused_remote_files_buffer_t buffer; int64_t num_files_in_buffer = 0; // assume a filename is 0.1KB, buffer size should not larger than 100MB @@ -2161,15 +2160,17 @@ void Tablet::remove_unused_remote_files() { return; } - std::vector files; + std::vector files; // FIXME(plat1ko): What if user reset resource in storage policy to another resource? // Maybe we should also list files in previously uploaded resources. - st = dest_fs->list(remote_tablet_path(t->tablet_id()), &files); + bool exists = true; + st = dest_fs->list(io::Path(remote_tablet_path(t->tablet_id())), true, &files, &exists); if (!st.ok()) { LOG(WARNING) << "encounter error when remove unused remote files, tablet_id=" << t->tablet_id() << " : " << st; + return; } - if (files.empty()) { + if (!exists || files.empty()) { return; } // get all cooldowned rowsets @@ -2197,8 +2198,8 @@ void Tablet::remove_unused_remote_files() { fmt::format("{}.{}.meta", cooldown_replica_id, cooldown_term); // filter out the paths that should be reserved // clang-format off - files.erase(std::remove_if(files.begin(), files.end(), [&](io::Path& path) { - const std::string& path_str = path.native(); + files.erase(std::remove_if(files.begin(), files.end(), [&](io::FileInfo& info) { + const std::string& path_str = info.file_name; if (StringPiece(path_str).ends_with(".meta")) { return path_str == remote_meta_path; } @@ -2251,10 +2252,14 @@ void Tablet::remove_unused_remote_files() { << " tablet_id=" << id; io::Path dir("data/" + std::to_string(id)); for (auto& file : files) { - file = dir / file; - LOG(INFO) << "delete unused file: " << file.native(); + auto delete_path = dir / io::Path(file.file_name); + LOG(INFO) << "delete unused file: " << delete_path.native(); } - st = fs->batch_delete(files); + std::vector file_names; + for (auto& info : files) { + file_names.emplace_back(info.file_name); + } + st = fs->batch_delete(file_names); if (!st.ok()) { LOG(WARNING) << "failed to delete unused files, tablet_id=" << id << " : " << st; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 5bc7019fda..e501e17f39 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -730,7 +730,7 @@ Status TabletManager::load_tablet_from_meta(DataDir* data_dir, TTabletId tablet_ if (!status.ok()) { LOG(WARNING) << "fail to load tablet because can not parse meta_binary string. " << "tablet_id=" << tablet_id << ", schema_hash=" << schema_hash - << ", path=" << data_dir->path(); + << ", path=" << data_dir->path() << ", status: " << status; return Status::Error(); } tablet_meta->init_rs_metas_fs(data_dir->fs()); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 4cea4baa4f..7a6ec8c765 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -20,7 +20,8 @@ #include #include "common/consts.h" -#include "olap/file_helper.h" +#include "io/fs/file_writer.h" +#include "olap/file_header.h" #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/tablet_meta_manager.h" @@ -321,22 +322,9 @@ void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tco } Status TabletMeta::create_from_file(const string& file_path) { - FileHeader file_header; - FileHandler file_handler; - - auto open_status = file_handler.open(file_path, O_RDONLY); - - if (!open_status.ok()) { - LOG(WARNING) << "fail to open ordinal file. file=" << file_path; - return open_status; - } - - // In file_header.unserialize(), it validates file length, signature, checksum of protobuf. - if (file_header.unserialize(&file_handler) != Status::OK()) { - LOG(WARNING) << "fail to unserialize tablet_meta. file='" << file_path; - return Status::Error(); - } - + FileHeader file_header(file_path); + // In file_header.deserialize(), it validates file length, signature, checksum of protobuf. + RETURN_IF_ERROR(file_header.deserialize()); TabletMetaPB tablet_meta_pb; try { tablet_meta_pb.CopyFrom(file_header.message()); @@ -398,30 +386,15 @@ Status TabletMeta::save(const string& file_path) { Status TabletMeta::save(const string& file_path, const TabletMetaPB& tablet_meta_pb) { DCHECK(!file_path.empty()); - - FileHeader file_header; - FileHandler file_handler; - - auto open_status = - file_handler.open_with_mode(file_path, O_CREAT | O_WRONLY | O_TRUNC, S_IRUSR | S_IWUSR); - if (!open_status.ok()) { - LOG(WARNING) << "fail to open header file. file='" << file_path; - return open_status; - } - + FileHeader file_header(file_path); try { file_header.mutable_message()->CopyFrom(tablet_meta_pb); } catch (...) { LOG(WARNING) << "fail to copy protocol buffer object. file='" << file_path; return Status::Error(); } - - if (file_header.prepare(&file_handler) != Status::OK() || - file_header.serialize(&file_handler) != Status::OK()) { - LOG(WARNING) << "fail to serialize to file header. file='" << file_path; - return Status::Error(); - } - + RETURN_IF_ERROR(file_header.prepare()); + RETURN_IF_ERROR(file_header.serialize()); return Status::OK(); } diff --git a/be/src/olap/utils.cpp b/be/src/olap/utils.cpp index 0ea8c6450d..755f73c08e 100644 --- a/be/src/olap/utils.cpp +++ b/be/src/olap/utils.cpp @@ -33,7 +33,6 @@ #include #include -#include "olap/file_helper.h" #include "util/file_utils.h" #ifdef DORIS_WITH_LZO @@ -47,6 +46,9 @@ #include "common/status.h" #include "env/env.h" #include "gutil/strings/substitute.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" #include "olap/olap_common.h" #include "olap/olap_define.h" #include "util/errno.h" @@ -449,13 +451,7 @@ Status read_write_test_file(const string& test_file_path) { return Status::Error(); } } - Status res = Status::OK(); - FileHandler file_handler; - if ((res = file_handler.open_with_mode(test_file_path.c_str(), O_RDWR | O_CREAT | O_SYNC, - S_IRUSR | S_IWUSR)) != Status::OK()) { - LOG(WARNING) << "fail to create test file. path=" << test_file_path; - return res; - } + const size_t TEST_FILE_BUF_SIZE = 4096; const size_t DIRECT_IO_ALIGNMENT = 512; char* write_test_buff = nullptr; @@ -476,30 +472,24 @@ Status read_write_test_file(const string& test_file_path) { int32_t tmp_value = rand_r(&rand_seed); write_test_buff[i] = static_cast(tmp_value); } - if (!(res = file_handler.pwrite(write_buff.get(), TEST_FILE_BUF_SIZE, SEEK_SET))) { - LOG(WARNING) << "fail to write test file. [file_name=" << test_file_path << "]"; - return res; - } - if ((res = file_handler.pread(read_buff.get(), TEST_FILE_BUF_SIZE, SEEK_SET)) != Status::OK()) { - LOG(WARNING) << "fail to read test file. [file_name=" << test_file_path << "]"; - return res; - } + + // write file + io::FileWriterPtr file_writer; + RETURN_IF_ERROR(io::global_local_filesystem()->create_file(test_file_path, &file_writer)); + RETURN_IF_ERROR(file_writer->append({write_buff.get(), TEST_FILE_BUF_SIZE})); + RETURN_IF_ERROR(file_writer->close()); + // read file + io::FileReaderSPtr file_reader; + RETURN_IF_ERROR(io::global_local_filesystem()->open_file(test_file_path, &file_reader)); + size_t bytes_read = 0; + RETURN_IF_ERROR(file_reader->read_at(0, {read_buff.get(), TEST_FILE_BUF_SIZE}, &bytes_read)); if (memcmp(write_buff.get(), read_buff.get(), TEST_FILE_BUF_SIZE) != 0) { LOG(WARNING) << "the test file write_buf and read_buf not equal, [file_name = " << test_file_path << "]"; return Status::Error(); } - if ((res = file_handler.close()) != Status::OK()) { - LOG(WARNING) << "fail to close test file. [file_name=" << test_file_path << "]"; - return res; - } - if (remove(test_file_path.c_str()) != 0) { - char errmsg[64]; - VLOG_NOTICE << "fail to delete test file. [err='" << strerror_r(errno, errmsg, 64) - << "' path='" << test_file_path << "']"; - return Status::Error(); - } - return res; + // delete file + return io::global_local_filesystem()->delete_file(test_file_path); } bool check_datapath_rw(const string& path) { diff --git a/be/src/runtime/routine_load/routine_load_task_executor.cpp b/be/src/runtime/routine_load/routine_load_task_executor.cpp index ae0422e675..3aa361d157 100644 --- a/be/src/runtime/routine_load/routine_load_task_executor.cpp +++ b/be/src/runtime/routine_load/routine_load_task_executor.cpp @@ -371,8 +371,7 @@ Status RoutineLoadTaskExecutor::_execute_plan_for_test(std::shared_ptrread_at(0, result, io_ctx, &read_bytes); + Status st = pipe->read_at(0, result, &read_bytes); if (!st.ok()) { LOG(WARNING) << "read failed"; ctx->promise.set_value(st); diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index c4b2a390df..846ce688e2 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -26,16 +26,17 @@ #include "gen_cpp/HeartbeatService_types.h" #include "gen_cpp/PaloBrokerService_types.h" #include "gen_cpp/TPaloBrokerService.h" -#include "olap/file_helper.h" +#include "io/fs/broker_file_system.h" +#include "io/fs/hdfs_file_system.h" +#include "io/fs/s3_file_system.h" +#include "io/hdfs_builder.h" #include "olap/snapshot_manager.h" #include "olap/storage_engine.h" #include "olap/tablet.h" #include "runtime/broker_mgr.h" #include "runtime/exec_env.h" -#include "util/broker_storage_backend.h" #include "util/file_utils.h" -#include "util/hdfs_storage_backend.h" -#include "util/s3_storage_backend.h" +#include "util/s3_uri.h" #include "util/thrift_rpc_helper.h" namespace doris { @@ -46,29 +47,42 @@ SnapshotLoader::SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id) _task_id(task_id), _broker_addr(TNetworkAddress()), _prop(std::map()), - _storage_backend(nullptr) {} + _remote_fs(nullptr) {} SnapshotLoader::SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id, const TNetworkAddress& broker_addr, - const std::map& prop, - TStorageBackendType::type type) - : _env(env), _job_id(job_id), _task_id(task_id), _broker_addr(broker_addr), _prop(prop) { + const std::map& prop) + : _env(env), _job_id(job_id), _task_id(task_id), _broker_addr(broker_addr), _prop(prop) {} + +Status SnapshotLoader::init(TStorageBackendType::type type, const std::string& location) { if (TStorageBackendType::type::S3 == type) { - _storage_backend.reset(new S3StorageBackend(_prop)); + S3Conf s3_conf; + S3URI s3_uri(location); + RETURN_IF_ERROR(s3_uri.parse()); + RETURN_IF_ERROR(S3ClientFactory::convert_properties_to_s3_conf(_prop, s3_uri, &s3_conf)); + std::shared_ptr fs; + RETURN_IF_ERROR(io::S3FileSystem::create(std::move(s3_conf), "", &fs)); + _remote_fs = std::move(fs); } else if (TStorageBackendType::type::HDFS == type) { - _storage_backend.reset(new HDFSStorageBackend(_prop)); + THdfsParams hdfs_params = parse_properties(_prop); + std::shared_ptr fs; + RETURN_IF_ERROR(io::HdfsFileSystem::create(hdfs_params, "", &fs)); + _remote_fs = std::move(fs); } else if (TStorageBackendType::type::BROKER == type) { - _storage_backend.reset(new BrokerStorageBackend(_env, _broker_addr, _prop)); + std::shared_ptr fs; + RETURN_IF_ERROR(io::BrokerFileSystem::create(_broker_addr, _prop, 0, &fs)); + _remote_fs = std::move(fs); } else { - _storage_backend = nullptr; + return Status::InternalError("Unknown storage tpye: {}", type); } + return Status::OK(); } SnapshotLoader::~SnapshotLoader() = default; Status SnapshotLoader::upload(const std::map& src_to_dest_path, std::map>* tablet_files) { - if (!_storage_backend) { + if (!_remote_fs) { return Status::InternalError("Storage backend not initialized."); } LOG(INFO) << "begin to upload snapshot files. num: " << src_to_dest_path.size() @@ -99,7 +113,7 @@ Status SnapshotLoader::upload(const std::map& src_to_d // 2.1 get existing files from remote path std::map remote_files; - RETURN_IF_ERROR(_storage_backend->list(dest_path, true, false, &remote_files)); + RETURN_IF_ERROR(_list_with_checksum(dest_path, &remote_files)); for (auto& tmp : remote_files) { VLOG_CRITICAL << "get remote file: " << tmp.first << ", checksum: " << tmp.second.md5; @@ -151,8 +165,8 @@ Status SnapshotLoader::upload(const std::map& src_to_d // upload std::string full_remote_file = dest_path + "/" + local_file; std::string full_local_file = src_path + "/" + local_file; - RETURN_IF_ERROR(_storage_backend->upload_with_checksum(full_local_file, - full_remote_file, md5sum)); + RETURN_IF_ERROR( + _remote_fs->upload_with_checksum(full_local_file, full_remote_file, md5sum)); } // end for each tablet's local files tablet_files->emplace(tablet_id, local_files_with_checksum); @@ -172,7 +186,7 @@ Status SnapshotLoader::upload(const std::map& src_to_d */ Status SnapshotLoader::download(const std::map& src_to_dest_path, std::vector* downloaded_tablet_ids) { - if (!_storage_backend) { + if (!_remote_fs) { return Status::InternalError("Storage backend not initialized."); } LOG(INFO) << "begin to download snapshot files. num: " << src_to_dest_path.size() @@ -213,7 +227,7 @@ Status SnapshotLoader::download(const std::map& src_to // 2.2. get remote files std::map remote_files; - RETURN_IF_ERROR(_storage_backend->list(remote_path, true, false, &remote_files)); + RETURN_IF_ERROR(_list_with_checksum(remote_path, &remote_files)); if (remote_files.empty()) { std::stringstream ss; ss << "get nothing from remote path: " << remote_path; @@ -287,7 +301,7 @@ Status SnapshotLoader::download(const std::map& src_to // remove file which will be downloaded now. // this file will be added to local_files if it be downloaded successfully. local_files.erase(find); - RETURN_IF_ERROR(_storage_backend->download(full_remote_file, full_local_file)); + RETURN_IF_ERROR(_remote_fs->download(full_remote_file, full_local_file)); // 3. check md5 of the downloaded file std::string downloaded_md5sum; @@ -628,4 +642,25 @@ Status SnapshotLoader::_report_every(int report_threshold, int* counter, int32_t return Status::OK(); } +Status SnapshotLoader::_list_with_checksum(const std::string& dir, + std::map* md5_files) { + bool exists = true; + std::vector files; + RETURN_IF_ERROR(_remote_fs->list(dir, true, &files, &exists)); + for (auto& tmp_file : files) { + io::Path path(tmp_file.file_name); + std::string file_name = path.filename(); + size_t pos = file_name.find_last_of("."); + if (pos == std::string::npos || pos == file_name.size() - 1) { + // Not found checksum separator, ignore this file + continue; + } + FileStat stat = {std::string(file_name, 0, pos), std::string(file_name, pos + 1), + tmp_file.file_size}; + md5_files->emplace(std::string(file_name, 0, pos), stat); + } + + return Status::OK(); +} + } // end namespace doris diff --git a/be/src/runtime/snapshot_loader.h b/be/src/runtime/snapshot_loader.h index 30eaa9f9f3..2451c07518 100644 --- a/be/src/runtime/snapshot_loader.h +++ b/be/src/runtime/snapshot_loader.h @@ -25,14 +25,19 @@ #include "common/status.h" #include "gen_cpp/Types_types.h" +#include "io/fs/remote_file_system.h" #include "olap/tablet.h" #include "runtime/client_cache.h" namespace doris { +struct FileStat { + std::string name; + std::string md5; + size_t size; +}; class ExecEnv; class StorageBackend; -struct FileStat; /* * Upload: @@ -59,11 +64,12 @@ public: SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id); SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id, const TNetworkAddress& broker_addr, - const std::map& broker_prop, - TStorageBackendType::type type); + const std::map& broker_prop); ~SnapshotLoader(); + Status init(TStorageBackendType::type type, const std::string& location); + Status upload(const std::map& src_to_dest_path, std::map>* tablet_files); @@ -92,13 +98,15 @@ private: Status _report_every(int report_threshold, int* counter, int finished_num, int total_num, TTaskType::type type); + Status _list_with_checksum(const std::string& dir, std::map* md5_files); + private: ExecEnv* _env; int64_t _job_id; int64_t _task_id; const TNetworkAddress _broker_addr; const std::map _prop; - std::unique_ptr _storage_backend; + std::shared_ptr _remote_fs; }; } // end namespace doris diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index 257509a97c..894c07e6eb 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -463,8 +463,8 @@ void PInternalServiceImpl::fetch_table_schema(google::protobuf::RpcController* c std::unique_ptr reader(nullptr); std::unique_ptr profile(new RuntimeProfile("FetchTableSchema")); - IOContext io_ctx; - FileCacheStatistics file_cache_statis; + io::IOContext io_ctx; + io::FileCacheStatistics file_cache_statis; io_ctx.file_cache_stats = &file_cache_statis; switch (params.format_type) { case TFileFormatType::FORMAT_CSV_PLAIN: diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt index 23e14e3b4c..b6d5f32e1e 100644 --- a/be/src/util/CMakeLists.txt +++ b/be/src/util/CMakeLists.txt @@ -50,7 +50,6 @@ set(UTIL_FILES progress_updater.cpp runtime_profile.cpp static_asserts.cpp - # storage_backend_mgr.cpp string_parser.cpp thrift_util.cpp thrift_client.cpp @@ -63,7 +62,6 @@ set(UTIL_FILES mysql_row_buffer.cpp error_util.cc filesystem_util.cc - broker_storage_backend.cpp time.cpp os_info.cpp os_util.cpp @@ -94,9 +92,7 @@ set(UTIL_FILES zlib.cpp pprof_utils.cpp s3_uri.cpp - s3_storage_backend.cpp s3_util.cpp - hdfs_storage_backend.cpp hdfs_util.cpp time_lut.cpp cityhash102/city.cc diff --git a/be/src/util/async_io.h b/be/src/util/async_io.h index 6d324e4f2a..fce50e79b7 100644 --- a/be/src/util/async_io.h +++ b/be/src/util/async_io.h @@ -81,10 +81,10 @@ public: cv.notify_one(); }; - if (file_type == io::FileSystemType::S3) { - AsyncIO::instance().remote_thread_pool()->offer(task); - } else { + if (file_type == io::FileSystemType::LOCAL) { AsyncIO::instance().io_thread_pool()->offer(task); + } else { + AsyncIO::instance().remote_thread_pool()->offer(task); } cv.wait(l); } @@ -102,4 +102,4 @@ private: PriorityThreadPool* remote_thread_pool() { return _remote_thread_pool; } }; -} // end namespace doris \ No newline at end of file +} // end namespace doris diff --git a/be/src/util/broker_storage_backend.cpp b/be/src/util/broker_storage_backend.cpp deleted file mode 100644 index af2d5a7192..0000000000 --- a/be/src/util/broker_storage_backend.cpp +++ /dev/null @@ -1,417 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/broker_storage_backend.h" - -#include "env/env.h" -#include "gen_cpp/FrontendService.h" -#include "gen_cpp/FrontendService_types.h" -#include "gen_cpp/HeartbeatService_types.h" -#include "gen_cpp/PaloBrokerService_types.h" -#include "gen_cpp/TPaloBrokerService.h" -#include "io/broker_writer.h" -#include "io/file_factory.h" -#include "io/fs/file_reader.h" -#include "io/fs/file_reader_options.h" -#include "olap/file_helper.h" -#include "olap/iterators.h" -#include "runtime/client_cache.h" -#include "runtime/exec_env.h" - -namespace doris { - -#ifdef BE_TEST -inline BrokerServiceClientCache* client_cache(ExecEnv* env) { - static BrokerServiceClientCache s_client_cache; - return &s_client_cache; -} -#else -inline BrokerServiceClientCache* client_cache(ExecEnv* env) { - return env->broker_client_cache(); -} -#endif - -void BrokerStorageBackend::_init_file_description(const std::string& remote) { - _file_description.path = remote; - _file_description.start_offset = 0; - _file_description.file_size = 0; -} - -BrokerStorageBackend::BrokerStorageBackend(ExecEnv* env, const TNetworkAddress& broker_addr, - const std::map& broker_prop) - : _env(env), _broker_addr(broker_addr), _broker_prop(broker_prop) {} - -Status BrokerStorageBackend::download(const std::string& remote, const std::string& local) { - // 1. open remote file for read - std::shared_ptr file_system; - io::FileReaderSPtr broker_reader = nullptr; - auto cache_policy = io::FileCachePolicy::NO_CACHE; - IOContext io_ctx; - if (config::enable_file_cache && io_ctx.enable_file_cache) { - cache_policy = io::FileCachePolicy::FILE_BLOCK_CACHE; - } - io::FileBlockCachePathPolicy file_block_cache; - io::FileReaderOptions reader_options(cache_policy, file_block_cache); - _init_file_description(remote); - RETURN_IF_ERROR(FileFactory::create_broker_reader(_broker_addr, _broker_prop, _file_description, - &file_system, &broker_reader, reader_options, - &io_ctx)); - - // 2. remove the existing local file if exist - if (std::filesystem::remove(local)) { - VLOG(2) << "remove the previously exist local file: " << local; - } - - // 3. open local file for write - FileHandler file_handler; - Status ost = - file_handler.open_with_mode(local, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR); - if (!ost.ok()) { - return Status::InternalError("failed to open file: {}", local); - } - - // 4. read remote and write to local - VLOG(2) << "read remote file: " << remote << " to local: " << local; - constexpr size_t buf_sz = 1024 * 1024; - std::unique_ptr read_buf(new uint8_t[buf_sz]); - size_t write_offset = 0; - size_t cur_offset = 0; - while (true) { - size_t read_len = 0; - Slice file_slice(read_buf.get(), buf_sz); - RETURN_IF_ERROR(broker_reader->read_at(cur_offset, file_slice, io_ctx, &read_len)); - cur_offset += read_len; - if (read_len == 0) { - break; - } - - ost = file_handler.pwrite(read_buf.get(), read_len, write_offset); - if (!ost.ok()) { - return Status::InternalError("failed to write file: {}", local); - } - write_offset += read_len; - } // file_handler should be closed before calculating checksum - - return Status::OK(); -} - -Status BrokerStorageBackend::direct_download(const std::string& remote, std::string* content) { - return Status::IOError("broker direct_download not support "); -} - -Status BrokerStorageBackend::upload(const std::string& local, const std::string& remote) { - // read file and write to broker - FileHandler file_handler; - Status ost = file_handler.open(local, O_RDONLY); - if (!ost.ok()) { - return Status::InternalError("failed to open file: {}", local); - } - - size_t file_len = file_handler.length(); - if (file_len == -1) { - return Status::InternalError("failed to get length of file: {}", local); - } - - // NOTICE: broker writer must be closed before calling rename - std::vector broker_addrs; - broker_addrs.push_back(_broker_addr); - std::unique_ptr broker_writer( - new BrokerWriter(_env, broker_addrs, _broker_prop, remote, 0 /* offset */)); - RETURN_IF_ERROR(broker_writer->open()); - - constexpr size_t buf_sz = 1024 * 1024; - char read_buf[buf_sz]; - size_t left_len = file_len; - size_t read_offset = 0; - while (left_len > 0) { - size_t read_len = left_len > buf_sz ? buf_sz : left_len; - ost = file_handler.pread(read_buf, read_len, read_offset); - if (!ost.ok()) { - return Status::InternalError("failed to read file: {}", local); - } - // write through broker - size_t write_len = 0; - RETURN_IF_ERROR(broker_writer->write(reinterpret_cast(read_buf), read_len, - &write_len)); - DCHECK_EQ(write_len, read_len); - - read_offset += read_len; - left_len -= read_len; - } - - // close manually, because we need to check its close status - RETURN_IF_ERROR(broker_writer->close()); - - LOG(INFO) << "finished to write file via broker. file: " << local << ", length: " << file_len; - return Status::OK(); -} - -Status BrokerStorageBackend::rename(const std::string& orig_name, const std::string& new_name) { - Status status = Status::OK(); - BrokerServiceConnection client(client_cache(_env), _broker_addr, config::thrift_rpc_timeout_ms, - &status); - if (!status.ok()) { - std::stringstream ss; - ss << "failed to get broker client. " - << "broker addr: " << _broker_addr << ". msg: " << status; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - try { - TBrokerOperationStatus op_status; - TBrokerRenamePathRequest rename_req; - rename_req.__set_version(TBrokerVersion::VERSION_ONE); - rename_req.__set_srcPath(orig_name); - rename_req.__set_destPath(new_name); - rename_req.__set_properties(_broker_prop); - - try { - client->renamePath(op_status, rename_req); - } catch (apache::thrift::transport::TTransportException& e) { - RETURN_IF_ERROR(client.reopen()); - client->renamePath(op_status, rename_req); - } - - if (op_status.statusCode != TBrokerOperationStatusCode::OK) { - std::stringstream ss; - ss << "Fail to rename file: " << orig_name << " to: " << new_name - << " msg:" << op_status.message; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - } catch (apache::thrift::TException& e) { - std::stringstream ss; - ss << "Fail to rename file: " << orig_name << " to: " << new_name << " msg:" << e.what(); - LOG(WARNING) << ss.str(); - return Status::RpcError(ss.str()); - } - - LOG(INFO) << "finished to rename file. orig: " << orig_name << ", new: " << new_name; - - return status; -} - -Status BrokerStorageBackend::rename_dir(const std::string& orig_name, const std::string& new_name) { - return rename(orig_name, new_name); -} - -Status BrokerStorageBackend::list(const std::string& remote_path, bool contain_md5, bool recursion, - std::map* files) { - Status status = Status::OK(); - BrokerServiceConnection client(client_cache(_env), _broker_addr, config::thrift_rpc_timeout_ms, - &status); - if (!status.ok()) { - std::stringstream ss; - ss << "failed to get broker client. " - << "broker addr: " << _broker_addr << ". msg: " << status; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - try { - // get existing files from remote path - TBrokerListResponse list_rep; - TBrokerListPathRequest list_req; - list_req.__set_version(TBrokerVersion::VERSION_ONE); - list_req.__set_path(remote_path + "/*"); - list_req.__set_isRecursive(false); - list_req.__set_properties(_broker_prop); - list_req.__set_fileNameOnly(true); // we only need file name, not abs path - - try { - client->listPath(list_rep, list_req); - } catch (apache::thrift::transport::TTransportException& e) { - RETURN_IF_ERROR(client.reopen()); - client->listPath(list_rep, list_req); - } - - if (list_rep.opStatus.statusCode == TBrokerOperationStatusCode::FILE_NOT_FOUND) { - LOG(INFO) << "path does not exist: " << remote_path; - return Status::OK(); - } else if (list_rep.opStatus.statusCode != TBrokerOperationStatusCode::OK) { - std::stringstream ss; - ss << "failed to list files from remote path: " << remote_path - << ", msg: " << list_rep.opStatus.message; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - LOG(INFO) << "finished to list files from remote path. file num: " << list_rep.files.size(); - - // split file name and checksum - for (const auto& file : list_rep.files) { - if (file.isDir) { - // this is not a file - continue; - } - - const std::string& file_name = file.path; - size_t pos = file_name.find_last_of("."); - if (pos == std::string::npos || pos == file_name.size() - 1) { - // Not found checksum separator, ignore this file - continue; - } - - FileStat stat = {std::string(file_name, 0, pos), std::string(file_name, pos + 1), - file.size}; - files->emplace(std::string(file_name, 0, pos), stat); - VLOG(2) << "split remote file: " << std::string(file_name, 0, pos) - << ", checksum: " << std::string(file_name, pos + 1); - } - - LOG(INFO) << "finished to split files. valid file num: " << files->size(); - - } catch (apache::thrift::TException& e) { - std::stringstream ss; - ss << "failed to list files in remote path: " << remote_path << ", msg: " << e.what(); - LOG(WARNING) << ss.str(); - return Status::RpcError(ss.str()); - } - - return status; -} -Status BrokerStorageBackend::direct_upload(const std::string& remote, const std::string& content) { - std::vector broker_addrs; - broker_addrs.push_back(_broker_addr); - std::unique_ptr broker_writer( - new BrokerWriter(_env, broker_addrs, _broker_prop, remote, 0 /* offset */)); - RETURN_IF_ERROR(broker_writer->open()); - size_t write_len = 0; - RETURN_IF_ERROR(broker_writer->write(reinterpret_cast(content.c_str()), - content.size(), &write_len)); - DCHECK_EQ(write_len, content.size()); - RETURN_IF_ERROR(broker_writer->close()); - return Status::OK(); -} - -Status BrokerStorageBackend::rm(const std::string& remote) { - Status status = Status::OK(); - BrokerServiceConnection client(client_cache(_env), _broker_addr, config::thrift_rpc_timeout_ms, - &status); - if (!status.ok()) { - std::stringstream ss; - ss << "failed to get broker client. " - << "broker addr: " << _broker_addr << ". msg: " << status; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - try { - // rm file from remote path - TBrokerDeletePathRequest del_req; - TBrokerOperationStatus del_rep; - del_req.__set_version(TBrokerVersion::VERSION_ONE); - del_req.__set_path(remote); - del_req.__set_properties(_broker_prop); - - try { - client->deletePath(del_rep, del_req); - } catch (apache::thrift::transport::TTransportException& e) { - RETURN_IF_ERROR(client.reopen()); - client->deletePath(del_rep, del_req); - } - - if (del_rep.statusCode == TBrokerOperationStatusCode::OK) { - return Status::OK(); - } else { - std::stringstream ss; - ss << "failed to delete from remote path: " << remote << ", msg: " << del_rep.message; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - } catch (apache::thrift::TException& e) { - std::stringstream ss; - ss << "failed to delete file in remote path: " << remote << ", msg: " << e.what(); - LOG(WARNING) << ss.str(); - return Status::RpcError(ss.str()); - } -} - -Status BrokerStorageBackend::rmdir(const std::string& remote) { - return rm(remote); -} - -Status BrokerStorageBackend::copy(const std::string& src, const std::string& dst) { - return Status::NotSupported("copy not implemented!"); -} - -Status BrokerStorageBackend::copy_dir(const std::string& src, const std::string& dst) { - return copy(src, dst); -} - -Status BrokerStorageBackend::mkdir(const std::string& path) { - return Status::NotSupported("mkdir not implemented!"); -} - -Status BrokerStorageBackend::mkdirs(const std::string& path) { - return Status::NotSupported("mkdirs not implemented!"); -} - -Status BrokerStorageBackend::exist(const std::string& path) { - Status status = Status::OK(); - BrokerServiceConnection client(client_cache(_env), _broker_addr, config::thrift_rpc_timeout_ms, - &status); - if (!status.ok()) { - std::stringstream ss; - ss << "failed to get broker client. " - << "broker addr: " << _broker_addr << ". msg: " << status; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - try { - TBrokerCheckPathExistRequest check_req; - TBrokerCheckPathExistResponse check_rep; - check_req.__set_version(TBrokerVersion::VERSION_ONE); - check_req.__set_path(path); - check_req.__set_properties(_broker_prop); - - try { - client->checkPathExist(check_rep, check_req); - } catch (apache::thrift::transport::TTransportException& e) { - RETURN_IF_ERROR(client.reopen()); - client->checkPathExist(check_rep, check_req); - } - - if (check_rep.opStatus.statusCode != TBrokerOperationStatusCode::OK) { - std::stringstream ss; - ss << "failed to check exist: " << path << ", msg: " << check_rep.opStatus.message; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } else if (!check_rep.isPathExist) { - return Status::NotFound("{} not exists!", path); - } else { - return Status::OK(); - } - } catch (apache::thrift::TException& e) { - std::stringstream ss; - ss << "failed to check exist: " << path << ", msg: " << e.what(); - LOG(WARNING) << ss.str(); - return Status::RpcError(ss.str()); - } -} - -Status BrokerStorageBackend::exist_dir(const std::string& path) { - return exist(path); -} - -Status BrokerStorageBackend::upload_with_checksum(const std::string& local, - const std::string& remote, - const std::string& checksum) { - std::string temp = remote + ".part"; - std::string final = remote + "." + checksum; - RETURN_IF_ERROR(upload(local, remote + ".part")); - return rename(temp, final); -} - -} // end namespace doris diff --git a/be/src/util/broker_storage_backend.h b/be/src/util/broker_storage_backend.h deleted file mode 100644 index 09cdfdb436..0000000000 --- a/be/src/util/broker_storage_backend.h +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "io/file_factory.h" -#include "util/storage_backend.h" - -namespace doris { - -class ExecEnv; -class TNetworkAddress; - -class BrokerStorageBackend : public StorageBackend { -public: - BrokerStorageBackend(ExecEnv* env, const TNetworkAddress& broker_addr, - const std::map& broker_prop); - ~BrokerStorageBackend() {} - Status download(const std::string& remote, const std::string& local) override; - Status direct_download(const std::string& remote, std::string* content) override; - Status upload(const std::string& local, const std::string& remote) override; - Status upload_with_checksum(const std::string& local, const std::string& remote, - const std::string& checksum) override; - Status rename(const std::string& orig_name, const std::string& new_name) override; - Status rename_dir(const std::string& orig_name, const std::string& new_name) override; - Status list(const std::string& remote_path, bool contain_md5, bool recursion, - std::map* files) override; - Status direct_upload(const std::string& remote, const std::string& content) override; - Status rm(const std::string& remote) override; - Status rmdir(const std::string& remote) override; - Status copy(const std::string& src, const std::string& dst) override; - Status copy_dir(const std::string& src, const std::string& dst) override; - Status mkdir(const std::string& path) override; - Status mkdirs(const std::string& path) override; - Status exist(const std::string& path) override; - Status exist_dir(const std::string& path) override; - -private: - void _init_file_description(const std::string& remote); - - ExecEnv* _env; - const TNetworkAddress& _broker_addr; - const std::map& _broker_prop; - FileDescription _file_description; -}; -} // end namespace doris diff --git a/be/src/util/file_utils.cpp b/be/src/util/file_utils.cpp index ae3b2837d0..d5c95caadd 100644 --- a/be/src/util/file_utils.cpp +++ b/be/src/util/file_utils.cpp @@ -33,7 +33,6 @@ #include "gutil/strings/split.h" #include "gutil/strings/strip.h" #include "gutil/strings/substitute.h" -#include "olap/file_helper.h" #include "runtime/thread_context.h" #include "util/defer_op.h" diff --git a/be/src/util/hdfs_storage_backend.cpp b/be/src/util/hdfs_storage_backend.cpp deleted file mode 100644 index 6f021a4b73..0000000000 --- a/be/src/util/hdfs_storage_backend.cpp +++ /dev/null @@ -1,335 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/hdfs_storage_backend.h" - -#include "io/file_factory.h" -#include "io/fs/file_reader.h" -#include "io/fs/file_reader_options.h" -#include "io/hdfs_writer.h" -#include "olap/file_helper.h" -#include "olap/iterators.h" -#include "util/hdfs_util.h" - -namespace doris { - -#ifndef CHECK_HDFS_CLIENT -#define CHECK_HDFS_CLIENT(client) \ - if (!client) { \ - return Status::InternalError("init hdfs client error."); \ - } -#endif - -static const std::string hdfs_file_prefix = "hdfs://"; - -HDFSStorageBackend::HDFSStorageBackend(const std::map& prop) - : _properties(prop) { - HDFSCommonBuilder builder; - Status st = createHDFSBuilder(_properties, &builder); - if (st.ok()) { - _hdfs_fs = HDFSHandle::instance().create_hdfs_fs(builder); - DCHECK(_hdfs_fs) << "init hdfs client error."; - } - // if createHDFSBuilder failed, _hdfs_fs will be null. - // and CHECK_HDFS_CLIENT will return error. - // TODO: refacotr StorageBackend, unify into File system -} - -HDFSStorageBackend::~HDFSStorageBackend() { - close(); -} - -Status HDFSStorageBackend::close() { - if (_hdfs_fs != nullptr) { - hdfsDisconnect(_hdfs_fs); - _hdfs_fs = nullptr; - } - return Status::OK(); -} - -// if the format of path is hdfs://ip:port/path, replace it to /path. -// path like hdfs://ip:port/path can't be used by libhdfs3. -std::string HDFSStorageBackend::parse_path(const std::string& path) { - if (path.find(hdfs_file_prefix) != std::string::npos) { - std::string temp = path.substr(hdfs_file_prefix.size()); - std::size_t pos = temp.find_first_of('/'); - return temp.substr(pos); - } else { - return path; - } -} - -Status HDFSStorageBackend::upload(const std::string& local, const std::string& remote) { - FileHandler file_handler; - Status ost = file_handler.open(local, O_RDONLY); - if (!ost.ok()) { - return Status::InternalError("failed to open file: " + local); - } - - size_t file_len = file_handler.length(); - if (file_len == -1) { - return Status::InternalError("failed to get length of file: " + local); - } - - std::unique_ptr hdfs_writer(new HDFSWriter(_properties, remote)); - RETURN_IF_ERROR(hdfs_writer->open()); - constexpr size_t buf_sz = 1024 * 1024; - char read_buf[buf_sz]; - size_t left_len = file_len; - size_t read_offset = 0; - while (left_len > 0) { - size_t read_len = left_len > buf_sz ? buf_sz : left_len; - ost = file_handler.pread(read_buf, read_len, read_offset); - if (!ost.ok()) { - return Status::InternalError("failed to read file: " + local); - } - - size_t write_len = 0; - RETURN_IF_ERROR(hdfs_writer->write(reinterpret_cast(read_buf), read_len, - &write_len)); - DCHECK_EQ(write_len, read_len); - - read_offset += read_len; - left_len -= read_len; - } - - LOG(INFO) << "finished to write file: " << local << ", length: " << file_len; - return Status::OK(); -} - -Status HDFSStorageBackend::direct_upload(const std::string& remote, const std::string& content) { - std::unique_ptr hdfs_writer(new HDFSWriter(_properties, remote)); - RETURN_IF_ERROR(hdfs_writer->open()); - size_t write_len = 0; - RETURN_IF_ERROR(hdfs_writer->write(reinterpret_cast(content.c_str()), - content.size(), &write_len)); - DCHECK_EQ(write_len, content.size()); - return Status::OK(); -} - -Status HDFSStorageBackend::list(const std::string& remote_path, bool contain_md5, bool recursion, - std::map* files) { - CHECK_HDFS_CLIENT(_hdfs_fs); - std::string normal_str = parse_path(remote_path); - int exists = hdfsExists(_hdfs_fs, normal_str.c_str()); - if (exists != 0) { - LOG(INFO) << "path does not exist: " << normal_str << ", err: " << strerror(errno); - return Status::OK(); - } - - int file_num = 0; - hdfsFileInfo* files_info = hdfsListDirectory(_hdfs_fs, normal_str.c_str(), &file_num); - if (files_info == nullptr) { - std::stringstream ss; - ss << "failed to list files from remote path: " << normal_str - << ", err: " << strerror(errno); - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } - - LOG(INFO) << "finished to list files from remote path:" << normal_str - << ", file num:" << file_num; - for (int i = 0; i < file_num; ++i) { - auto file_info = files_info[i]; - if (file_info.mKind == kObjectKindDirectory) { - continue; - } - const std::string& file_name_with_path(file_info.mName); - // get filename - std::filesystem::path file_path(file_name_with_path); - std::string file_name = file_path.filename(); - size_t pos = file_name.find_last_of('.'); - if (pos == std::string::npos || pos == file_name.size() - 1) { - // Not found checksum separator, ignore this file - continue; - } - - FileStat stat = {std::string(file_name, 0, pos), std::string(file_name, pos + 1), - file_info.mSize}; - files->emplace(std::string(file_name, 0, pos), stat); - VLOG(2) << "split remote file: " << std::string(file_name, 0, pos) - << ", checksum: " << std::string(file_name, pos + 1); - } - - hdfsFreeFileInfo(files_info, file_num); - LOG(INFO) << "finished to split files. valid file num: " << files->size(); - return Status::OK(); -} - -Status HDFSStorageBackend::download(const std::string& remote, const std::string& local) { - // 1. open remote file for read - std::shared_ptr file_system; - io::FileReaderSPtr hdfs_reader = nullptr; - auto cache_policy = io::FileCachePolicy::NO_CACHE; - IOContext io_ctx; - if (config::enable_file_cache && io_ctx.enable_file_cache) { - cache_policy = io::FileCachePolicy::FILE_BLOCK_CACHE; - } - io::FileBlockCachePathPolicy file_block_cache; - io::FileReaderOptions reader_options(cache_policy, file_block_cache); - THdfsParams hdfs_params = parse_properties(_properties); - RETURN_IF_ERROR(FileFactory::create_hdfs_reader(hdfs_params, remote, &file_system, &hdfs_reader, - reader_options, &io_ctx)); - - // 2. remove the existing local file if exist - if (std::filesystem::remove(local)) { - LOG(INFO) << "remove the previously exist local file: " << local; - } - - // 3. open local file for write - FileHandler file_handler; - Status ost = - file_handler.open_with_mode(local, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR); - if (!ost.ok()) { - return Status::InternalError("failed to open file: " + local); - } - - // 4. read remote and write to local - LOG(INFO) << "read remote file: " << remote << " to local: " << local; - constexpr size_t buf_sz = 1024 * 1024; - std::unique_ptr read_buf(new char[buf_sz]); - size_t write_offset = 0; - size_t cur_offset = 0; - while (true) { - size_t read_len = 0; - Slice file_slice(read_buf.get(), buf_sz); - RETURN_IF_ERROR(hdfs_reader->read_at(cur_offset, file_slice, io_ctx, &read_len)); - cur_offset += read_len; - if (read_len == 0) { - break; - } - - ost = file_handler.pwrite(read_buf.get(), read_len, write_offset); - if (!ost.ok()) { - return Status::InternalError("failed to write file: " + local); - } - write_offset += read_len; - } - - return Status::OK(); -} - -Status HDFSStorageBackend::direct_download(const std::string& remote, std::string* content) { - std::shared_ptr file_system; - io::FileReaderSPtr hdfs_reader = nullptr; - auto cache_policy = io::FileCachePolicy::NO_CACHE; - IOContext io_ctx; - if (config::enable_file_cache && io_ctx.enable_file_cache) { - cache_policy = io::FileCachePolicy::FILE_BLOCK_CACHE; - } - io::FileBlockCachePathPolicy file_block_cache; - io::FileReaderOptions reader_options(cache_policy, file_block_cache); - THdfsParams hdfs_params = parse_properties(_properties); - RETURN_IF_ERROR(FileFactory::create_hdfs_reader(hdfs_params, remote, &file_system, &hdfs_reader, - reader_options, &io_ctx)); - - constexpr size_t buf_sz = 1024 * 1024; - std::unique_ptr read_buf(new char[buf_sz]); - size_t write_offset = 0; - size_t cur_offset = 0; - while (true) { - size_t read_len = 0; - Slice file_slice(read_buf.get(), buf_sz); - RETURN_IF_ERROR(hdfs_reader->read_at(cur_offset, file_slice, io_ctx, &read_len)); - cur_offset += read_len; - if (read_len == 0) { - break; - } - - content->insert(write_offset, read_buf.get(), read_len); - write_offset += read_len; - } - return Status::OK(); -} - -Status HDFSStorageBackend::upload_with_checksum(const std::string& local, const std::string& remote, - const std::string& checksum) { - std::string temp = remote + ".part"; - std::string final = remote + "." + checksum; - RETURN_IF_ERROR(upload(local, temp)); - return rename(temp, final); -} - -Status HDFSStorageBackend::rename(const std::string& orig_name, const std::string& new_name) { - CHECK_HDFS_CLIENT(_hdfs_fs); - std::string normal_orig_name = parse_path(orig_name); - std::string normal_new_name = parse_path(new_name); - int ret = hdfsRename(_hdfs_fs, normal_orig_name.c_str(), normal_new_name.c_str()); - if (ret == 0) { - LOG(INFO) << "finished to rename file. orig: " << normal_orig_name - << ", new: " << normal_new_name; - return Status::OK(); - } else { - std::stringstream ss; - ss << "Fail to rename file: " << normal_orig_name << " to: " << normal_new_name; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } -} - -Status HDFSStorageBackend::rename_dir(const std::string& orig_name, const std::string& new_name) { - return rename(orig_name, new_name); -} - -Status HDFSStorageBackend::copy(const std::string& src, const std::string& dst) { - return Status::NotSupported("copy not implemented!"); -} - -Status HDFSStorageBackend::copy_dir(const std::string& src, const std::string& dst) { - return copy(src, dst); -} - -Status HDFSStorageBackend::mkdir(const std::string& path) { - return Status::NotSupported("mkdir not implemented!"); -} - -Status HDFSStorageBackend::mkdirs(const std::string& path) { - return Status::NotSupported("mkdirs not implemented!"); -} - -Status HDFSStorageBackend::exist(const std::string& path) { - CHECK_HDFS_CLIENT(_hdfs_fs); - int exists = hdfsExists(_hdfs_fs, path.c_str()); - if (exists == 0) { - return Status::OK(); - } else { - return Status::NotFound(path + " not exists!"); - } -} - -Status HDFSStorageBackend::exist_dir(const std::string& path) { - return exist(path); -} - -Status HDFSStorageBackend::rm(const std::string& remote) { - CHECK_HDFS_CLIENT(_hdfs_fs); - int ret = hdfsDelete(_hdfs_fs, remote.c_str(), 1); - if (ret == 0) { - return Status::OK(); - } else { - std::stringstream ss; - ss << "failed to delete from remote path: " << remote; - LOG(WARNING) << ss.str(); - return Status::InternalError(ss.str()); - } -} - -Status HDFSStorageBackend::rmdir(const std::string& remote) { - return rm(remote); -} - -} // end namespace doris diff --git a/be/src/util/hdfs_storage_backend.h b/be/src/util/hdfs_storage_backend.h deleted file mode 100644 index acbf18d2d0..0000000000 --- a/be/src/util/hdfs_storage_backend.h +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "io/hdfs_builder.h" -#include "util/storage_backend.h" - -namespace doris { - -class HDFSStorageBackend : public StorageBackend { -public: - HDFSStorageBackend(const std::map& prop); - ~HDFSStorageBackend(); - - Status close(); - Status download(const std::string& remote, const std::string& local) override; - Status direct_download(const std::string& remote, std::string* content) override; - Status upload(const std::string& local, const std::string& remote) override; - Status upload_with_checksum(const std::string& local, const std::string& remote, - const std::string& checksum) override; - Status list(const std::string& remote_path, bool contain_md5, bool recursion, - std::map* files) override; - Status rename(const std::string& orig_name, const std::string& new_name) override; - Status rename_dir(const std::string& orig_name, const std::string& new_name) override; - Status direct_upload(const std::string& remote, const std::string& content) override; - Status rm(const std::string& remote) override; - Status rmdir(const std::string& remote) override; - Status copy(const std::string& src, const std::string& dst) override; - Status copy_dir(const std::string& src, const std::string& dst) override; - Status mkdir(const std::string& path) override; - Status mkdirs(const std::string& path) override; - Status exist(const std::string& path) override; - Status exist_dir(const std::string& path) override; - -private: - // if the format of path is hdfs://ip:port/path, replace it to /path. - // path like hdfs://ip:port/path can't be used by libhdfs3. - std::string parse_path(const std::string& path); - -private: - std::map _properties; - hdfsFS _hdfs_fs = nullptr; -}; - -} // end namespace doris diff --git a/be/src/util/hdfs_util.cpp b/be/src/util/hdfs_util.cpp index b58fc75e46..a5ad09ec01 100644 --- a/be/src/util/hdfs_util.cpp +++ b/be/src/util/hdfs_util.cpp @@ -23,6 +23,7 @@ #include "common/logging.h" namespace doris { +namespace io { HDFSHandle& HDFSHandle::instance() { static HDFSHandle hdfs_handle; @@ -39,4 +40,14 @@ hdfsFS HDFSHandle::create_hdfs_fs(HDFSCommonBuilder& hdfs_builder) { return hdfs_fs; } +Path convert_path(const Path& path, const std::string& namenode) { + Path real_path(path); + if (path.string().find(namenode) != std::string::npos) { + std::string real_path_str = path.string().substr(namenode.size()); + real_path = real_path_str; + } + return real_path; +} + +} // namespace io } // namespace doris diff --git a/be/src/util/hdfs_util.h b/be/src/util/hdfs_util.h index f7bfc14b3a..f98bdd5ab3 100644 --- a/be/src/util/hdfs_util.h +++ b/be/src/util/hdfs_util.h @@ -24,9 +24,11 @@ #include #include "common/status.h" +#include "io/fs/path.h" #include "io/hdfs_builder.h" namespace doris { +namespace io { class HDFSHandle { public: @@ -40,4 +42,9 @@ private: HDFSHandle() {} }; -} // namespace doris \ No newline at end of file +// if the format of path is hdfs://ip:port/path, replace it to /path. +// path like hdfs://ip:port/path can't be used by libhdfs3. +Path convert_path(const Path& path, const std::string& namenode); + +} // namespace io +} // namespace doris diff --git a/be/src/util/s3_storage_backend.cpp b/be/src/util/s3_storage_backend.cpp deleted file mode 100644 index f074e682c1..0000000000 --- a/be/src/util/s3_storage_backend.cpp +++ /dev/null @@ -1,307 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/s3_storage_backend.h" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "common/logging.h" -#include "gutil/strings/strip.h" -#include "util/s3_uri.h" -#include "util/s3_util.h" - -namespace doris { - -#ifndef CHECK_S3_CLIENT -#define CHECK_S3_CLIENT(client) \ - if (!client) { \ - return Status::InternalError("init aws s3 client error."); \ - } -#endif - -#ifndef CHECK_S3_PATH -#define CHECK_S3_PATH(uri, path) \ - S3URI uri(path); \ - if (!uri.parse()) { \ - return Status::InvalidArgument("s3 uri is invalid: {}", path); \ - } -#endif - -#ifndef RETRUN_S3_STATUS -#define RETRUN_S3_STATUS(response) \ - if (response.IsSuccess()) { \ - return Status::OK(); \ - } else { \ - return Status::InternalError(error_msg(response)); \ - } -#endif - -S3StorageBackend::S3StorageBackend(const std::map& prop) - : _properties(prop) { - _client = ClientFactory::instance().create(_properties); - DCHECK(_client) << "init aws s3 client error."; -} - -S3StorageBackend::~S3StorageBackend() {} - -Status S3StorageBackend::download(const std::string& remote, const std::string& local) { - CHECK_S3_CLIENT(_client); - CHECK_S3_PATH(uri, remote); - Aws::S3::Model::GetObjectRequest request; - request.WithBucket(uri.get_bucket()).WithKey(uri.get_key()); - Aws::S3::Model::GetObjectOutcome response = _client->GetObject(request); - if (response.IsSuccess()) { - Aws::OFStream local_file; - local_file.open(local, std::ios::out | std::ios::binary); - if (local_file.good()) { - local_file << response.GetResult().GetBody().rdbuf(); - } - if (!local_file.good()) { - return Status::InternalError("failed to write file: {}", local); - } - } else { - return Status::IOError("s3 download error: {}", error_msg(response)); - } - return Status::OK(); -} - -Status S3StorageBackend::direct_download(const std::string& remote, std::string* content) { - CHECK_S3_CLIENT(_client); - CHECK_S3_PATH(uri, remote); - Aws::S3::Model::GetObjectRequest request; - request.WithBucket(uri.get_bucket()).WithKey(uri.get_key()); - Aws::S3::Model::GetObjectOutcome response = _client->GetObject(request); - if (response.IsSuccess()) { - std::stringstream ss; - ss << response.GetResult().GetBody().rdbuf(); - *content = ss.str(); - } else { - return Status::IOError("s3 direct_download error: {}", error_msg(response)); - } - return Status::OK(); -} - -Status S3StorageBackend::upload(const std::string& local, const std::string& remote) { - CHECK_S3_CLIENT(_client); - CHECK_S3_PATH(uri, remote); - Aws::S3::Model::PutObjectRequest request; - request.WithBucket(uri.get_bucket()).WithKey(uri.get_key()); - - const std::shared_ptr input_data = Aws::MakeShared( - local.c_str(), local.c_str(), std::ios_base::in | std::ios_base::binary); - if (input_data->good()) { - request.SetBody(input_data); - } - if (!input_data->good()) { - return Status::InternalError("failed to read file: {}", local); - } - Aws::S3::Model::PutObjectOutcome response = _client->PutObject(request); - - RETRUN_S3_STATUS(response); -} - -Status S3StorageBackend::list(const std::string& remote_path, bool contain_md5, bool recursion, - std::map* files) { - std::string normal_str(remote_path); - if (!normal_str.empty() && normal_str.at(normal_str.size() - 1) != '/') { - normal_str += '/'; - } - CHECK_S3_CLIENT(_client); - CHECK_S3_PATH(uri, normal_str); - - Aws::S3::Model::ListObjectsRequest request; - request.WithBucket(uri.get_bucket()).WithPrefix(uri.get_key()); - if (!recursion) { - request.WithDelimiter("/"); - } - Aws::S3::Model::ListObjectsOutcome response = _client->ListObjects(request); - if (response.IsSuccess()) { - Aws::Vector objects = response.GetResult().GetContents(); - - for (Aws::S3::Model::Object& object : objects) { - std::string key = object.GetKey(); - if (key.at(key.size() - 1) == '/') { - continue; - } - key = boost::algorithm::replace_first_copy(key, uri.get_key(), ""); - if (contain_md5) { - size_t pos = key.find_last_of("."); - if (pos == std::string::npos || pos == key.size() - 1) { - // Not found checksum separator, ignore this file - continue; - } - FileStat stat = {std::string(key, 0, pos), std::string(key, pos + 1), - object.GetSize()}; - files->emplace(std::string(key, 0, pos), stat); - } else { - FileStat stat = {key, "", object.GetSize()}; - files->emplace(key, stat); - } - } - return Status::OK(); - } else { - return Status::InternalError("list form s3 error: {}", error_msg(response)); - } -} - -Status S3StorageBackend::rename(const std::string& orig_name, const std::string& new_name) { - RETURN_IF_ERROR(copy(orig_name, new_name)); - return rm(orig_name); -} - -Status S3StorageBackend::rename_dir(const std::string& orig_name, const std::string& new_name) { - RETURN_IF_ERROR(copy_dir(orig_name, new_name)); - return rmdir(orig_name); -} - -Status S3StorageBackend::direct_upload(const std::string& remote, const std::string& content) { - CHECK_S3_CLIENT(_client); - CHECK_S3_PATH(uri, remote); - Aws::S3::Model::PutObjectRequest request; - request.WithBucket(uri.get_bucket()).WithKey(uri.get_key()); - const std::shared_ptr input_data = - Aws::MakeShared("upload_directly"); - *input_data << content.c_str(); - if (input_data->good()) { - request.SetBody(input_data); - } - if (!input_data->good()) { - return Status::InternalError("failed to read from string"); - } - Aws::S3::Model::PutObjectOutcome response = _client->PutObject(request); - - RETRUN_S3_STATUS(response); -} - -Status S3StorageBackend::rm(const std::string& remote) { - LOG(INFO) << "S3 rm file: " << remote; - CHECK_S3_CLIENT(_client); - CHECK_S3_PATH(uri, remote); - Aws::S3::Model::DeleteObjectRequest request; - - request.WithKey(uri.get_key()).WithBucket(uri.get_bucket()); - - Aws::S3::Model::DeleteObjectOutcome response = _client->DeleteObject(request); - - RETRUN_S3_STATUS(response); -} - -Status S3StorageBackend::rmdir(const std::string& remote) { - CHECK_S3_CLIENT(_client); - std::map files; - std::string normal_path(remote); - if (!normal_path.empty() && normal_path.at(normal_path.size() - 1) != '/') { - normal_path += '/'; - } - LOG(INFO) << "Remove S3 dir: " << remote; - RETURN_IF_ERROR(list(normal_path, false, true, &files)); - - for (auto& file : files) { - std::string file_path = normal_path + file.second.name; - RETURN_IF_ERROR(rm(file_path)); - } - return Status::OK(); -} - -Status S3StorageBackend::copy(const std::string& src, const std::string& dst) { - CHECK_S3_CLIENT(_client); - CHECK_S3_PATH(src_uri, src); - CHECK_S3_PATH(dst_uri, dst); - Aws::S3::Model::CopyObjectRequest request; - request.WithCopySource(src_uri.get_bucket() + "/" + src_uri.get_key()) - .WithKey(dst_uri.get_key()) - .WithBucket(dst_uri.get_bucket()); - - Aws::S3::Model::CopyObjectOutcome response = _client->CopyObject(request); - - RETRUN_S3_STATUS(response); -} - -Status S3StorageBackend::copy_dir(const std::string& src, const std::string& dst) { - std::map files; - LOG(INFO) << "Copy S3 dir: " << src << " -> " << dst; - RETURN_IF_ERROR(list(src, false, true, &files)); - if (files.size() == 0) { - LOG(WARNING) << "Nothing need to copy: " << src << " -> " << dst; - return Status::OK(); - } - for (auto& kv : files) { - RETURN_IF_ERROR(copy(src + "/" + kv.first, dst + "/" + kv.first)); - } - return Status::OK(); -} - -// dir is not supported by s3, it will cause something confusion. -Status S3StorageBackend::mkdir(const std::string& path) { - return Status::OK(); -} - -// dir is not supported by s3, it will cause something confusion. -Status S3StorageBackend::mkdirs(const std::string& path) { - return Status::OK(); -} - -Status S3StorageBackend::exist(const std::string& path) { - CHECK_S3_CLIENT(_client); - CHECK_S3_PATH(uri, path); - Aws::S3::Model::HeadObjectRequest request; - request.WithBucket(uri.get_bucket()).WithKey(uri.get_key()); - Aws::S3::Model::HeadObjectOutcome response = _client->HeadObject(request); - if (response.IsSuccess()) { - return Status::OK(); - } else if (response.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return Status::NotFound("{} not exists!", path); - } else { - return Status::InternalError(error_msg(response)); - } -} - -Status S3StorageBackend::exist_dir(const std::string& path) { - std::map files; - RETURN_IF_ERROR(list(path, false, true, &files)); - if (files.size() > 0) { - return Status::OK(); - } - return Status::NotFound("{} not exists!", path); -} - -Status S3StorageBackend::upload_with_checksum(const std::string& local, const std::string& remote, - const std::string& checksum) { - return upload(local, remote + "." + checksum); -} - -template -std::string S3StorageBackend::error_msg(const AwsOutcome& outcome) { - std::stringstream out; - out << "Error: [" << outcome.GetError().GetExceptionName() << ":" - << outcome.GetError().GetMessage(); - return out.str(); -} - -} // end namespace doris diff --git a/be/src/util/s3_storage_backend.h b/be/src/util/s3_storage_backend.h deleted file mode 100644 index b6d31c5593..0000000000 --- a/be/src/util/s3_storage_backend.h +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -namespace Aws { -namespace S3 { -class S3Client; -} // namespace S3 -} // namespace Aws - -namespace doris { - -class S3StorageBackend : public StorageBackend { -public: - S3StorageBackend(const std::map& prop); - ~S3StorageBackend(); - Status download(const std::string& remote, const std::string& local) override; - Status direct_download(const std::string& remote, std::string* content) override; - Status upload(const std::string& local, const std::string& remote) override; - Status upload_with_checksum(const std::string& local, const std::string& remote, - const std::string& checksum) override; - Status list(const std::string& remote_path, bool contain_md5, bool recursion, - std::map* files) override; - Status rename(const std::string& orig_name, const std::string& new_name) override; - Status rename_dir(const std::string& orig_name, const std::string& new_name) override; - Status direct_upload(const std::string& remote, const std::string& content) override; - Status rm(const std::string& remote) override; - Status rmdir(const std::string& remote) override; - Status copy(const std::string& src, const std::string& dst) override; - Status copy_dir(const std::string& src, const std::string& dst) override; - Status mkdir(const std::string& path) override; - Status mkdirs(const std::string& path) override; - Status exist(const std::string& path) override; - Status exist_dir(const std::string& path) override; - -private: - template - std::string error_msg(const AwsOutcome& outcome); - const std::map& _properties; - std::shared_ptr _client; -}; - -} // end namespace doris diff --git a/be/src/util/s3_uri.cpp b/be/src/util/s3_uri.cpp index 36ceec0eae..bbc664237c 100644 --- a/be/src/util/s3_uri.cpp +++ b/be/src/util/s3_uri.cpp @@ -29,39 +29,42 @@ const std::string S3URI::_SCHEME_DELIM = "://"; const std::string S3URI::_PATH_DELIM = "/"; const std::string S3URI::_QUERY_DELIM = "?"; const std::string S3URI::_FRAGMENT_DELIM = "#"; -const StringCaseSet S3URI::_VALID_SCHEMES = {"http", "https", "s3", "s3a", "s3n", "bos", "oss"}; -bool S3URI::parse() { +const StringCaseSet S3URI::_VALID_SCHEMES = {"http", "https", "s3", "s3a", "s3n", + "bos", "oss", "cos", "obs"}; + +/// eg: +/// s3://bucket1/path/to/file.txt +/// _schema: s3 +/// _bucket: bucket1 +/// _key: path/to/file.txt +Status S3URI::parse() { if (_location.empty()) { - return false; + return Status::InvalidArgument("location is empty"); } std::vector scheme_split = strings::Split(_location, _SCHEME_DELIM); if (scheme_split.size() != 2) { - LOG(WARNING) << "Invalid S3 URI: " << _location; - return false; + return Status::InvalidArgument("Invalid S3 URI: {}", _location); } _scheme = scheme_split[0]; if (_VALID_SCHEMES.find(_scheme) == _VALID_SCHEMES.end()) { - LOG(WARNING) << "Invalid scheme: " << _scheme; - return false; + return Status::InvalidArgument("Invalid scheme: {}", _scheme); } std::vector authority_split = strings::Split(scheme_split[1], strings::delimiter::Limit(_PATH_DELIM, 1)); if (authority_split.size() != 2) { - LOG(WARNING) << "Invalid S3 URI: " << _location; - return false; + return Status::InvalidArgument("Invalid S3 URI: {}", _location); } _key = authority_split[1]; StripWhiteSpace(&_key); if (_key.empty()) { - LOG(WARNING) << "Invalid S3 key: " << _location; - return false; + return Status::InvalidArgument("Invalid S3 key: {}", _location); } _bucket = authority_split[0]; // Strip query and fragment if they exist std::vector _query_split = strings::Split(_key, _QUERY_DELIM); std::vector _fragment_split = strings::Split(_query_split[0], _FRAGMENT_DELIM); _key = _fragment_split[0]; - return true; + return Status::OK(); } std::string S3URI::to_string() const { diff --git a/be/src/util/s3_uri.h b/be/src/util/s3_uri.h index 46e61b3282..de4bd05547 100644 --- a/be/src/util/s3_uri.h +++ b/be/src/util/s3_uri.h @@ -19,6 +19,7 @@ #include +#include "common/status.h" #include "util/string_util.h" namespace doris { @@ -26,7 +27,7 @@ namespace doris { class S3URI { public: S3URI(const std::string& location) : _location(location) {} - bool parse(); + Status parse(); const std::string& get_bucket() const { return _bucket; } const std::string& get_key() const { return _key; } const std::string& get_location() const { return _location; } diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index 7b1c88b220..c797dcb203 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -77,7 +77,7 @@ private: const static std::string USE_PATH_STYLE = "use_path_style"; -ClientFactory::ClientFactory() { +S3ClientFactory::S3ClientFactory() { _aws_options = Aws::SDKOptions {}; Aws::Utils::Logging::LogLevel logLevel = static_cast(config::aws_log_level); @@ -88,16 +88,16 @@ ClientFactory::ClientFactory() { Aws::InitAPI(_aws_options); } -ClientFactory::~ClientFactory() { +S3ClientFactory::~S3ClientFactory() { Aws::ShutdownAPI(_aws_options); } -ClientFactory& ClientFactory::instance() { - static ClientFactory ret; +S3ClientFactory& S3ClientFactory::instance() { + static S3ClientFactory ret; return ret; } -bool ClientFactory::is_s3_conf_valid(const std::map& prop) { +bool S3ClientFactory::is_s3_conf_valid(const std::map& prop) { StringCaseMap properties(prop.begin(), prop.end()); if (properties.find(S3_AK) == properties.end() || properties.find(S3_SK) == properties.end() || properties.find(S3_ENDPOINT) == properties.end() || @@ -109,7 +109,7 @@ bool ClientFactory::is_s3_conf_valid(const std::map& p return true; } -std::shared_ptr ClientFactory::create( +std::shared_ptr S3ClientFactory::create( const std::map& prop) { if (!is_s3_conf_valid(prop)) { return nullptr; @@ -148,11 +148,11 @@ std::shared_ptr ClientFactory::create( Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, use_virtual_addressing); } -bool ClientFactory::is_s3_conf_valid(const S3Conf& s3_conf) { +bool S3ClientFactory::is_s3_conf_valid(const S3Conf& s3_conf) { return !s3_conf.ak.empty() && !s3_conf.sk.empty() && !s3_conf.endpoint.empty(); } -std::shared_ptr ClientFactory::create(const S3Conf& s3_conf) { +std::shared_ptr S3ClientFactory::create(const S3Conf& s3_conf) { if (!is_s3_conf_valid(s3_conf)) { return nullptr; } @@ -177,8 +177,8 @@ std::shared_ptr ClientFactory::create(const S3Conf& s3_conf) s3_conf.use_virtual_addressing); } -Status ClientFactory::convert_properties_to_s3_conf(const std::map& prop, - const S3URI& s3_uri, S3Conf* s3_conf) { +Status S3ClientFactory::convert_properties_to_s3_conf( + const std::map& prop, const S3URI& s3_uri, S3Conf* s3_conf) { if (!is_s3_conf_valid(prop)) { return Status::InvalidArgument("S3 properties are incorrect, please check properties."); } diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 4adcc72874..7ef5ca00c5 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -65,11 +65,11 @@ struct S3Conf { } }; -class ClientFactory { +class S3ClientFactory { public: - ~ClientFactory(); + ~S3ClientFactory(); - static ClientFactory& instance(); + static S3ClientFactory& instance(); std::shared_ptr create(const std::map& prop); @@ -83,7 +83,7 @@ public: const S3URI& s3_uri, S3Conf* s3_conf); private: - ClientFactory(); + S3ClientFactory(); Aws::SDKOptions _aws_options; }; diff --git a/be/src/util/storage_backend.h b/be/src/util/storage_backend.h deleted file mode 100644 index d18b4e73b0..0000000000 --- a/be/src/util/storage_backend.h +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "common/status.h" - -namespace doris { - -struct FileStat { - std::string name; - std::string md5; - int64_t size; -}; - -class StorageBackend { -public: - virtual Status download(const std::string& remote, const std::string& local) = 0; - virtual Status direct_download(const std::string& remote, std::string* content) = 0; - virtual Status upload(const std::string& local, const std::string& remote) = 0; - virtual Status upload_with_checksum(const std::string& local, const std::string& remote, - const std::string& checksum) = 0; - virtual Status list(const std::string& remote_path, bool contain_md5, bool recursion, - std::map* files) = 0; - virtual Status rename(const std::string& orig_name, const std::string& new_name) = 0; - virtual Status rename_dir(const std::string& orig_name, const std::string& new_name) = 0; - virtual Status direct_upload(const std::string& remote, const std::string& content) = 0; - virtual Status copy(const std::string& src, const std::string& dst) = 0; - virtual Status copy_dir(const std::string& src, const std::string& dst) = 0; - virtual Status rm(const std::string& remote) = 0; - virtual Status rmdir(const std::string& remote) = 0; - virtual Status mkdir(const std::string& path) = 0; - virtual Status mkdirs(const std::string& path) = 0; - virtual Status exist(const std::string& path) = 0; - virtual Status exist_dir(const std::string& path) = 0; - - virtual ~StorageBackend() = default; -}; -} // end namespace doris diff --git a/be/src/util/storage_backend_mgr.cpp b/be/src/util/storage_backend_mgr.cpp deleted file mode 100644 index 4d940faf05..0000000000 --- a/be/src/util/storage_backend_mgr.cpp +++ /dev/null @@ -1,217 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/storage_backend_mgr.h" - -#include "common/config.h" -#include "common/status.h" -#include "env/env.h" -#include "env/env_util.h" -#include "gutil/strings/substitute.h" -#include "util/file_utils.h" -#include "util/s3_storage_backend.h" -#include "util/s3_util.h" -#include "util/storage_backend.h" - -namespace doris { -using namespace ErrorCode; - -Status StorageBackendMgr::init(const std::string& storage_param_dir) { - if (_is_inited) { - return Status::OK(); - } - Status exist_status = Env::Default()->path_exists(storage_param_dir); - if (!exist_status.ok() && - (!exist_status.is() || !Env::Default()->create_dirs(storage_param_dir).ok())) { - RETURN_NOT_OK_STATUS_WITH_WARN( - Status::IOError("failed to create remote storage_param root path {}", - storage_param_dir), - "create_dirs failed"); - } - - std::vector file_names; - RETURN_IF_ERROR(FileUtils::list_files(Env::Default(), storage_param_dir, &file_names)); - for (auto& file_name : file_names) { - std::string buf; - RETURN_NOT_OK_STATUS_WITH_WARN( - env_util::read_file_to_string(Env::Default(), storage_param_dir + "/" + file_name, - &buf), - strings::Substitute("load storage_name failed. $0", file_name)); - StorageParamPB storage_param_pb; - RETURN_IF_ERROR(_deserialize_param(buf, &storage_param_pb)); - RETURN_IF_ERROR(_create_remote_storage_internal(storage_param_pb)); - LOG(INFO) << "init remote_storage_param successfully. storage_name: " << file_name; - } - _storage_param_dir = storage_param_dir; - _is_inited = true; - return Status::OK(); -} - -Status StorageBackendMgr::create_remote_storage(const StorageParamPB& storage_param_pb) { - if (_check_exist(storage_param_pb)) { - return Status::OK(); - } - RETURN_IF_ERROR(_create_remote_storage_internal(storage_param_pb)); - - std::string storage_name = storage_param_pb.storage_name(); - - string storage_param_path = _storage_param_dir + "/" + storage_name; - RETURN_NOT_OK_STATUS_WITH_WARN( - FileUtils::remove(storage_param_path), - strings::Substitute("rm storage_param_pb file failed: $0", storage_param_path)); - std::string param_binary; - RETURN_NOT_OK_STATUS_WITH_WARN(_serialize_param(storage_param_pb, ¶m_binary), - "_serialize_param storage_param_pb failed."); - RETURN_NOT_OK_STATUS_WITH_WARN( - env_util::write_string_to_file(Env::Default(), Slice(param_binary), storage_param_path), - strings::Substitute("write_string_to_file failed: $0", storage_param_path)); - std::string buf; - RETURN_NOT_OK_STATUS_WITH_WARN( - env_util::read_file_to_string(Env::Default(), storage_param_path, &buf), - strings::Substitute("read storage_name failed. $0", storage_param_path)); - if (buf != param_binary) { - LOG(ERROR) << "storage_param written failed. storage_name: (" - << storage_param_pb.storage_name() << "<->" << storage_name << ")"; - return Status::InternalError("storage_param written failed"); - } - LOG(INFO) << "create remote_storage_param successfully. storage_name: " << storage_name; - return Status::OK(); -} - -Status StorageBackendMgr::_create_remote_storage_internal(const StorageParamPB& storage_param_pb) { - std::string storage_name = storage_param_pb.storage_name(); - std::unique_lock wrlock(_storage_backend_lock); - if (_storage_backend_map.size() >= doris::config::max_remote_storage_count) { - std::map::iterator itr = _storage_backend_active_time.begin(); - std::string timeout_key = itr->first; - time_t min_active_time = itr->second; - ++itr; - for (; itr != _storage_backend_active_time.end(); ++itr) { - if (itr->second < min_active_time) { - timeout_key = itr->first; - min_active_time = itr->second; - } - } - _storage_backend_map.erase(storage_name); - _storage_param_map.erase(storage_name); - _storage_backend_active_time.erase(storage_name); - } - std::map storage_prop; - switch (storage_param_pb.storage_medium()) { - case StorageMediumPB::S3: - default: - S3StorageParamPB s3_storage_param = storage_param_pb.s3_storage_param(); - if (s3_storage_param.s3_ak().empty() || s3_storage_param.s3_sk().empty() || - s3_storage_param.s3_endpoint().empty() || s3_storage_param.s3_region().empty()) { - return Status::InternalError("s3_storage_param param is invalid"); - } - storage_prop[S3_AK] = s3_storage_param.s3_ak(); - storage_prop[S3_SK] = s3_storage_param.s3_sk(); - storage_prop[S3_ENDPOINT] = s3_storage_param.s3_endpoint(); - storage_prop[S3_REGION] = s3_storage_param.s3_region(); - storage_prop[S3_MAX_CONN_SIZE] = s3_storage_param.s3_max_conn(); - storage_prop[S3_REQUEST_TIMEOUT_MS] = s3_storage_param.s3_request_timeout_ms(); - storage_prop[S3_CONN_TIMEOUT_MS] = s3_storage_param.s3_conn_timeout_ms(); - - if (!ClientFactory::is_s3_conf_valid(storage_prop)) { - return Status::InternalError("s3_storage_param is invalid"); - } - _storage_backend_map[storage_name] = std::make_shared(storage_prop); - } - _storage_param_map[storage_name] = storage_param_pb; - _storage_backend_active_time[storage_name] = time(nullptr); - - return Status::OK(); -} - -std::shared_ptr StorageBackendMgr::get_storage_backend( - const std::string& storage_name) { - std::shared_lock rdlock(_storage_backend_lock); - if (_storage_backend_map.find(storage_name) == _storage_backend_map.end()) { - return nullptr; - } - _storage_backend_active_time[storage_name] = time(nullptr); - return _storage_backend_map[storage_name]; -} - -Status StorageBackendMgr::get_storage_param(const std::string& storage_name, - StorageParamPB* storage_param) { - std::shared_lock rdlock(_storage_backend_lock); - if (_storage_backend_map.find(storage_name) == _storage_backend_map.end()) { - return Status::InternalError("storage_name not exist: {}", storage_name); - } - *storage_param = _storage_param_map[storage_name]; - return Status::OK(); -} - -Status StorageBackendMgr::get_root_path(const std::string& storage_name, std::string* root_path) { - std::shared_lock rdlock(_storage_backend_lock); - if (_storage_backend_map.find(storage_name) == _storage_backend_map.end()) { - return Status::InternalError("storage_name not exist: {}", storage_name); - } - *root_path = get_root_path_from_param(_storage_param_map[storage_name]); - return Status::OK(); -} - -std::string StorageBackendMgr::get_root_path_from_param(const StorageParamPB& storage_param) { - switch (storage_param.storage_medium()) { - case StorageMediumPB::S3: - default: { - return storage_param.s3_storage_param().root_path(); - } - } -} - -Status StorageBackendMgr::_check_exist(const StorageParamPB& storage_param_pb) { - StorageParamPB old_storage_param; - RETURN_IF_ERROR(get_storage_param(storage_param_pb.storage_name(), &old_storage_param)); - std::shared_lock rdlock(_storage_backend_lock); - std::string old_param_binary; - RETURN_NOT_OK_STATUS_WITH_WARN(_serialize_param(old_storage_param, &old_param_binary), - "_serialize_param old_storage_param_pb failed."); - std::string param_binary; - RETURN_NOT_OK_STATUS_WITH_WARN(_serialize_param(storage_param_pb, ¶m_binary), - "_serialize_param storage_param_pb failed."); - if (old_param_binary != param_binary) { - LOG(ERROR) << "storage_param has been changed: " << storage_param_pb.storage_name(); - return Status::InternalError("storage_param has been changed"); - } - return Status::OK(); -} - -Status StorageBackendMgr::_serialize_param(const StorageParamPB& storage_param_pb, - std::string* param_binary) { - bool serialize_success = storage_param_pb.SerializeToString(param_binary); - if (!serialize_success) { - LOG(WARNING) << "failed to serialize storage_param " << storage_param_pb.storage_name(); - return Status::InternalError("failed to serialize storage_param: {}", - storage_param_pb.storage_name()); - } - return Status::OK(); -} - -Status StorageBackendMgr::_deserialize_param(const std::string& param_binary, - StorageParamPB* storage_param_pb) { - bool parsed = storage_param_pb->ParseFromString(param_binary); - if (!parsed) { - LOG(WARNING) << "parse storage_param failed"; - return Status::InternalError("parse storage_param failed"); - } - return Status::OK(); -} - -} // namespace doris \ No newline at end of file diff --git a/be/src/util/storage_backend_mgr.h b/be/src/util/storage_backend_mgr.h deleted file mode 100644 index 3fd19dcb99..0000000000 --- a/be/src/util/storage_backend_mgr.h +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "common/status.h" - -namespace doris { - -class StorageBackend; -class StorageParamPB; - -// StorageBackendMgr is used to manage StorageBackend, it has (key -> StorageBackend) map used to connect remote storage -class StorageBackendMgr { -public: - StorageBackendMgr() {} - ~StorageBackendMgr() {} - - static StorageBackendMgr* instance() { - static StorageBackendMgr s_instance; - return &s_instance; - } - - // init() is called when be is started, storage_name_dir is the file path for remote parameter in local cache_path. - Status init(const std::string& storage_name_dir); - - // get_storage_backend by storage_name, one storage_name matches a remote storage_backend. - std::shared_ptr get_storage_backend(const std::string& storage_name); - - // create a new remote storage_backend when it doesn't exist. - Status create_remote_storage(const StorageParamPB& storage_param); - - // get storage_param by storage_name. - Status get_storage_param(const std::string& storage_name, StorageParamPB* storage_param); - - // get root_path of remote storage by storage_name - Status get_root_path(const std::string& storage_name, std::string* root_path); - - // get root_path of remote storage from storage_param - static std::string get_root_path_from_param(const StorageParamPB& storage_param); - -private: - Status _create_remote_storage_internal(const StorageParamPB& storage_param); - Status _check_exist(const StorageParamPB& storage_param_pb); - Status _serialize_param(const StorageParamPB& storage_param_pb, std::string* meta_binary); - Status _deserialize_param(const std::string& meta_binary, StorageParamPB* storage_param_pb); - - std::shared_mutex _storage_backend_lock; - std::map _storage_backend_active_time; - // key is storage_name, value is StorageBackend with one storage_backend. - std::map> _storage_backend_map; - std::map _storage_param_map; - std::string _storage_param_dir; - bool _is_inited = false; -}; - -} // namespace doris \ No newline at end of file diff --git a/be/src/vec/core/block_spill_reader.cpp b/be/src/vec/core/block_spill_reader.cpp index 945d14f9ff..c7871f30a5 100644 --- a/be/src/vec/core/block_spill_reader.cpp +++ b/be/src/vec/core/block_spill_reader.cpp @@ -38,9 +38,8 @@ Status BlockSpillReader::open() { FileDescription file_description; file_description.path = file_path_; - IOContext io_ctx; RETURN_IF_ERROR(FileFactory::create_file_reader(nullptr, system_properties, file_description, - &file_system, &file_reader_, &io_ctx)); + &file_system, &file_reader_)); if (delete_after_read_) { unlink(file_path_.c_str()); } @@ -51,11 +50,11 @@ Status BlockSpillReader::open() { // read block count size_t bytes_read = 0; - RETURN_IF_ERROR(file_reader_->read_at(file_size - sizeof(size_t), result, {}, &bytes_read)); + RETURN_IF_ERROR(file_reader_->read_at(file_size - sizeof(size_t), result, &bytes_read)); // read max sub block size result.data = (char*)&max_sub_block_size_; - RETURN_IF_ERROR(file_reader_->read_at(file_size - sizeof(size_t) * 2, result, {}, &bytes_read)); + RETURN_IF_ERROR(file_reader_->read_at(file_size - sizeof(size_t) * 2, result, &bytes_read)); size_t buff_size = std::max(block_count_ * sizeof(size_t), max_sub_block_size_); read_buff_.reset(new char[buff_size]); @@ -65,7 +64,7 @@ Status BlockSpillReader::open() { result.data = read_buff_.get(); result.size = block_count_ * sizeof(size_t); - RETURN_IF_ERROR(file_reader_->read_at(read_offset, result, {}, &bytes_read)); + RETURN_IF_ERROR(file_reader_->read_at(read_offset, result, &bytes_read)); DCHECK(bytes_read == block_count_ * sizeof(size_t)); block_start_offsets_.resize(block_count_ + 1); @@ -96,7 +95,7 @@ Status BlockSpillReader::read(Block* block, bool* eos) { { SCOPED_TIMER(read_time_); - RETURN_IF_ERROR(file_reader_->read_at(block_start_offsets_[read_block_index_], result, {}, + RETURN_IF_ERROR(file_reader_->read_at(block_start_offsets_[read_block_index_], result, &bytes_read)); } DCHECK(bytes_read == bytes_to_read); diff --git a/be/src/vec/core/block_spill_writer.cpp b/be/src/vec/core/block_spill_writer.cpp index c1a3d8911c..b171eac19a 100644 --- a/be/src/vec/core/block_spill_writer.cpp +++ b/be/src/vec/core/block_spill_writer.cpp @@ -33,9 +33,6 @@ void BlockSpillWriter::_init_profile() { Status BlockSpillWriter::open() { RETURN_IF_ERROR(FileFactory::create_file_writer(TFileType::FILE_LOCAL, ExecEnv::GetInstance(), {}, {}, file_path_, 0, file_writer_)); - - RETURN_IF_ERROR(file_writer_->open()); - is_open_ = true; return Status::OK(); } @@ -53,17 +50,17 @@ Status BlockSpillWriter::close() { meta_.append((const char*)&written_blocks_, sizeof(written_blocks_)); Status status; - size_t written_bytes; // meta: block1 offset, block2 offset, ..., blockn offset, n { SCOPED_TIMER(write_timer_); - status = file_writer_->write((const uint8_t*)meta_.data(), meta_.size(), &written_bytes); + status = file_writer_->append(meta_); } if (!status.ok()) { unlink(file_path_.c_str()); return status; } + RETURN_IF_ERROR(file_writer_->close()); file_writer_.reset(); return Status::OK(); } @@ -125,13 +122,13 @@ Status BlockSpillWriter::_write_internal(const Block& block) { { SCOPED_TIMER(write_timer_); - status = file_writer_->write((const uint8_t*)buff.data(), buff.size(), &written_bytes); + status = file_writer_->append(buff); + written_bytes = buff.size(); } if (!status.ok()) { unlink(file_path_.c_str()); return status; } - DCHECK(written_bytes == buff.size()); max_sub_block_size_ = std::max(max_sub_block_size_, written_bytes); @@ -144,4 +141,4 @@ Status BlockSpillWriter::_write_internal(const Block& block) { } } // namespace vectorized -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/vec/core/block_spill_writer.h b/be/src/vec/core/block_spill_writer.h index f21f38550f..817fb04b93 100644 --- a/be/src/vec/core/block_spill_writer.h +++ b/be/src/vec/core/block_spill_writer.h @@ -17,7 +17,7 @@ #pragma once -#include "io/local_file_writer.h" +#include "io/fs/local_file_writer.h" #include "vec/core/block.h" namespace doris { namespace vectorized { @@ -61,7 +61,7 @@ private: size_t batch_size_; size_t max_sub_block_size_ = 0; std::string file_path_; - std::unique_ptr file_writer_; + std::unique_ptr file_writer_; size_t written_blocks_ = 0; size_t total_written_bytes_ = 0; @@ -78,4 +78,4 @@ private: using BlockSpillWriterUPtr = std::unique_ptr; } // namespace vectorized -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/vec/exec/format/csv/csv_reader.cpp b/be/src/vec/exec/format/csv/csv_reader.cpp index 620a167cee..65d1fb355b 100644 --- a/be/src/vec/exec/format/csv/csv_reader.cpp +++ b/be/src/vec/exec/format/csv/csv_reader.cpp @@ -41,7 +41,7 @@ const static Slice _s_null_slice = Slice("\\N"); CsvReader::CsvReader(RuntimeState* state, RuntimeProfile* profile, ScannerCounter* counter, const TFileScanRangeParams& params, const TFileRangeDesc& range, - const std::vector& file_slot_descs, IOContext* io_ctx) + const std::vector& file_slot_descs, io::IOContext* io_ctx) : _state(state), _profile(profile), _counter(counter), @@ -69,7 +69,7 @@ CsvReader::CsvReader(RuntimeState* state, RuntimeProfile* profile, ScannerCounte CsvReader::CsvReader(RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, - const std::vector& file_slot_descs, IOContext* io_ctx) + const std::vector& file_slot_descs, io::IOContext* io_ctx) : _state(nullptr), _profile(profile), _params(params), @@ -138,9 +138,8 @@ Status CsvReader::init_reader(bool is_load) { if (_params.file_type == TFileType::FILE_STREAM) { RETURN_IF_ERROR(FileFactory::create_pipe_reader(_range.load_id, &_file_reader)); } else { - RETURN_IF_ERROR(FileFactory::create_file_reader(_profile, _system_properties, - _file_description, &_file_system, - &_file_reader, _io_ctx)); + RETURN_IF_ERROR(FileFactory::create_file_reader( + _profile, _system_properties, _file_description, &_file_system, &_file_reader)); } if (_file_reader->size() == 0 && _params.file_type != TFileType::FILE_STREAM && _params.file_type != TFileType::FILE_BROKER) { @@ -230,7 +229,7 @@ Status CsvReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { while (rows < batch_size && !_line_reader_eof) { const uint8_t* ptr = nullptr; size_t size = 0; - RETURN_IF_ERROR(_line_reader->read_line(&ptr, &size, &_line_reader_eof)); + RETURN_IF_ERROR(_line_reader->read_line(&ptr, &size, &_line_reader_eof, _io_ctx)); if (_skip_lines > 0) { _skip_lines--; continue; @@ -625,7 +624,7 @@ Status CsvReader::_prepare_parse(size_t* read_line, bool* is_parse_name) { _file_description.start_offset = start_offset; RETURN_IF_ERROR(FileFactory::create_file_reader(_profile, _system_properties, _file_description, - &_file_system, &_file_reader, _io_ctx)); + &_file_system, &_file_reader)); if (_file_reader->size() == 0 && _params.file_type != TFileType::FILE_STREAM && _params.file_type != TFileType::FILE_BROKER) { return Status::EndOfFile("get parsed schema failed, empty csv file: " + _range.path); @@ -651,7 +650,7 @@ Status CsvReader::_prepare_parse(size_t* read_line, bool* is_parse_name) { Status CsvReader::_parse_col_nums(size_t* col_nums) { const uint8_t* ptr = nullptr; size_t size = 0; - RETURN_IF_ERROR(_line_reader->read_line(&ptr, &size, &_line_reader_eof)); + RETURN_IF_ERROR(_line_reader->read_line(&ptr, &size, &_line_reader_eof, _io_ctx)); if (size == 0) { return Status::InternalError("The first line is empty, can not parse column numbers"); } @@ -667,7 +666,7 @@ Status CsvReader::_parse_col_names(std::vector* col_names) { const uint8_t* ptr = nullptr; size_t size = 0; // no use of _line_reader_eof - RETURN_IF_ERROR(_line_reader->read_line(&ptr, &size, &_line_reader_eof)); + RETURN_IF_ERROR(_line_reader->read_line(&ptr, &size, &_line_reader_eof, _io_ctx)); if (size == 0) { return Status::InternalError("The first line is empty, can not parse column names"); } diff --git a/be/src/vec/exec/format/csv/csv_reader.h b/be/src/vec/exec/format/csv/csv_reader.h index fc644a33ad..f2c2009777 100644 --- a/be/src/vec/exec/format/csv/csv_reader.h +++ b/be/src/vec/exec/format/csv/csv_reader.h @@ -36,11 +36,11 @@ class CsvReader : public GenericReader { public: CsvReader(RuntimeState* state, RuntimeProfile* profile, ScannerCounter* counter, const TFileScanRangeParams& params, const TFileRangeDesc& range, - const std::vector& file_slot_descs, IOContext* io_ctx); + const std::vector& file_slot_descs, io::IOContext* io_ctx); CsvReader(RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& file_slot_descs, - IOContext* io_ctx); + io::IOContext* io_ctx); ~CsvReader() override; Status init_reader(bool is_query); @@ -122,7 +122,7 @@ private: int _line_delimiter_length; bool _trim_double_quotes = false; - IOContext* _io_ctx; + io::IOContext* _io_ctx; // save source text which have been splitted. std::vector _split_values; diff --git a/be/src/vec/exec/format/file_reader/new_plain_binary_line_reader.cpp b/be/src/vec/exec/format/file_reader/new_plain_binary_line_reader.cpp index 35948eb7ca..0e6c34a368 100644 --- a/be/src/vec/exec/format/file_reader/new_plain_binary_line_reader.cpp +++ b/be/src/vec/exec/format/file_reader/new_plain_binary_line_reader.cpp @@ -33,7 +33,8 @@ NewPlainBinaryLineReader::~NewPlainBinaryLineReader() { void NewPlainBinaryLineReader::close() {} -Status NewPlainBinaryLineReader::read_line(const uint8_t** ptr, size_t* size, bool* eof) { +Status NewPlainBinaryLineReader::read_line(const uint8_t** ptr, size_t* size, bool* eof, + const io::IOContext* /*io_ctx*/) { size_t read_size = 0; RETURN_IF_ERROR((dynamic_cast(_file_reader.get())) ->read_one_message(&_file_buf, &read_size)); diff --git a/be/src/vec/exec/format/file_reader/new_plain_binary_line_reader.h b/be/src/vec/exec/format/file_reader/new_plain_binary_line_reader.h index 784682cb56..a96af1c158 100644 --- a/be/src/vec/exec/format/file_reader/new_plain_binary_line_reader.h +++ b/be/src/vec/exec/format/file_reader/new_plain_binary_line_reader.h @@ -33,7 +33,8 @@ public: ~NewPlainBinaryLineReader() override; - Status read_line(const uint8_t** ptr, size_t* size, bool* eof) override; + Status read_line(const uint8_t** ptr, size_t* size, bool* eof, + const io::IOContext* io_ctx) override; void close() override; diff --git a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp index 89089ff375..e4417f90bd 100644 --- a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp +++ b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp @@ -174,7 +174,8 @@ void NewPlainTextLineReader::extend_output_buf() { } while (false); } -Status NewPlainTextLineReader::read_line(const uint8_t** ptr, size_t* size, bool* eof) { +Status NewPlainTextLineReader::read_line(const uint8_t** ptr, size_t* size, bool* eof, + const io::IOContext* io_ctx) { if (_eof || update_eof()) { *size = 0; *eof = true; @@ -229,9 +230,8 @@ Status NewPlainTextLineReader::read_line(const uint8_t** ptr, size_t* size, bool { SCOPED_TIMER(_read_timer); Slice file_slice(file_buf, buffer_len); - IOContext io_ctx; RETURN_IF_ERROR( - _file_reader->read_at(_current_offset, file_slice, io_ctx, &read_len)); + _file_reader->read_at(_current_offset, file_slice, &read_len, io_ctx)); _current_offset += read_len; if (read_len == 0) { _file_eof = true; diff --git a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.h b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.h index 99debf6d18..5d5eb74dfb 100644 --- a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.h +++ b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.h @@ -22,6 +22,9 @@ #include "util/runtime_profile.h" namespace doris { +namespace io { +class IOContext; +} class Decompressor; class Status; @@ -35,7 +38,8 @@ public: ~NewPlainTextLineReader() override; - Status read_line(const uint8_t** ptr, size_t* size, bool* eof) override; + Status read_line(const uint8_t** ptr, size_t* size, bool* eof, + const io::IOContext* io_ctx) override; void close() override; diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index 2d8cf37077..b28a97cfdf 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -42,7 +42,7 @@ using namespace ErrorCode; NewJsonReader::NewJsonReader(RuntimeState* state, RuntimeProfile* profile, ScannerCounter* counter, const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& file_slot_descs, bool* scanner_eof, - IOContext* io_ctx, bool is_dynamic_schema) + io::IOContext* io_ctx, bool is_dynamic_schema) : _vhandle_json_callback(nullptr), _state(state), _profile(profile), @@ -73,7 +73,8 @@ NewJsonReader::NewJsonReader(RuntimeState* state, RuntimeProfile* profile, Scann NewJsonReader::NewJsonReader(RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, - const std::vector& file_slot_descs, IOContext* io_ctx) + const std::vector& file_slot_descs, + io::IOContext* io_ctx) : _vhandle_json_callback(nullptr), _state(nullptr), _profile(profile), @@ -156,7 +157,7 @@ Status NewJsonReader::get_next_block(Block* block, size_t* read_rows, bool* eof) if (UNLIKELY(_read_json_by_line && _skip_first_line)) { size_t size = 0; const uint8_t* line_ptr = nullptr; - RETURN_IF_ERROR(_line_reader->read_line(&line_ptr, &size, &_reader_eof)); + RETURN_IF_ERROR(_line_reader->read_line(&line_ptr, &size, &_reader_eof, _io_ctx)); _skip_first_line = false; continue; } @@ -199,7 +200,7 @@ Status NewJsonReader::get_parsed_schema(std::vector* col_names, std::unique_ptr json_str_ptr; size_t size = 0; if (_line_reader != nullptr) { - RETURN_IF_ERROR(_line_reader->read_line(&json_str, &size, &eof)); + RETURN_IF_ERROR(_line_reader->read_line(&json_str, &size, &eof, _io_ctx)); } else { size_t read_size = 0; RETURN_IF_ERROR(_read_one_message(&json_str_ptr, &read_size)); @@ -334,9 +335,8 @@ Status NewJsonReader::_open_file_reader() { if (_params.file_type == TFileType::FILE_STREAM) { RETURN_IF_ERROR(FileFactory::create_pipe_reader(_range.load_id, &_file_reader)); } else { - RETURN_IF_ERROR(FileFactory::create_file_reader(_profile, _system_properties, - _file_description, &_file_system, - &_file_reader, _io_ctx)); + RETURN_IF_ERROR(FileFactory::create_file_reader( + _profile, _system_properties, _file_description, &_file_system, &_file_reader)); } return Status::OK(); } @@ -400,7 +400,7 @@ Status NewJsonReader::_parse_dynamic_json(bool* is_empty_row, bool* eof, Block& const uint8_t* json_str = nullptr; std::unique_ptr json_str_ptr; if (_line_reader != nullptr) { - RETURN_IF_ERROR(_line_reader->read_line(&json_str, &size, eof)); + RETURN_IF_ERROR(_line_reader->read_line(&json_str, &size, eof, _io_ctx)); } else { size_t length = 0; RETURN_IF_ERROR(_read_one_message(&json_str_ptr, &length)); @@ -637,7 +637,7 @@ Status NewJsonReader::_parse_json_doc(size_t* size, bool* eof) { const uint8_t* json_str = nullptr; std::unique_ptr json_str_ptr; if (_line_reader != nullptr) { - RETURN_IF_ERROR(_line_reader->read_line(&json_str, size, eof)); + RETURN_IF_ERROR(_line_reader->read_line(&json_str, size, eof, _io_ctx)); } else { RETURN_IF_ERROR(_read_one_message(&json_str_ptr, size)); json_str = json_str_ptr.release(); @@ -1014,7 +1014,7 @@ Status NewJsonReader::_read_one_message(std::unique_ptr* file_buf, si size_t file_size = _file_reader->size(); file_buf->reset(new uint8_t[file_size]); Slice result(file_buf->get(), file_size); - RETURN_IF_ERROR(_file_reader->read_at(_current_offset, result, *_io_ctx, read_size)); + RETURN_IF_ERROR(_file_reader->read_at(_current_offset, result, read_size, _io_ctx)); break; } case TFileType::FILE_STREAM: { @@ -1514,7 +1514,7 @@ Status NewJsonReader::_simdjson_parse_json_doc(size_t* size, bool* eof) { const uint8_t* json_str = nullptr; std::unique_ptr json_str_ptr; if (_line_reader != nullptr) { - RETURN_IF_ERROR(_line_reader->read_line(&json_str, size, eof)); + RETURN_IF_ERROR(_line_reader->read_line(&json_str, size, eof, _io_ctx)); } else { size_t length = 0; RETURN_IF_ERROR(_read_one_message(&json_str_ptr, &length)); diff --git a/be/src/vec/exec/format/json/new_json_reader.h b/be/src/vec/exec/format/json/new_json_reader.h index b67f9602ce..5d9b35a8c2 100644 --- a/be/src/vec/exec/format/json/new_json_reader.h +++ b/be/src/vec/exec/format/json/new_json_reader.h @@ -47,11 +47,11 @@ public: NewJsonReader(RuntimeState* state, RuntimeProfile* profile, ScannerCounter* counter, const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& file_slot_descs, bool* scanner_eof, - IOContext* io_ctx, bool is_dynamic_schema = false); + io::IOContext* io_ctx, bool is_dynamic_schema = false); NewJsonReader(RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& file_slot_descs, - IOContext* io_ctx); + io::IOContext* io_ctx); ~NewJsonReader() override = default; Status init_reader(); @@ -192,7 +192,7 @@ private: size_t _current_offset; - IOContext* _io_ctx; + io::IOContext* _io_ctx; RuntimeProfile::Counter* _bytes_read_counter; RuntimeProfile::Counter* _read_timer; diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index ecfab2412a..a0c2e04768 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -53,11 +53,10 @@ void ORCFileInputStream::read(void* buf, uint64_t length, uint64_t offset) { SCOPED_RAW_TIMER(&_statistics->fs_read_time); uint64_t has_read = 0; char* out = reinterpret_cast(buf); - IOContext io_ctx; while (has_read < length) { size_t loop_read; Slice result(out + has_read, length - has_read); - Status st = _file_reader->read_at(offset + has_read, result, io_ctx, &loop_read); + Status st = _file_reader->read_at(offset + has_read, result, &loop_read, _io_ctx); if (!st.ok()) { throw orc::ParseError( strings::Substitute("Failed to read $0: $1", _file_name, st.to_string())); @@ -75,7 +74,7 @@ void ORCFileInputStream::read(void* buf, uint64_t length, uint64_t offset) { OrcReader::OrcReader(RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& column_names, - size_t batch_size, const std::string& ctz, IOContext* io_ctx) + size_t batch_size, const std::string& ctz, io::IOContext* io_ctx) : _profile(profile), _scan_params(params), _scan_range(range), @@ -94,7 +93,7 @@ OrcReader::OrcReader(RuntimeProfile* profile, const TFileScanRangeParams& params OrcReader::OrcReader(const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& column_names, const std::string& ctz, - IOContext* io_ctx) + io::IOContext* io_ctx) : _profile(nullptr), _scan_params(params), _scan_range(range), @@ -154,11 +153,10 @@ void OrcReader::_init_profile() { Status OrcReader::_create_file_reader() { if (_file_input_stream == nullptr) { io::FileReaderSPtr inner_reader; - RETURN_IF_ERROR(FileFactory::create_file_reader(_profile, _system_properties, - _file_description, &_file_system, - &inner_reader, _io_ctx)); + RETURN_IF_ERROR(FileFactory::create_file_reader( + _profile, _system_properties, _file_description, &_file_system, &inner_reader)); _file_input_stream.reset( - new ORCFileInputStream(_scan_range.path, inner_reader, &_statistics)); + new ORCFileInputStream(_scan_range.path, inner_reader, &_statistics, _io_ctx)); } if (_file_input_stream->getLength() == 0) { return Status::EndOfFile("empty orc file: " + _scan_range.path); diff --git a/be/src/vec/exec/format/orc/vorc_reader.h b/be/src/vec/exec/format/orc/vorc_reader.h index 741addc345..4383129bd8 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.h +++ b/be/src/vec/exec/format/orc/vorc_reader.h @@ -47,11 +47,11 @@ public: OrcReader(RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& column_names, - size_t batch_size, const std::string& ctz, IOContext* io_ctx); + size_t batch_size, const std::string& ctz, io::IOContext* io_ctx); OrcReader(const TFileScanRangeParams& params, const TFileRangeDesc& range, const std::vector& column_names, const std::string& ctz, - IOContext* io_ctx); + io::IOContext* io_ctx); ~OrcReader() override; @@ -277,7 +277,7 @@ private: std::shared_ptr _file_system; - IOContext* _io_ctx; + io::IOContext* _io_ctx; // only for decimal DecimalScaleParams _decimal_scale_params; @@ -286,8 +286,11 @@ private: class ORCFileInputStream : public orc::InputStream { public: ORCFileInputStream(const std::string& file_name, io::FileReaderSPtr file_reader, - OrcReader::Statistics* statistics) - : _file_name(file_name), _file_reader(file_reader), _statistics(statistics) {} + OrcReader::Statistics* statistics, const io::IOContext* io_ctx) + : _file_name(file_name), + _file_reader(file_reader), + _statistics(statistics), + _io_ctx(io_ctx) {} ~ORCFileInputStream() override = default; @@ -304,6 +307,7 @@ private: io::FileReaderSPtr _file_reader; // Owned by OrcReader OrcReader::Statistics* _statistics; + const io::IOContext* _io_ctx; }; } // namespace doris::vectorized diff --git a/be/src/vec/exec/format/parquet/parquet_thrift_util.h b/be/src/vec/exec/format/parquet/parquet_thrift_util.h index 184d4c786a..cbab6ac75d 100644 --- a/be/src/vec/exec/format/parquet/parquet_thrift_util.h +++ b/be/src/vec/exec/format/parquet/parquet_thrift_util.h @@ -24,6 +24,7 @@ #include "common/logging.h" #include "gen_cpp/parquet_types.h" #include "io/fs/file_reader.h" +#include "io/io_common.h" #include "olap/iterators.h" #include "util/coding.h" #include "util/thrift_util.h" @@ -40,8 +41,7 @@ static Status parse_thrift_footer(io::FileReaderSPtr file, int64_t file_size = file->size(); size_t bytes_read = 0; Slice result(footer, PARQUET_FOOTER_SIZE); - IOContext io_ctx; - RETURN_IF_ERROR(file->read_at(file_size - PARQUET_FOOTER_SIZE, result, io_ctx, &bytes_read)); + RETURN_IF_ERROR(file->read_at(file_size - PARQUET_FOOTER_SIZE, result, &bytes_read)); DCHECK_EQ(bytes_read, PARQUET_FOOTER_SIZE); // validate magic @@ -61,8 +61,8 @@ static Status parse_thrift_footer(io::FileReaderSPtr file, // deserialize footer std::unique_ptr meta_buff(new uint8_t[metadata_size]); Slice res(meta_buff.get(), metadata_size); - RETURN_IF_ERROR(file->read_at(file_size - PARQUET_FOOTER_SIZE - metadata_size, res, io_ctx, - &bytes_read)); + RETURN_IF_ERROR( + file->read_at(file_size - PARQUET_FOOTER_SIZE - metadata_size, res, &bytes_read)); DCHECK_EQ(bytes_read, metadata_size); RETURN_IF_ERROR(deserialize_thrift_msg(meta_buff.get(), &metadata_size, true, &t_metadata)); file_metadata.reset(new FileMetaData(t_metadata)); diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp index 9308624246..b74d9c3db0 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp @@ -19,7 +19,7 @@ namespace doris::vectorized { -ColumnChunkReader::ColumnChunkReader(BufferedStreamReader* reader, +ColumnChunkReader::ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, FieldSchema* field_schema, cctz::time_zone* ctz) : _field_schema(field_schema), diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h index 303af52104..6075275a7e 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h @@ -25,7 +25,7 @@ #include "common/status.h" #include "decoder.h" #include "gen_cpp/parquet_types.h" -#include "io/buffered_reader.h" +#include "io/fs/buffered_reader.h" #include "level_decoder.h" #include "schema_desc.h" #include "util/block_compression.h" @@ -64,7 +64,7 @@ public: int64_t decode_level_time = 0; }; - ColumnChunkReader(BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, FieldSchema* field_schema, cctz::time_zone* ctz); ~ColumnChunkReader() = default; @@ -172,7 +172,7 @@ private: level_t _max_def_level; tparquet::LogicalType _parquet_logical_type; - BufferedStreamReader* _stream_reader; + io::BufferedStreamReader* _stream_reader; tparquet::ColumnMetaData _metadata; cctz::time_zone* _ctz; diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp index 0357c77b22..86ce6e30e4 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp @@ -171,8 +171,8 @@ Status ScalarColumnReader::init(io::FileReaderSPtr file, FieldSchema* field, siz ? chunk_meta.dictionary_page_offset : chunk_meta.data_page_offset; size_t chunk_len = chunk_meta.total_compressed_size; - _stream_reader = std::make_unique(file, chunk_start, chunk_len, - std::min(chunk_len, max_buf_size)); + _stream_reader = std::make_unique( + file, chunk_start, chunk_len, std::min(chunk_len, max_buf_size)); _chunk_reader = std::make_unique(_stream_reader.get(), &_chunk_meta, field, _ctz); RETURN_IF_ERROR(_chunk_reader->init()); diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_reader.h index 7f9764ec06..181f93d974 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h @@ -39,7 +39,7 @@ public: decode_level_time(0), decode_null_map_time(0) {} - Statistics(BufferedStreamReader::Statistics& fs, ColumnChunkReader::Statistics& cs, + Statistics(io::BufferedStreamReader::Statistics& fs, ColumnChunkReader::Statistics& cs, int64_t null_map_time) : read_time(fs.read_time), read_calls(fs.read_calls), @@ -147,7 +147,7 @@ public: private: tparquet::ColumnChunk _chunk_meta; - std::unique_ptr _stream_reader; + std::unique_ptr _stream_reader; std::unique_ptr _chunk_reader; std::vector _rep_levels; std::vector _def_levels; @@ -250,4 +250,4 @@ private: std::vector> _child_readers; }; -}; // namespace doris::vectorized \ No newline at end of file +}; // namespace doris::vectorized diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp index f32d571a7a..a74b328dee 100644 --- a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp @@ -26,7 +26,7 @@ namespace doris::vectorized { static constexpr size_t INIT_PAGE_HEADER_SIZE = 128; -PageReader::PageReader(BufferedStreamReader* reader, uint64_t offset, uint64_t length) +PageReader::PageReader(io::BufferedStreamReader* reader, uint64_t offset, uint64_t length) : _reader(reader), _start_offset(offset), _end_offset(offset + length) {} Status PageReader::next_page_header() { diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.h b/be/src/vec/exec/format/parquet/vparquet_page_reader.h index eabbd5ba4a..ab42b45d6b 100644 --- a/be/src/vec/exec/format/parquet/vparquet_page_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.h @@ -19,7 +19,7 @@ #include "common/status.h" #include "gen_cpp/parquet_types.h" -#include "io/buffered_reader.h" +#include "io/fs/buffered_reader.h" namespace doris::vectorized { @@ -32,7 +32,7 @@ public: int64_t decode_header_time = 0; }; - PageReader(BufferedStreamReader* reader, uint64_t offset, uint64_t length); + PageReader(io::BufferedStreamReader* reader, uint64_t offset, uint64_t length); ~PageReader() = default; bool has_next_page() const { return _offset < _end_offset; } @@ -56,7 +56,7 @@ public: private: enum PageReaderState { INITIALIZED, HEADER_PARSED }; - BufferedStreamReader* _reader; + io::BufferedStreamReader* _reader; tparquet::PageHeader _cur_page_header; Statistics _statistics; PageReaderState _state = INITIALIZED; diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp index 7c68f1925a..6da31fee04 100644 --- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp @@ -34,7 +34,7 @@ namespace doris::vectorized { ParquetReader::ParquetReader(RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, size_t batch_size, cctz::time_zone* ctz, - IOContext* io_ctx, RuntimeState* state) + io::IOContext* io_ctx, RuntimeState* state) : _profile(profile), _scan_params(params), _scan_range(range), @@ -50,7 +50,7 @@ ParquetReader::ParquetReader(RuntimeProfile* profile, const TFileScanRangeParams } ParquetReader::ParquetReader(const TFileScanRangeParams& params, const TFileRangeDesc& range, - IOContext* io_ctx, RuntimeState* state) + io::IOContext* io_ctx, RuntimeState* state) : _profile(nullptr), _scan_params(params), _scan_range(range), @@ -154,9 +154,8 @@ void ParquetReader::close() { Status ParquetReader::_open_file() { if (_file_reader == nullptr) { - RETURN_IF_ERROR(FileFactory::create_file_reader(_profile, _system_properties, - _file_description, &_file_system, - &_file_reader, _io_ctx)); + RETURN_IF_ERROR(FileFactory::create_file_reader( + _profile, _system_properties, _file_description, &_file_system, &_file_reader)); } if (_file_metadata == nullptr) { if (_file_reader->size() == 0) { @@ -576,15 +575,14 @@ Status ParquetReader::_process_page_index(const tparquet::RowGroup& row_group, uint8_t col_index_buff[page_index._column_index_size]; size_t bytes_read = 0; Slice result(col_index_buff, page_index._column_index_size); - IOContext io_ctx; RETURN_IF_ERROR( - _file_reader->read_at(page_index._column_index_start, result, io_ctx, &bytes_read)); + _file_reader->read_at(page_index._column_index_start, result, &bytes_read, _io_ctx)); auto& schema_desc = _file_metadata->schema(); std::vector skipped_row_ranges; uint8_t off_index_buff[page_index._offset_index_size]; Slice res(off_index_buff, page_index._offset_index_size); RETURN_IF_ERROR( - _file_reader->read_at(page_index._offset_index_start, res, io_ctx, &bytes_read)); + _file_reader->read_at(page_index._offset_index_start, res, &bytes_read, _io_ctx)); for (auto& read_col : _read_columns) { auto conjunct_iter = _colname_to_value_range->find(read_col._file_slot_name); if (_colname_to_value_range->end() == conjunct_iter) { diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.h b/be/src/vec/exec/format/parquet/vparquet_reader.h index f21996a1df..26e8ccda1f 100644 --- a/be/src/vec/exec/format/parquet/vparquet_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_reader.h @@ -58,10 +58,10 @@ public: ParquetReader(RuntimeProfile* profile, const TFileScanRangeParams& params, const TFileRangeDesc& range, size_t batch_size, cctz::time_zone* ctz, - IOContext* io_ctx, RuntimeState* state); + io::IOContext* io_ctx, RuntimeState* state); ParquetReader(const TFileScanRangeParams& params, const TFileRangeDesc& range, - IOContext* io_ctx, RuntimeState* state); + io::IOContext* io_ctx, RuntimeState* state); ~ParquetReader() override; // for test @@ -208,7 +208,7 @@ private: ParquetColumnReader::Statistics _column_statistics; ParquetProfile _parquet_profile; bool _closed = false; - IOContext* _io_ctx; + io::IOContext* _io_ctx; RuntimeState* _state; const TupleDescriptor* _tuple_descriptor; const RowDescriptor* _row_descriptor; diff --git a/be/src/vec/exec/format/table/iceberg_reader.cpp b/be/src/vec/exec/format/table/iceberg_reader.cpp index 353b3bb198..9c03cb78d4 100644 --- a/be/src/vec/exec/format/table/iceberg_reader.cpp +++ b/be/src/vec/exec/format/table/iceberg_reader.cpp @@ -39,7 +39,7 @@ const std::string ICEBERG_FILE_PATH = "file_path"; IcebergTableReader::IcebergTableReader(GenericReader* file_format_reader, RuntimeProfile* profile, RuntimeState* state, const TFileScanRangeParams& params, const TFileRangeDesc& range, KVCache& kv_cache, - IOContext* io_ctx) + io::IOContext* io_ctx) : TableFormatReader(file_format_reader), _profile(profile), _state(state), diff --git a/be/src/vec/exec/format/table/iceberg_reader.h b/be/src/vec/exec/format/table/iceberg_reader.h index 3e57eda86b..26dc630cac 100644 --- a/be/src/vec/exec/format/table/iceberg_reader.h +++ b/be/src/vec/exec/format/table/iceberg_reader.h @@ -19,6 +19,7 @@ #include +#include "io/io_common.h" #include "table_format_reader.h" #include "vec/columns/column_dictionary.h" #include "vec/exec/format/format_common.h" @@ -28,8 +29,6 @@ namespace doris { -struct IOContext; - namespace vectorized { class IcebergTableReader : public TableFormatReader { @@ -42,7 +41,7 @@ public: IcebergTableReader(GenericReader* file_format_reader, RuntimeProfile* profile, RuntimeState* state, const TFileScanRangeParams& params, const TFileRangeDesc& range, KVCache& kv_cache, - IOContext* io_ctx); + io::IOContext* io_ctx); ~IcebergTableReader() override = default; Status init_row_filters(const TFileRangeDesc& range) override; @@ -121,7 +120,7 @@ private: // col names in table but not in parquet file std::vector _not_in_file_col_names; - IOContext* _io_ctx; + io::IOContext* _io_ctx; bool _has_schema_change = false; bool _has_iceberg_schema = false; }; diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index 7ab2f5bc50..67b70465fb 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -76,8 +76,8 @@ Status VFileScanner::prepare( _convert_to_output_block_timer = ADD_TIMER(_parent->_scanner_profile, "FileScannerConvertOuputBlockTime"); - _file_cache_statistics.reset(new FileCacheStatistics()); - _io_ctx.reset(new IOContext()); + _file_cache_statistics.reset(new io::FileCacheStatistics()); + _io_ctx.reset(new io::IOContext()); _io_ctx->file_cache_stats = _file_cache_statistics.get(); _io_ctx->query_id = &_state->query_id(); _io_ctx->enable_file_cache = _state->query_options().enable_file_cache; diff --git a/be/src/vec/exec/scan/vfile_scanner.h b/be/src/vec/exec/scan/vfile_scanner.h index 410d357d18..d4c6f0b33f 100644 --- a/be/src/vec/exec/scan/vfile_scanner.h +++ b/be/src/vec/exec/scan/vfile_scanner.h @@ -119,8 +119,8 @@ protected: // for tracing dynamic schema std::unique_ptr _full_base_schema_view; - std::unique_ptr _file_cache_statistics; - std::unique_ptr _io_ctx; + std::unique_ptr _file_cache_statistics; + std::unique_ptr _io_ctx; private: RuntimeProfile::Counter* _get_block_timer = nullptr; diff --git a/be/src/vec/exec/varrow_scanner.cpp b/be/src/vec/exec/varrow_scanner.cpp index 10ab22858e..388a0e78ce 100644 --- a/be/src/vec/exec/varrow_scanner.cpp +++ b/be/src/vec/exec/varrow_scanner.cpp @@ -81,10 +81,8 @@ Status VArrowScanner::_open_next_reader() { _init_system_properties(range); _init_file_description(range); // no use - doris::IOContext io_ctx; - RETURN_IF_ERROR(FileFactory::create_file_reader(_profile, _system_properties, - _file_description, &_file_system, - &file_reader, &io_ctx)); + RETURN_IF_ERROR(FileFactory::create_file_reader( + _profile, _system_properties, _file_description, &_file_system, &file_reader)); if (file_reader->size() == 0) { continue; diff --git a/be/src/vec/olap/vertical_merge_iterator.h b/be/src/vec/olap/vertical_merge_iterator.h index a45b983288..4fb3332892 100644 --- a/be/src/vec/olap/vertical_merge_iterator.h +++ b/be/src/vec/olap/vertical_merge_iterator.h @@ -16,6 +16,7 @@ // under the License. #include "common/status.h" +#include "io/io_common.h" #include "olap/iterators.h" #include "olap/schema.h" #include "vec/columns/columns_number.h" diff --git a/be/src/vec/runtime/vfile_result_writer.cpp b/be/src/vec/runtime/vfile_result_writer.cpp index 16d48bde08..5a109fa711 100644 --- a/be/src/vec/runtime/vfile_result_writer.cpp +++ b/be/src/vec/runtime/vfile_result_writer.cpp @@ -22,7 +22,7 @@ #include "gutil/strings/numbers.h" #include "gutil/strings/substitute.h" #include "io/file_factory.h" -#include "io/file_writer.h" +#include "io/fs/file_writer.h" #include "runtime/buffer_control_block.h" #include "runtime/descriptors.h" #include "runtime/large_int_value.h" @@ -78,9 +78,13 @@ void VFileResultWriter::_init_profile() { Status VFileResultWriter::_create_success_file() { std::string file_name; RETURN_IF_ERROR(_get_success_file_name(&file_name)); - RETURN_IF_ERROR(_create_file_writer(file_name)); - // set only close to true to avoid dead loop - return _close_file_writer(true, true); + RETURN_IF_ERROR(FileFactory::create_file_writer( + FileFactory::convert_storage_type(_storage_type), _state->exec_env(), + _file_opts->broker_addresses, _file_opts->broker_properties, file_name, 0, + _file_writer_impl)); + // must write somthing because s3 file writer can not writer empty file + RETURN_IF_ERROR(_file_writer_impl->append({"success"})); + return _file_writer_impl->close(); } Status VFileResultWriter::_get_success_file_name(std::string* file_name) { @@ -113,7 +117,6 @@ Status VFileResultWriter::_create_file_writer(const std::string& file_name) { FileFactory::convert_storage_type(_storage_type), _state->exec_env(), _file_opts->broker_addresses, _file_opts->broker_properties, file_name, 0, _file_writer_impl)); - RETURN_IF_ERROR(_file_writer_impl->open()); switch (_file_opts->file_format) { case TFileFormatType::FORMAT_CSV_PLAIN: // just use file writer is enough @@ -381,10 +384,7 @@ Status VFileResultWriter::write_csv_header() { if (_header_type == BeConsts::CSV_WITH_NAMES_AND_TYPES) { tmp_header += gen_types(); } - size_t written_len = 0; - RETURN_IF_ERROR( - _file_writer_impl->write(reinterpret_cast(tmp_header.c_str()), - tmp_header.size(), &written_len)); + RETURN_IF_ERROR(_file_writer_impl->append(tmp_header)); _header_sent = true; } return Status::OK(); @@ -398,9 +398,8 @@ Status VFileResultWriter::_flush_plain_text_outstream(bool eos) { } const std::string& buf = _plain_text_outstream.str(); - size_t written_len = 0; - RETURN_IF_ERROR(_file_writer_impl->write(reinterpret_cast(buf.c_str()), - buf.size(), &written_len)); + size_t written_len = buf.size(); + RETURN_IF_ERROR(_file_writer_impl->append(buf)); COUNTER_UPDATE(_written_data_bytes, written_len); _current_written_bytes += written_len; @@ -426,7 +425,7 @@ Status VFileResultWriter::_create_new_file_if_exceed_size() { return Status::OK(); } -Status VFileResultWriter::_close_file_writer(bool done, bool only_close) { +Status VFileResultWriter::_close_file_writer(bool done) { if (_vfile_writer) { _vfile_writer->close(); COUNTER_UPDATE(_written_data_bytes, _current_written_bytes); @@ -435,10 +434,6 @@ Status VFileResultWriter::_close_file_writer(bool done, bool only_close) { _file_writer_impl->close(); } - if (only_close) { - return Status::OK(); - } - if (!done) { // not finished, create new file writer for next file RETURN_IF_ERROR(_create_next_file_writer()); diff --git a/be/src/vec/runtime/vfile_result_writer.h b/be/src/vec/runtime/vfile_result_writer.h index 0c305ef4dd..6243244eb4 100644 --- a/be/src/vec/runtime/vfile_result_writer.h +++ b/be/src/vec/runtime/vfile_result_writer.h @@ -17,7 +17,7 @@ #pragma once -#include "io/file_writer.h" +#include "io/fs/file_writer.h" #include "vec/runtime/vparquet_writer.h" #include "vec/sink/vresult_sink.h" @@ -65,8 +65,7 @@ private: Status _get_file_url(std::string* file_url); std::string _file_format_to_name(); // close file writer, and if !done, it will create new writer for next file. - // if only_close is true, this method will just close the file writer and return. - Status _close_file_writer(bool done, bool only_close = false); + Status _close_file_writer(bool done); // create a new file if current file size exceed limit Status _create_new_file_if_exceed_size(); // send the final statistic result @@ -82,7 +81,7 @@ private: // If the result file format is plain text, like CSV, this _file_writer is owned by this FileResultWriter. // If the result file format is Parquet, this _file_writer is owned by _parquet_writer. - std::unique_ptr _file_writer_impl; + std::unique_ptr _file_writer_impl; // Used to buffer the export data of plain text // TODO(cmy): I simply use a stringstrteam to buffer the data, to avoid calling // file writer's write() for every single row. diff --git a/be/src/vec/runtime/vorc_writer.cpp b/be/src/vec/runtime/vorc_writer.cpp index 3ea250a65c..d6809b026e 100644 --- a/be/src/vec/runtime/vorc_writer.cpp +++ b/be/src/vec/runtime/vorc_writer.cpp @@ -17,7 +17,7 @@ #include "vec/runtime/vorc_writer.h" -#include "io/file_writer.h" +#include "io/fs/file_writer.h" #include "vec/columns/column_complex.h" #include "vec/columns/column_nullable.h" #include "vec/columns/column_string.h" @@ -28,7 +28,7 @@ #include "vec/functions/function_helpers.h" namespace doris::vectorized { -VOrcOutputStream::VOrcOutputStream(doris::FileWriter* file_writer) +VOrcOutputStream::VOrcOutputStream(doris::io::FileWriter* file_writer) : _file_writer(file_writer), _cur_pos(0), _written_len(0), _name("VOrcOutputStream") {} VOrcOutputStream::~VOrcOutputStream() { @@ -49,14 +49,13 @@ void VOrcOutputStream::close() { void VOrcOutputStream::write(const void* data, size_t length) { if (!_is_closed) { - size_t written_len = 0; - Status st = _file_writer->write(static_cast(data), length, &written_len); + Status st = _file_writer->append({static_cast(data), length}); if (!st.ok()) { LOG(WARNING) << "Write to ORC file failed: " << st; return; } - _cur_pos += written_len; - _written_len += written_len; + _cur_pos += length; + _written_len += length; } } @@ -64,7 +63,7 @@ void VOrcOutputStream::set_written_len(int64_t written_len) { _written_len = written_len; } -VOrcWriterWrapper::VOrcWriterWrapper(doris::FileWriter* file_writer, +VOrcWriterWrapper::VOrcWriterWrapper(doris::io::FileWriter* file_writer, const std::vector& output_vexpr_ctxs, const std::string& schema, bool output_object_data) : VFileWriterWrapper(output_vexpr_ctxs, output_object_data), diff --git a/be/src/vec/runtime/vorc_writer.h b/be/src/vec/runtime/vorc_writer.h index 284e8b7887..ef939ed5e8 100644 --- a/be/src/vec/runtime/vorc_writer.h +++ b/be/src/vec/runtime/vorc_writer.h @@ -24,17 +24,16 @@ #include #include "common/status.h" -#include "io/file_writer.h" +#include "io/fs/file_writer.h" #include "vec/core/block.h" #include "vec/exprs/vexpr_context.h" #include "vec/runtime/vfile_result_writer.h" namespace doris::vectorized { -class FileWriter; class VOrcOutputStream : public orc::OutputStream { public: - VOrcOutputStream(doris::FileWriter* file_writer); + VOrcOutputStream(doris::io::FileWriter* file_writer); ~VOrcOutputStream() override; @@ -51,8 +50,8 @@ public: void set_written_len(int64_t written_len); private: - doris::FileWriter* _file_writer; // not owned - int64_t _cur_pos = 0; // current write position + doris::io::FileWriter* _file_writer; // not owned + int64_t _cur_pos = 0; // current write position bool _is_closed = false; int64_t _written_len = 0; const std::string _name; @@ -61,7 +60,7 @@ private: // a wrapper of parquet output stream class VOrcWriterWrapper final : public VFileWriterWrapper { public: - VOrcWriterWrapper(doris::FileWriter* file_writer, + VOrcWriterWrapper(doris::io::FileWriter* file_writer, const std::vector& output_vexpr_ctxs, const std::string& schema, bool output_object_data); @@ -78,7 +77,7 @@ public: private: std::unique_ptr _create_row_batch(size_t sz); - doris::FileWriter* _file_writer; + doris::io::FileWriter* _file_writer; std::unique_ptr _output_stream; std::unique_ptr _write_options; const std::string& _schema_str; diff --git a/be/src/vec/runtime/vparquet_writer.cpp b/be/src/vec/runtime/vparquet_writer.cpp index 3431e0ee7a..146d5f9cc7 100644 --- a/be/src/vec/runtime/vparquet_writer.cpp +++ b/be/src/vec/runtime/vparquet_writer.cpp @@ -21,7 +21,7 @@ #include #include -#include "io/file_writer.h" +#include "io/fs/file_writer.h" #include "util/mysql_global.h" #include "util/types.h" #include "vec/columns/column_complex.h" @@ -35,7 +35,7 @@ namespace doris::vectorized { -ParquetOutputStream::ParquetOutputStream(FileWriter* file_writer) +ParquetOutputStream::ParquetOutputStream(doris::io::FileWriter* file_writer) : _file_writer(file_writer), _cur_pos(0), _written_len(0) { set_mode(arrow::io::FileMode::WRITE); } @@ -51,8 +51,8 @@ arrow::Status ParquetOutputStream::Write(const void* data, int64_t nbytes) { if (_is_closed) { return arrow::Status::OK(); } - size_t written_len = 0; - Status st = _file_writer->write(static_cast(data), nbytes, &written_len); + size_t written_len = nbytes; + Status st = _file_writer->append({static_cast(data), written_len}); if (!st.ok()) { return arrow::Status::IOError(st.to_string()); } @@ -204,7 +204,7 @@ void ParquetBuildHelper::build_version(parquet::WriterProperties::Builder& build } } -VParquetWriterWrapper::VParquetWriterWrapper(doris::FileWriter* file_writer, +VParquetWriterWrapper::VParquetWriterWrapper(doris::io::FileWriter* file_writer, const std::vector& output_vexpr_ctxs, const std::vector& parquet_schemas, const TParquetCompressionType::type& compression_type, diff --git a/be/src/vec/runtime/vparquet_writer.h b/be/src/vec/runtime/vparquet_writer.h index 5537885a11..796fbbf983 100644 --- a/be/src/vec/runtime/vparquet_writer.h +++ b/be/src/vec/runtime/vparquet_writer.h @@ -42,8 +42,8 @@ namespace doris::vectorized { class ParquetOutputStream : public arrow::io::OutputStream { public: - ParquetOutputStream(FileWriter* file_writer); - ParquetOutputStream(FileWriter* file_writer, const int64_t& written_len); + ParquetOutputStream(doris::io::FileWriter* file_writer); + ParquetOutputStream(doris::io::FileWriter* file_writer, const int64_t& written_len); ~ParquetOutputStream() override; arrow::Status Write(const void* data, int64_t nbytes) override; @@ -58,8 +58,8 @@ public: void set_written_len(int64_t written_len); private: - FileWriter* _file_writer; // not owned - int64_t _cur_pos = 0; // current write position + doris::io::FileWriter* _file_writer; // not owned + int64_t _cur_pos = 0; // current write position bool _is_closed = false; int64_t _written_len = 0; }; @@ -106,7 +106,7 @@ protected: // a wrapper of parquet output stream class VParquetWriterWrapper final : public VFileWriterWrapper { public: - VParquetWriterWrapper(doris::FileWriter* file_writer, + VParquetWriterWrapper(doris::io::FileWriter* file_writer, const std::vector& output_vexpr_ctxs, const std::vector& parquet_schemas, const TParquetCompressionType::type& compression_type, diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index 3ab5067e89..9dc13eb400 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -72,6 +72,7 @@ set(HTTP_TEST_FILES set(IO_TEST_FILES io/cache/remote_file_cache_test.cpp io/cache/file_block_cache_test.cpp + io/fs/file_system_test.cpp ) set(OLAP_TEST_FILES olap/engine_storage_migration_task_test.cpp @@ -82,7 +83,7 @@ set(OLAP_TEST_FILES olap/lru_cache_test.cpp olap/bloom_filter_test.cpp olap/itoken_extractor_test.cpp - olap/file_helper_test.cpp + olap/file_header_test.cpp olap/file_utils_test.cpp olap/cumulative_compaction_policy_test.cpp #olap/row_cursor_test.cpp @@ -208,13 +209,10 @@ set(UTIL_TEST_FILES util/http_channel_test.cpp util/histogram_test.cpp util/s3_uri_test.cpp - util/s3_storage_backend_test.cpp - util/broker_storage_backend_test.cpp util/sort_heap_test.cpp util/counts_test.cpp util/date_func_test.cpp util/quantile_state_test.cpp - util/hdfs_storage_backend_test.cpp util/interval_tree_test.cpp util/key_util_test.cpp ) diff --git a/be/test/io/cache/remote_file_cache_test.cpp b/be/test/io/cache/remote_file_cache_test.cpp index 74aa73ede7..121c8b05ae 100644 --- a/be/test/io/cache/remote_file_cache_test.cpp +++ b/be/test/io/cache/remote_file_cache_test.cpp @@ -164,7 +164,11 @@ protected: // just use to create s3 filesystem, otherwise won't use cache S3Conf s3_conf; - auto fs = io::S3FileSystem::create(std::move(s3_conf), resource_id); + std::shared_ptr fs; + Status st = io::S3FileSystem::create(std::move(s3_conf), resource_id, &fs); + // io::S3FileSystem::create will call connect, which will fail because s3_conf is empty. + // but it does affect the following unit test + ASSERT_FALSE(st.ok()) << st; rowset.rowset_meta()->set_resource_id(resource_id); rowset.rowset_meta()->set_num_segments(1); rowset.rowset_meta()->set_fs(fs); @@ -172,7 +176,7 @@ protected: rowset.rowset_meta()->set_rowset_id(rowset_id); std::vector segments; - Status st = rowset.load_segments(&segments); + st = rowset.load_segments(&segments); ASSERT_TRUE(st.ok()) << st; } }; diff --git a/be/test/io/fs/file_system_test.cpp b/be/test/io/fs/file_system_test.cpp new file mode 100644 index 0000000000..0c5ac73f0f --- /dev/null +++ b/be/test/io/fs/file_system_test.cpp @@ -0,0 +1,643 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "io/fs/broker_file_system.h" +#include "io/fs/file_reader.h" +#include "io/fs/file_writer.h" +#include "io/fs/hdfs_file_system.h" +#include "io/fs/local_file_system.h" +#include "io/fs/s3_file_system.h" +#include "io/hdfs_builder.h" +#include "util/s3_uri.h" + +namespace doris { + +#ifndef CHECK_STATUS_OK +#define CHECK_STATUS_OK(stmt) \ + do { \ + Status _status_ = (stmt); \ + ASSERT_TRUE(_status_.ok()) << _status_; \ + } while (false) +#endif + +// set your own info +// s3 +static std::string ak = ""; +static std::string sk = ""; +static std::string endpoint = "http://cos.ap-beijing.myqcloud.com"; +static std::string region = "ap-beijing"; +static std::string s3_location = ""; + +// hdfs +static std::string fs_name = "hdfs://my_nameservice"; +static std::string username = "hadoop"; +static std::string nameservices = "my_nameservice"; +static std::string nn = "nn1,nn2"; +static std::string rpc1 = "172.21.0.1:4007"; +static std::string rpc2 = "172.21.0.2:4007"; +static std::string provider = + "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"; +static std::string hdfs_location = "/user/doris/"; + +// broker +static std::string broker_ip = "127.0.0.1"; +static int broker_port = 8008; +static std::string broker_location = "hdfs://my_nameservice/user/doris"; + +// commend out to enable specified test +#define TestHdfsFileSystem DISABLED_TestHdfsFileSystem +#define TestS3FileSystem DISABLED_TestS3FileSystem +#define TestBrokerFileSystem DISABLED_TestBrokerFileSystem + +class FileSystemTest : public testing::Test { +public: + virtual void SetUp() { + s3_prop.emplace("AWS_ACCESS_KEY", ak); + s3_prop.emplace("AWS_SECRET_KEY", sk); + s3_prop.emplace("AWS_ENDPOINT", endpoint); + s3_prop.emplace("AWS_REGION", region); + + hdfs_prop.emplace("fs.defaultFS", fs_name); + hdfs_prop.emplace("hadoop.username", username); + hdfs_prop.emplace("username", username); // for broker hdfs + hdfs_prop.emplace("dfs.nameservices", nameservices); + hdfs_prop.emplace("dfs.ha.namenodes." + nameservices, nn); + hdfs_prop.emplace("dfs.namenode.rpc-address." + nameservices + ".nn1", rpc1); + hdfs_prop.emplace("dfs.namenode.rpc-address." + nameservices + ".nn2", rpc2); + hdfs_prop.emplace("dfs.client.failover.proxy.provider." + nameservices, provider); + + broker_addr.__set_hostname(broker_ip); + broker_addr.__set_port(broker_port); + } + + virtual void TearDown() {} + +private: + std::map s3_prop; + std::map hdfs_prop; + TNetworkAddress broker_addr; +}; + +TEST_F(FileSystemTest, TestBrokerFileSystem) { + std::shared_ptr fs; + CHECK_STATUS_OK(io::BrokerFileSystem::create(broker_addr, hdfs_prop, 0, &fs)); + + // delete directory + io::Path delete_path = broker_location + "/tmp1"; + CHECK_STATUS_OK(fs->delete_directory(delete_path)); + io::Path delete_path2 = broker_location + "/tmp2"; + CHECK_STATUS_OK(fs->delete_directory(delete_path2)); + // create directory not implemented + // io::Path create_path = delete_path; + // CHECK_STATUS_OK(fs->create_directory(create_path)); + // write file + std::string file1 = broker_location + "/tmp1/file1.txt"; + io::FileWriterPtr writer; + CHECK_STATUS_OK(fs->create_file(file1, &writer)); + CHECK_STATUS_OK(writer->append({"content"})); + CHECK_STATUS_OK(writer->close()); + // read file + io::FileReaderSPtr reader; + CHECK_STATUS_OK(fs->open_file(file1, &reader)); + char read_buf[10]; + size_t bytes_read = 0; + CHECK_STATUS_OK(reader->read_at(0, {read_buf, 10}, &bytes_read)); + ASSERT_EQ(7, bytes_read); + + // exist + bool exists = false; + CHECK_STATUS_OK(fs->exists(file1, &exists)); + ASSERT_TRUE(exists); + std::string file_non_exist = broker_location + "/non-exist"; + CHECK_STATUS_OK(fs->exists(file_non_exist, &exists)); + ASSERT_FALSE(exists); + CHECK_STATUS_OK(fs->exists(delete_path, &exists)); + ASSERT_TRUE(exists); + + // file size + size_t file_size = 0; + CHECK_STATUS_OK(fs->file_size(file1, &file_size)); + // file size is not implemented + ASSERT_EQ(0, file_size); + + // write more files + for (int i = 0; i < 10; i++) { + std::string tmp_file = fmt::format(broker_location + "/tmp1/tmp_file_{}", i); + io::FileWriterPtr writer; + CHECK_STATUS_OK(fs->create_file(tmp_file, &writer)); + CHECK_STATUS_OK(writer->append({"content"})); + CHECK_STATUS_OK(writer->close()); + } + + // list files + std::vector files; + CHECK_STATUS_OK(fs->list(delete_path, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(11, files.size()); + for (auto& file_info : files) { + std::cout << "file name: " << file_info.file_name << std::endl; + ASSERT_EQ(7, file_info.file_size); + ASSERT_TRUE(file_info.is_file); + } + std::string non_exist_path = broker_location + "/non_exist/"; + files.clear(); + CHECK_STATUS_OK(fs->list(non_exist_path, true, &files, &exists)); + ASSERT_FALSE(exists); + ASSERT_EQ(0, files.size()); + + // rename + std::string src_name = file1; + std::string dst_name = broker_location + "/tmp1/new_file1.txt"; + CHECK_STATUS_OK(fs->rename(src_name, dst_name)); + CHECK_STATUS_OK(fs->exists(src_name, &exists)); + ASSERT_FALSE(exists); + CHECK_STATUS_OK(fs->exists(dst_name, &exists)); + ASSERT_TRUE(exists); + + // rename dir + std::string src_dir = delete_path; + std::string dst_dir = broker_location + "/tmp2"; + CHECK_STATUS_OK(fs->rename_dir(src_dir, dst_dir)); + CHECK_STATUS_OK(fs->exists(dst_name, &exists)); + ASSERT_FALSE(exists); + std::string new_dst_name = dst_dir + "/new_file1.txt"; + CHECK_STATUS_OK(fs->exists(new_dst_name, &exists)); + ASSERT_TRUE(exists); + + // batch delete + std::vector delete_files; + for (int i = 0; i < 10; i++) { + std::string tmp_file = fmt::format(broker_location + "/tmp2/tmp_file_{}", i); + delete_files.emplace_back(tmp_file); + CHECK_STATUS_OK(fs->batch_delete(delete_files)); + } + + // list to check + files.clear(); + CHECK_STATUS_OK(fs->list(dst_dir, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(1, files.size()); + + // upload + std::string upload_file = "./be/test/io/fs/test_data/upload_file.txt"; + std::string remote_file = dst_dir + "/upload_file.txt"; + CHECK_STATUS_OK(fs->upload(upload_file, remote_file)); + exists = false; + CHECK_STATUS_OK(fs->exists(remote_file, &exists)); + ASSERT_TRUE(exists); + + // batch upload + std::vector local_files; + std::vector remote_files; + for (int i = 0; i < 10; i++) { + local_files.push_back(upload_file); + remote_files.push_back(fmt::format(dst_dir + "/upload_file_{}", i)); + } + CHECK_STATUS_OK(fs->batch_upload(local_files, remote_files)); + files.clear(); + CHECK_STATUS_OK(fs->list(dst_dir, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(12, files.size()); + for (auto& file : remote_files) { + CHECK_STATUS_OK(fs->file_size(file, &file_size)); + // file size is not implemented + ASSERT_EQ(0, file_size); + } + + // direct_upload + std::string direct_remote_file = dst_dir + "/direct_upload_file"; + CHECK_STATUS_OK(fs->direct_upload(direct_remote_file, "abc")); + files.clear(); + CHECK_STATUS_OK(fs->list(dst_dir, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(13, files.size()); + CHECK_STATUS_OK(fs->file_size(direct_remote_file, &file_size)); + ASSERT_EQ(0, file_size); + + // download + std::string local_download_file = "./be/test/io/fs/test_data/download_file.txt"; + CHECK_STATUS_OK(io::global_local_filesystem()->delete_file(local_download_file)); + CHECK_STATUS_OK(fs->download(direct_remote_file, local_download_file)); + CHECK_STATUS_OK(io::global_local_filesystem()->file_size(local_download_file, &file_size)); + ASSERT_EQ(3, file_size); + + // direct download + std::string download_content; + CHECK_STATUS_OK(fs->direct_download(direct_remote_file, &download_content)); + ASSERT_EQ("abc", download_content); +} + +TEST_F(FileSystemTest, TestHdfsFileSystem) { + THdfsParams hdfs_params = parse_properties(hdfs_prop); + std::shared_ptr fs; + CHECK_STATUS_OK(io::HdfsFileSystem::create(hdfs_params, hdfs_location, &fs)); + + // delete directory + io::Path delete_path = "tmp1"; + CHECK_STATUS_OK(fs->delete_directory(delete_path)); + io::Path delete_path2 = "tmp2"; + CHECK_STATUS_OK(fs->delete_directory(delete_path2)); + // create directory + io::Path create_path = delete_path; + CHECK_STATUS_OK(fs->create_directory(create_path)); + // write file + std::string file1 = "tmp1/file1.txt"; + io::FileWriterPtr writer; + CHECK_STATUS_OK(fs->create_file(file1, &writer)); + CHECK_STATUS_OK(writer->append({"content"})); + CHECK_STATUS_OK(writer->close()); + // read file + io::FileReaderSPtr reader; + CHECK_STATUS_OK(fs->open_file(file1, &reader)); + char read_buf[10]; + size_t bytes_read = 0; + CHECK_STATUS_OK(reader->read_at(0, {read_buf, 10}, &bytes_read)); + ASSERT_EQ(7, bytes_read); + + // exist + bool exists = false; + CHECK_STATUS_OK(fs->exists(file1, &exists)); + ASSERT_TRUE(exists); + std::string file_non_exist = "non-exist"; + CHECK_STATUS_OK(fs->exists(file_non_exist, &exists)); + ASSERT_FALSE(exists); + CHECK_STATUS_OK(fs->exists(delete_path, &exists)); + ASSERT_TRUE(exists); + + // file size + size_t file_size = 0; + CHECK_STATUS_OK(fs->file_size(file1, &file_size)); + ASSERT_EQ(7, file_size); + + // write more files + for (int i = 0; i < 10; i++) { + std::string tmp_file = fmt::format("tmp1/tmp_file_{}", i); + io::FileWriterPtr writer; + CHECK_STATUS_OK(fs->create_file(tmp_file, &writer)); + CHECK_STATUS_OK(writer->append({"content"})); + CHECK_STATUS_OK(writer->close()); + } + + // list files + std::vector files; + CHECK_STATUS_OK(fs->list(delete_path, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(11, files.size()); + for (auto& file_info : files) { + std::cout << "file name: " << file_info.file_name << std::endl; + ASSERT_EQ(7, file_info.file_size); + ASSERT_TRUE(file_info.is_file); + } + std::string non_exist_path = "non_exist/"; + files.clear(); + CHECK_STATUS_OK(fs->list(non_exist_path, true, &files, &exists)); + ASSERT_FALSE(exists); + ASSERT_EQ(0, files.size()); + + // rename + std::string src_name = file1; + std::string dst_name = "tmp1/new_file1.txt"; + CHECK_STATUS_OK(fs->rename(src_name, dst_name)); + CHECK_STATUS_OK(fs->exists(src_name, &exists)); + ASSERT_FALSE(exists); + CHECK_STATUS_OK(fs->exists(dst_name, &exists)); + ASSERT_TRUE(exists); + + // rename dir + std::string src_dir = delete_path; + std::string dst_dir = "tmp2"; + CHECK_STATUS_OK(fs->rename_dir(src_dir, dst_dir)); + CHECK_STATUS_OK(fs->exists(dst_name, &exists)); + ASSERT_FALSE(exists); + std::string new_dst_name = dst_dir + "/new_file1.txt"; + CHECK_STATUS_OK(fs->exists(new_dst_name, &exists)); + ASSERT_TRUE(exists); + + // batch delete + std::vector delete_files; + for (int i = 0; i < 10; i++) { + std::string tmp_file = fmt::format("tmp2/tmp_file_{}", i); + delete_files.emplace_back(tmp_file); + CHECK_STATUS_OK(fs->batch_delete(delete_files)); + } + + // list to check + files.clear(); + CHECK_STATUS_OK(fs->list(dst_dir, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(1, files.size()); + + // upload + std::string upload_file = "./be/test/io/fs/test_data/upload_file.txt"; + std::string remote_file = dst_dir + "/upload_file.txt"; + CHECK_STATUS_OK(fs->upload(upload_file, remote_file)); + exists = false; + CHECK_STATUS_OK(fs->exists(remote_file, &exists)); + ASSERT_TRUE(exists); + + // batch upload + std::vector local_files; + std::vector remote_files; + for (int i = 0; i < 10; i++) { + local_files.push_back(upload_file); + remote_files.push_back(fmt::format(dst_dir + "/upload_file_{}", i)); + } + CHECK_STATUS_OK(fs->batch_upload(local_files, remote_files)); + files.clear(); + CHECK_STATUS_OK(fs->list(dst_dir, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(12, files.size()); + for (auto& file : remote_files) { + CHECK_STATUS_OK(fs->file_size(file, &file_size)); + ASSERT_EQ(11, file_size); + } + + // direct_upload + std::string direct_remote_file = dst_dir + "/direct_upload_file"; + CHECK_STATUS_OK(fs->direct_upload(direct_remote_file, "abc")); + files.clear(); + CHECK_STATUS_OK(fs->list(dst_dir, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(13, files.size()); + CHECK_STATUS_OK(fs->file_size(direct_remote_file, &file_size)); + ASSERT_EQ(3, file_size); + + // download + std::string local_download_file = "./be/test/io/fs/test_data/download_file.txt"; + CHECK_STATUS_OK(io::global_local_filesystem()->delete_file(local_download_file)); + CHECK_STATUS_OK(fs->download(direct_remote_file, local_download_file)); + CHECK_STATUS_OK(io::global_local_filesystem()->file_size(local_download_file, &file_size)); + ASSERT_EQ(3, file_size); + + // direct download + std::string download_content; + CHECK_STATUS_OK(fs->direct_download(direct_remote_file, &download_content)); + ASSERT_EQ("abc", download_content); +} + +TEST_F(FileSystemTest, TestLocalFileSystem) { + std::shared_ptr fs = io::LocalFileSystem::create("./"); + // delete directory + io::Path delete_path = "tmp1"; + CHECK_STATUS_OK(fs->delete_directory(delete_path)); + io::Path delete_path2 = "tmp2"; + CHECK_STATUS_OK(fs->delete_directory(delete_path2)); + // create directory + io::Path create_path = delete_path; + CHECK_STATUS_OK(fs->create_directory(create_path)); + // write file + std::string file1 = "tmp1/file1.txt"; + io::FileWriterPtr writer; + CHECK_STATUS_OK(fs->create_file(file1, &writer)); + CHECK_STATUS_OK(writer->append({"content"})); + CHECK_STATUS_OK(writer->close()); + // read file + io::FileReaderSPtr reader; + CHECK_STATUS_OK(fs->open_file(file1, &reader)); + char read_buf[10]; + size_t bytes_read = 0; + CHECK_STATUS_OK(reader->read_at(0, {read_buf, 10}, &bytes_read)); + ASSERT_EQ(7, bytes_read); + + // exist + bool exists = false; + CHECK_STATUS_OK(fs->exists(file1, &exists)); + ASSERT_TRUE(exists); + std::string file_non_exist = "non-exist"; + CHECK_STATUS_OK(fs->exists(file_non_exist, &exists)); + ASSERT_FALSE(exists); + CHECK_STATUS_OK(fs->exists(delete_path, &exists)); + ASSERT_TRUE(exists); + + // file size + size_t file_size = 0; + CHECK_STATUS_OK(fs->file_size(file1, &file_size)); + ASSERT_EQ(7, file_size); + + // write more files + for (int i = 0; i < 10; i++) { + std::string tmp_file = fmt::format("tmp1/tmp_file_{}", i); + io::FileWriterPtr writer; + CHECK_STATUS_OK(fs->create_file(tmp_file, &writer)); + CHECK_STATUS_OK(writer->append({"content"})); + CHECK_STATUS_OK(writer->close()); + } + + // list files + std::vector files; + CHECK_STATUS_OK(fs->list(delete_path, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(11, files.size()); + for (auto& file_info : files) { + std::cout << "file name: " << file_info.file_name << std::endl; + ASSERT_EQ(7, file_info.file_size); + ASSERT_TRUE(file_info.is_file); + } + std::string non_exist_path = "non_exist/"; + files.clear(); + CHECK_STATUS_OK(fs->list(non_exist_path, true, &files, &exists)); + ASSERT_FALSE(exists); + ASSERT_EQ(0, files.size()); + + // rename + std::string src_name = file1; + std::string dst_name = "tmp1/new_file1.txt"; + CHECK_STATUS_OK(fs->rename(src_name, dst_name)); + CHECK_STATUS_OK(fs->exists(src_name, &exists)); + ASSERT_FALSE(exists); + CHECK_STATUS_OK(fs->exists(dst_name, &exists)); + ASSERT_TRUE(exists); + + // rename dir + std::string src_dir = delete_path; + std::string dst_dir = "tmp2"; + CHECK_STATUS_OK(fs->rename_dir(src_dir, dst_dir)); + CHECK_STATUS_OK(fs->exists(dst_name, &exists)); + ASSERT_FALSE(exists); + std::string new_dst_name = "tmp2/new_file1.txt"; + CHECK_STATUS_OK(fs->exists(new_dst_name, &exists)); + ASSERT_TRUE(exists); + + // batch delete + std::vector delete_files; + for (int i = 0; i < 10; i++) { + std::string tmp_file = fmt::format("tmp2/tmp_file_{}", i); + delete_files.emplace_back(tmp_file); + CHECK_STATUS_OK(fs->batch_delete(delete_files)); + } + + // list to check + files.clear(); + CHECK_STATUS_OK(fs->list(dst_dir, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(1, files.size()); +} + +TEST_F(FileSystemTest, TestS3FileSystem) { + S3Conf s3_conf; + S3URI s3_uri(s3_location); + CHECK_STATUS_OK(s3_uri.parse()); + CHECK_STATUS_OK(S3ClientFactory::convert_properties_to_s3_conf(s3_prop, s3_uri, &s3_conf)); + std::shared_ptr fs; + CHECK_STATUS_OK(io::S3FileSystem::create(std::move(s3_conf), "", &fs)); + + // delete directory + io::Path delete_path = s3_location + "/tmp1"; + CHECK_STATUS_OK(fs->delete_directory(delete_path)); + io::Path delete_path2 = s3_location + "/tmp2"; + CHECK_STATUS_OK(fs->delete_directory(delete_path2)); + // create directory + io::Path create_path = delete_path; + CHECK_STATUS_OK(fs->create_directory(create_path)); + // write file + std::string file1 = s3_location + "/tmp1/file1.txt"; + io::FileWriterPtr writer; + CHECK_STATUS_OK(fs->create_file(file1, &writer)); + CHECK_STATUS_OK(writer->append({"content"})); + CHECK_STATUS_OK(writer->close()); + // read file + io::FileReaderSPtr reader; + CHECK_STATUS_OK(fs->open_file(file1, &reader)); + char read_buf[10]; + size_t bytes_read = 0; + CHECK_STATUS_OK(reader->read_at(0, {read_buf, 10}, &bytes_read)); + ASSERT_EQ(7, bytes_read); + + // exist + bool exists = false; + CHECK_STATUS_OK(fs->exists(file1, &exists)); + ASSERT_TRUE(exists); + std::string file_non_exist = s3_location + "/non-exist"; + CHECK_STATUS_OK(fs->exists(file_non_exist, &exists)); + ASSERT_FALSE(exists); + CHECK_STATUS_OK(fs->exists(delete_path, &exists)); + ASSERT_FALSE(exists); + + // file size + size_t file_size = 0; + CHECK_STATUS_OK(fs->file_size(file1, &file_size)); + ASSERT_EQ(7, file_size); + + // write more files + for (int i = 0; i < 10; i++) { + std::string tmp_file = fmt::format(s3_location + "/tmp1/tmp_file_{}", i); + io::FileWriterPtr writer; + CHECK_STATUS_OK(fs->create_file(tmp_file, &writer)); + CHECK_STATUS_OK(writer->append({"content"})); + CHECK_STATUS_OK(writer->close()); + } + + // list files + std::vector files; + CHECK_STATUS_OK(fs->list(delete_path, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(11, files.size()); + for (auto& file_info : files) { + std::cout << "file name: " << file_info.file_name << std::endl; + ASSERT_EQ(7, file_info.file_size); + ASSERT_TRUE(file_info.is_file); + } + std::string non_exist_path = s3_location + "/non_exist/"; + files.clear(); + CHECK_STATUS_OK(fs->list(non_exist_path, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(0, files.size()); + + // rename + std::string src_name = file1; + std::string dst_name = s3_location + "/tmp1/new_file1.txt"; + CHECK_STATUS_OK(fs->rename(src_name, dst_name)); + CHECK_STATUS_OK(fs->exists(src_name, &exists)); + ASSERT_FALSE(exists); + CHECK_STATUS_OK(fs->exists(dst_name, &exists)); + ASSERT_TRUE(exists); + + // rename dir + std::string src_dir = delete_path; + std::string dst_dir = s3_location + "/tmp2"; + CHECK_STATUS_OK(fs->rename_dir(src_dir, dst_dir)); + CHECK_STATUS_OK(fs->exists(dst_name, &exists)); + ASSERT_FALSE(exists); + std::string new_dst_name = dst_dir + "/new_file1.txt"; + CHECK_STATUS_OK(fs->exists(new_dst_name, &exists)); + ASSERT_TRUE(exists); + + // batch delete + std::vector delete_files; + for (int i = 0; i < 10; i++) { + std::string tmp_file = fmt::format(s3_location + "/tmp2/tmp_file_{}", i); + delete_files.emplace_back(tmp_file); + CHECK_STATUS_OK(fs->batch_delete(delete_files)); + } + + // list to check + files.clear(); + CHECK_STATUS_OK(fs->list(dst_dir, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(1, files.size()); + + // upload + std::string upload_file = "./be/test/io/fs/test_data/upload_file.txt"; + std::string remote_file = dst_dir + "/upload_file.txt"; + CHECK_STATUS_OK(fs->upload(upload_file, remote_file)); + exists = false; + CHECK_STATUS_OK(fs->exists(remote_file, &exists)); + ASSERT_TRUE(exists); + + // batch upload + std::vector local_files; + std::vector remote_files; + for (int i = 0; i < 10; i++) { + local_files.push_back(upload_file); + remote_files.push_back(fmt::format(dst_dir + "/upload_file_{}", i)); + } + CHECK_STATUS_OK(fs->batch_upload(local_files, remote_files)); + files.clear(); + CHECK_STATUS_OK(fs->list(dst_dir, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(12, files.size()); + for (auto& file : remote_files) { + CHECK_STATUS_OK(fs->file_size(file, &file_size)); + ASSERT_EQ(11, file_size); + } + + // direct_upload + std::string direct_remote_file = dst_dir + "/direct_upload_file"; + CHECK_STATUS_OK(fs->direct_upload(direct_remote_file, "abc")); + files.clear(); + CHECK_STATUS_OK(fs->list(dst_dir, true, &files, &exists)); + ASSERT_TRUE(exists); + ASSERT_EQ(13, files.size()); + CHECK_STATUS_OK(fs->file_size(direct_remote_file, &file_size)); + ASSERT_EQ(3, file_size); + + // download + std::string local_download_file = "./be/test/io/fs/test_data/download_file.txt"; + CHECK_STATUS_OK(io::global_local_filesystem()->delete_file(local_download_file)); + CHECK_STATUS_OK(fs->download(direct_remote_file, local_download_file)); + CHECK_STATUS_OK(io::global_local_filesystem()->file_size(local_download_file, &file_size)); + ASSERT_EQ(3, file_size); + + // direct download + std::string download_content; + CHECK_STATUS_OK(fs->direct_download(direct_remote_file, &download_content)); + ASSERT_EQ("abc", download_content); +} + +} // namespace doris diff --git a/be/test/io/fs/test_data/download_file.txt b/be/test/io/fs/test_data/download_file.txt new file mode 100644 index 0000000000..f2ba8f84ab --- /dev/null +++ b/be/test/io/fs/test_data/download_file.txt @@ -0,0 +1 @@ +abc \ No newline at end of file diff --git a/be/test/io/fs/test_data/upload_file.txt b/be/test/io/fs/test_data/upload_file.txt new file mode 100644 index 0000000000..a32a4347a4 --- /dev/null +++ b/be/test/io/fs/test_data/upload_file.txt @@ -0,0 +1 @@ +1234567890 diff --git a/be/test/olap/file_helper_test.cpp b/be/test/olap/file_header_test.cpp similarity index 63% rename from be/test/olap/file_helper_test.cpp rename to be/test/olap/file_header_test.cpp index fe8ab22980..286672e76f 100644 --- a/be/test/olap/file_helper_test.cpp +++ b/be/test/olap/file_header_test.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "olap/file_helper.h" +#include "olap/file_header.h" #include #include @@ -33,7 +33,7 @@ using std::string; namespace doris { -class FileHandlerTest : public testing::Test { +class FileHeaderTest : public testing::Test { public: // create a mock cgroup folder virtual void SetUp() { @@ -49,49 +49,32 @@ public: static std::string _s_test_data_path; }; -std::string FileHandlerTest::_s_test_data_path = "./log/file_handler_testxxxx123"; +std::string FileHeaderTest::_s_test_data_path = "./log/file_handler_testxxxx123"; -TEST_F(FileHandlerTest, TestWrite) { - FileHandler file_handler; +TEST_F(FileHeaderTest, TestWrite) { + std::shared_ptr fs = io::global_local_filesystem(); std::string file_name = _s_test_data_path + "/abcd123.txt"; // create a file using open - EXPECT_FALSE(std::filesystem::exists(file_name)); - Status op_status = - file_handler.open_with_mode(file_name, O_CREAT | O_EXCL | O_WRONLY, S_IRUSR | S_IWUSR); - EXPECT_EQ(Status::OK(), op_status); - EXPECT_TRUE(std::filesystem::exists(file_name)); + bool exists = true; + EXPECT_TRUE(fs->exists(file_name, &exists).ok()); + EXPECT_FALSE(exists); - // tell current offset - off_t cur_offset = file_handler.tell(); - EXPECT_EQ(0, cur_offset); - off_t length = file_handler.length(); - EXPECT_EQ(0, length); - - // seek to 10 and test offset - off_t res = file_handler.seek(10, SEEK_SET); - EXPECT_EQ(10, res); - length = file_handler.length(); - EXPECT_EQ(0, length); - - cur_offset = file_handler.tell(); - EXPECT_EQ(10, cur_offset); + io::FileWriterPtr file_writer; + EXPECT_TRUE(fs->create_file(file_name, &file_writer).ok()); // write 12 bytes to disk char ten_bytes[12]; memset(&ten_bytes, 0, sizeof(ten_bytes)); - file_handler.write(&ten_bytes, sizeof(ten_bytes)); - cur_offset = file_handler.tell(); - EXPECT_EQ(22, cur_offset); - length = file_handler.length(); - EXPECT_EQ(22, length); + EXPECT_TRUE(file_writer->append({ten_bytes, sizeof(ten_bytes)}).ok()); char large_bytes2[(1 << 10)]; memset(&large_bytes2, 0, sizeof(large_bytes2)); int i = 1; while (i < LOOP_LESS_OR_MORE(1 << 10, 1 << 17)) { - file_handler.write(&large_bytes2, sizeof(large_bytes2)); + EXPECT_TRUE(file_writer->append({large_bytes2, sizeof(large_bytes2)}).ok()); ++i; } + EXPECT_TRUE(file_writer->close().ok()); } } // namespace doris diff --git a/be/test/olap/file_utils_test.cpp b/be/test/olap/file_utils_test.cpp index 8417b9b409..85634182d9 100644 --- a/be/test/olap/file_utils_test.cpp +++ b/be/test/olap/file_utils_test.cpp @@ -27,7 +27,7 @@ #include "env/env.h" #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "olap/file_helper.h" +#include "olap/file_header.h" using ::testing::_; using ::testing::Return; @@ -60,32 +60,28 @@ public: std::string FileUtilsTest::_s_test_data_path = "./file_utils_testxxxx123"; TEST_F(FileUtilsTest, TestCopyFile) { - FileHandler src_file_handler; std::string src_file_name = _s_test_data_path + "/abcd12345.txt"; // create a file using open - EXPECT_FALSE(std::filesystem::exists(src_file_name)); - Status op_status = src_file_handler.open_with_mode(src_file_name, O_CREAT | O_EXCL | O_WRONLY, - S_IRUSR | S_IWUSR); - EXPECT_EQ(Status::OK(), op_status); - EXPECT_TRUE(std::filesystem::exists(src_file_name)); + std::shared_ptr fs = io::global_local_filesystem(); + io::FileWriterPtr file_writer; + EXPECT_TRUE(fs->create_file(src_file_name, &file_writer).ok()); char large_bytes2[(1 << 12)]; memset(&large_bytes2, 0, sizeof(large_bytes2)); int i = 0; while (i < 1 << 10) { - src_file_handler.write(&large_bytes2, sizeof(large_bytes2)); + EXPECT_TRUE(file_writer->append({large_bytes2, sizeof(large_bytes2)}).ok()); ++i; } - src_file_handler.write(&large_bytes2, 13); - src_file_handler.close(); + EXPECT_TRUE(file_writer->append({large_bytes2, 13}).ok()); + EXPECT_TRUE(file_writer->close().ok()); std::string dst_file_name = _s_test_data_path + "/abcd123456.txt"; FileUtils::copy_file(src_file_name, dst_file_name); - FileHandler dst_file_handler; - dst_file_handler.open(dst_file_name, O_RDONLY); - int64_t dst_length = dst_file_handler.length(); - int64_t src_length = 4194317; - EXPECT_EQ(src_length, dst_length); + + io::FileReaderSPtr file_reader; + EXPECT_TRUE(fs->open_file(dst_file_name, &file_reader).ok()); + EXPECT_EQ(4194317, file_reader->size()); } TEST_F(FileUtilsTest, TestRemove) { diff --git a/be/test/olap/primary_key_index_test.cpp b/be/test/olap/primary_key_index_test.cpp index 7f9e33263b..442a004603 100644 --- a/be/test/olap/primary_key_index_test.cpp +++ b/be/test/olap/primary_key_index_test.cpp @@ -70,7 +70,7 @@ TEST_F(PrimaryKeyIndexTest, builder) { FilePathDesc path_desc(filename); PrimaryKeyIndexReader index_reader; io::FileReaderSPtr file_reader; - EXPECT_TRUE(fs->open_file(filename, &file_reader, nullptr).ok()); + EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); EXPECT_TRUE(index_reader.parse_index(file_reader, index_meta).ok()); EXPECT_TRUE(index_reader.parse_bf(file_reader, index_meta).ok()); EXPECT_EQ(num_rows, index_reader.num_rows()); diff --git a/be/test/olap/remote_rowset_gc_test.cpp b/be/test/olap/remote_rowset_gc_test.cpp index 6b09183af5..6db84ba171 100644 --- a/be/test/olap/remote_rowset_gc_test.cpp +++ b/be/test/olap/remote_rowset_gc_test.cpp @@ -53,8 +53,10 @@ public: s3_conf.region = config::test_s3_region; s3_conf.bucket = config::test_s3_bucket; s3_conf.prefix = "remote_rowset_gc_test"; - auto s3_fs = io::S3FileSystem::create(std::move(s3_conf), std::to_string(kResourceId)); - ASSERT_TRUE(s3_fs->connect().ok()); + std::shared_ptr s3_fs; + ASSERT_TRUE( + io::S3FileSystem::create(std::move(s3_conf), std::to_string(kResourceId), &s3_fs) + .ok()); put_storage_resource(kResourceId, {s3_fs, 1}); auto storage_policy = std::make_shared(); storage_policy->name = "TabletCooldownTest"; diff --git a/be/test/olap/rowset/beta_rowset_test.cpp b/be/test/olap/rowset/beta_rowset_test.cpp index de05810d21..91b2b06954 100644 --- a/be/test/olap/rowset/beta_rowset_test.cpp +++ b/be/test/olap/rowset/beta_rowset_test.cpp @@ -231,9 +231,8 @@ TEST_F(BetaRowsetTest, ReadTest) { s3_conf.bucket = "bucket"; s3_conf.prefix = "prefix"; std::string resource_id = "10000"; - auto fs = io::S3FileSystem::create(std::move(s3_conf), resource_id); - Aws::SDKOptions aws_options = Aws::SDKOptions {}; - Aws::InitAPI(aws_options); + std::shared_ptr fs; + ASSERT_TRUE(io::S3FileSystem::create(std::move(s3_conf), resource_id, &fs).ok()); // failed to head object { Aws::Auth::AWSCredentials aws_cred("ak", "sk"); @@ -280,8 +279,6 @@ TEST_F(BetaRowsetTest, ReadTest) { Status st = rowset.load_segments(&segments); ASSERT_FALSE(st.ok()); } - - Aws::ShutdownAPI(aws_options); } } // namespace doris diff --git a/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp b/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp index 74a5e77bdf..fc1e9cd62f 100644 --- a/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp +++ b/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp @@ -80,8 +80,7 @@ template void get_bitmap_reader_iter(const std::string& file_name, const ColumnIndexMetaPB& meta, BitmapIndexReader** reader, BitmapIndexIterator** iter) { io::FileReaderSPtr file_reader; - ASSERT_EQ(io::global_local_filesystem()->open_file(file_name, &file_reader, nullptr), - Status::OK()); + ASSERT_EQ(io::global_local_filesystem()->open_file(file_name, &file_reader), Status::OK()); *reader = new BitmapIndexReader(std::move(file_reader), &meta.bitmap_index()); auto st = (*reader)->load(true, false); EXPECT_TRUE(st.ok()); diff --git a/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp index 30061af1d4..661eb497eb 100644 --- a/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp +++ b/be/test/olap/rowset/segment_v2/bloom_filter_index_reader_writer_test.cpp @@ -91,7 +91,7 @@ void get_bloom_filter_reader_iter(const std::string& file_name, const ColumnInde std::unique_ptr* iter) { std::string fname = dname + "/" + file_name; io::FileReaderSPtr file_reader; - ASSERT_EQ(io::global_local_filesystem()->open_file(fname, &file_reader, nullptr), Status::OK()); + ASSERT_EQ(io::global_local_filesystem()->open_file(fname, &file_reader), Status::OK()); *reader = new BloomFilterIndexReader(std::move(file_reader), &meta.bloom_filter_index()); auto st = (*reader)->load(true, false); EXPECT_TRUE(st.ok()); diff --git a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp index 3e8ce71e80..f3c4f395e4 100644 --- a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp +++ b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp @@ -129,7 +129,7 @@ void test_nullable_data(uint8_t* src_data, uint8_t* src_is_null, int num_rows, } auto type_info = get_scalar_type_info(type); io::FileReaderSPtr file_reader; - ASSERT_EQ(fs->open_file(fname, &file_reader, nullptr), Status::OK()); + ASSERT_EQ(fs->open_file(fname, &file_reader), Status::OK()); // read and check { // sequence read @@ -305,7 +305,7 @@ void test_array_nullable_data(CollectionValue* src_data, uint8_t* src_is_null, i } auto type_info = get_type_info(&meta); io::FileReaderSPtr file_reader; - ASSERT_EQ(fs->open_file(fname, &file_reader, nullptr), Status::OK()); + ASSERT_EQ(fs->open_file(fname, &file_reader), Status::OK()); // read and check { ColumnReaderOptions reader_opts; diff --git a/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp b/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp index 03a23ef860..cb7d730973 100644 --- a/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp +++ b/be/test/olap/rowset/segment_v2/ordinal_page_index_test.cpp @@ -77,7 +77,7 @@ TEST_F(OrdinalPageIndexTest, normal) { } io::FileReaderSPtr file_reader; - EXPECT_TRUE(fs->open_file(filename, &file_reader, nullptr).ok()); + EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); OrdinalIndexReader index(file_reader, &index_meta.ordinal_index(), 16 * 1024 * 4096 + 1); EXPECT_TRUE(index.load(true, false).ok()); EXPECT_EQ(16 * 1024, index.num_data_pages()); diff --git a/be/test/olap/rowset/segment_v2/zone_map_index_test.cpp b/be/test/olap/rowset/segment_v2/zone_map_index_test.cpp index 8df497870d..f7607d8ca9 100644 --- a/be/test/olap/rowset/segment_v2/zone_map_index_test.cpp +++ b/be/test/olap/rowset/segment_v2/zone_map_index_test.cpp @@ -84,7 +84,7 @@ public: } io::FileReaderSPtr file_reader; - EXPECT_TRUE(fs->open_file(filename, &file_reader, nullptr).ok()); + EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); ZoneMapIndexReader column_zone_map(file_reader, &index_meta.zone_map_index()); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); @@ -131,7 +131,7 @@ public: } io::FileReaderSPtr file_reader; - EXPECT_TRUE(fs->open_file(filename, &file_reader, nullptr).ok()); + EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); ZoneMapIndexReader column_zone_map(file_reader, &index_meta.zone_map_index()); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); @@ -184,7 +184,7 @@ TEST_F(ColumnZoneMapTest, NormalTestIntPage) { } io::FileReaderSPtr file_reader; - EXPECT_TRUE(fs->open_file(filename, &file_reader, nullptr).ok()); + EXPECT_TRUE(fs->open_file(filename, &file_reader).ok()); ZoneMapIndexReader column_zone_map(file_reader, &index_meta.zone_map_index()); Status status = column_zone_map.load(true, false); EXPECT_TRUE(status.ok()); diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 6ee90236c9..d43e56164d 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -58,7 +58,7 @@ static constexpr int32_t kPartitionId2 = 50003; using io::Path; -static io::FileSystemSPtr s_fs; +static io::RemoteFileSystemSPtr s_fs; static std::string get_remote_path(const Path& path) { return fmt::format("{}/remote/{}", config::storage_root_path, path.string()); @@ -66,8 +66,12 @@ static std::string get_remote_path(const Path& path) { class FileWriterMock : public io::FileWriter { public: - FileWriterMock(Path path) : io::FileWriter(std::move(path)) { - io::global_local_filesystem()->create_file(get_remote_path(_path), &_local_file_writer); + FileWriterMock(Path path) : io::FileWriter(std::move(path), io::global_local_filesystem()) { + Status st = io::global_local_filesystem()->create_file(get_remote_path(_path), + &_local_file_writer); + if (!st.ok()) { + std::cerr << "create file writer failed: " << st << std::endl; + } } ~FileWriterMock() override = default; @@ -76,8 +80,6 @@ public: Status abort() override { return _local_file_writer->abort(); } - Status append(const Slice& data) override { return _local_file_writer->append(data); } - Status appendv(const Slice* data, size_t data_cnt) override { return _local_file_writer->appendv(data, data_cnt); } @@ -88,10 +90,6 @@ public: Status finalize() override { return _local_file_writer->finalize(); } - size_t bytes_appended() const override { return _local_file_writer->bytes_appended(); } - - io::FileSystemSPtr fs() const override { return s_fs; } - private: std::unique_ptr _local_file_writer; }; @@ -104,69 +102,91 @@ public: } ~RemoteFileSystemMock() override = default; - Status create_file(const Path& path, io::FileWriterPtr* writer) override { +protected: + Status create_file_impl(const Path& path, io::FileWriterPtr* writer) override { Path fs_path = path; *writer = std::make_unique(fs_path); return Status::OK(); } - Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { - return _local_fs->open_file(get_remote_path(path), reader, io_ctx); - } - - Status delete_file(const Path& path) override { - return _local_fs->delete_file(get_remote_path(path)); - } - - Status create_directory(const Path& path) override { + Status create_directory_impl(const Path& path) override { return _local_fs->create_directory(get_remote_path(path)); } - Status delete_directory(const Path& path) override { - return _local_fs->delete_directory(get_remote_path(path)); + Status delete_file_impl(const Path& path) override { + return _local_fs->delete_file(get_remote_path(path)); } - Status link_file(const Path& src, const Path& dest) override { - return _local_fs->link_file(get_remote_path(src), get_remote_path(dest)); - } - - Status exists(const Path& path, bool* res) const override { - return _local_fs->exists(get_remote_path(path), res); - } - - Status file_size(const Path& path, size_t* file_size) const override { - return _local_fs->file_size(get_remote_path(path), file_size); - } - - Status list(const Path& path, std::vector* files) override { - std::vector local_paths; - RETURN_IF_ERROR(_local_fs->list(get_remote_path(path), &local_paths)); - for (Path path : local_paths) { - files->emplace_back(path.string().substr(config::storage_root_path.size() + 1)); - } - return Status::OK(); - } - - Status upload(const Path& local_path, const Path& dest_path) override { - return _local_fs->link_file(local_path.string(), get_remote_path(dest_path)); - } - - Status batch_upload(const std::vector& local_paths, - const std::vector& dest_paths) override { - for (int i = 0; i < local_paths.size(); ++i) { - RETURN_IF_ERROR(upload(local_paths[i], dest_paths[i])); - } - return Status::OK(); - } - - Status batch_delete(const std::vector& paths) override { + Status batch_delete_impl(const std::vector& paths) override { for (int i = 0; i < paths.size(); ++i) { RETURN_IF_ERROR(delete_file(paths[i])); } return Status::OK(); } - Status connect() override { return Status::OK(); } + Status delete_directory_impl(const Path& path) override { + return _local_fs->delete_directory(get_remote_path(path)); + } + + Status exists_impl(const Path& path, bool* res) const override { + return _local_fs->exists(get_remote_path(path), res); + } + + Status file_size_impl(const Path& path, size_t* file_size) const override { + return _local_fs->file_size(get_remote_path(path), file_size); + } + + Status list_impl(const Path& dir, bool regular_file, std::vector* files, + bool* exists) override { + RETURN_IF_ERROR(_local_fs->list(get_remote_path(dir), true, files, exists)); + // for (auto& path : local_paths) { + // files->emplace_back(path.file_name.substr(config::storage_root_path.size() + 1)); + // } + return Status::OK(); + } + + Status upload_impl(const Path& local_path, const Path& dest_path) override { + return _local_fs->link_file(local_path.string(), get_remote_path(dest_path)); + } + + Status batch_upload_impl(const std::vector& local_paths, + const std::vector& dest_paths) override { + for (int i = 0; i < local_paths.size(); ++i) { + RETURN_IF_ERROR(upload_impl(local_paths[i], dest_paths[i])); + } + return Status::OK(); + } + + Status direct_upload_impl(const Path& remote_file, const std::string& content) override { + return Status::OK(); + } + + Status upload_with_checksum_impl(const Path& local_file, const Path& remote_file, + const std::string& checksum) override { + return Status::OK(); + } + + Status download_impl(const Path& remote_file, const Path& local_file) override { + return Status::OK(); + } + + Status direct_download_impl(const Path& remote_file, std::string* content) override { + return Status::OK(); + } + + Status open_file_internal(const Path& file, io::FileReaderSPtr* reader) override { + return _local_fs->open_file(get_remote_path(file), io::FileReaderOptions::DEFAULT, reader); + } + + Status connect_impl() override { return Status::OK(); } + + Status rename_impl(const Path& orig_name, const Path& new_name) override { + return Status::OK(); + } + + Status rename_dir_impl(const Path& orig_name, const Path& new_name) override { + return Status::OK(); + } private: std::shared_ptr _local_fs; @@ -175,8 +195,8 @@ private: class TabletCooldownTest : public testing::Test { public: static void SetUpTestSuite() { - s_fs.reset(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), - io::FileSystemType::S3)); + s_fs.reset( + new RemoteFileSystemMock("", std::to_string(kResourceId), io::FileSystemType::S3)); StorageResource resource = {s_fs, 1}; put_storage_resource(kResourceId, resource); auto storage_policy = std::make_shared(); diff --git a/be/test/util/broker_storage_backend_test.cpp b/be/test/util/broker_storage_backend_test.cpp deleted file mode 100644 index 002ba9d977..0000000000 --- a/be/test/util/broker_storage_backend_test.cpp +++ /dev/null @@ -1,190 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/broker_storage_backend.h" - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "runtime/exec_env.h" -#include "util/file_utils.h" -#include "util/storage_backend.h" - -namespace doris { -using namespace ErrorCode; -static const std::string AK = "AK"; -static const std::string SK = "SK"; -static const std::string ENDPOINT = "http://bj.bcebos.com"; -static const std::string BUCKET = "bos://yang-repo/"; -static const std::string BROKER_IP = "127.0.0.1"; - -#define StorageBackendTest DISABLED_StorageBackendTest -class StorageBackendTest : public testing::Test { -public: - StorageBackendTest() - : _broker_properties({{"bos_accesskey", AK}, - {"bos_secret_accesskey", SK}, - {"bos_endpoint", ENDPOINT}}), - _env(ExecEnv::GetInstance()) { - _broker_addr.__set_hostname("127.0.0.1"); - _broker_addr.__set_port(8111); - _broker.reset(new BrokerStorageBackend(_env, _broker_addr, _broker_properties)); - _broker_base_path = BUCKET + "broker/" + gen_uuid(); - } - virtual ~StorageBackendTest() {} - -protected: - virtual void SetUp() { - _test_file = "/tmp/" + gen_uuid(); - std::ofstream out(_test_file); - out << _content; - out.close(); - } - virtual void TearDown() { remove(_test_file.c_str()); } - std::string gen_uuid() { - auto id = boost::uuids::random_generator()(); - return boost::uuids::to_string(id); - } - std::unique_ptr _broker; - std::map _broker_properties; - std::string _test_file; - ExecEnv* _env; - std::string _broker_base_path; - TNetworkAddress _broker_addr; - std::string _content = - "O wild West Wind, thou breath of Autumn's being\n" - "Thou, from whose unseen presence the leaves dead\n" - "Are driven, like ghosts from an enchanter fleeing,\n" - "Yellow, and black, and pale, and hectic red,\n" - "Pestilence-stricken multitudes:O thou\n" - "Who chariotest to their dark wintry bed\n" - "The winged seeds, where they lie cold and low,\n" - "Each like a corpse within its grave, until\n" - "Thine azure sister of the Spring shall blow\n" - "Her clarion o'er the dreaming earth, and fill\n" - "(Driving sweet buds like flocks to feed in air)\n" - "With living hues and odors plain and hill:\n" - "Wild Spirit, which art moving everywhere;\n" - "Destroyer and preserver; hear, oh, hear!"; -}; - -TEST_F(StorageBackendTest, broker_upload) { - Status status = _broker->upload(_test_file, _broker_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - status = _broker->exist(_broker_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - std::string orig_md5sum; - FileUtils::md5sum(_test_file, &orig_md5sum); - status = _broker->download(_broker_base_path + "/Ode_to_the_West_Wind.txt", - _test_file + ".download"); - EXPECT_TRUE(status.ok()); - std::string download_md5sum; - FileUtils::md5sum(_test_file + ".download", &download_md5sum); - EXPECT_EQ(orig_md5sum, download_md5sum); - status = _broker->upload(_test_file + "_not_found", - _broker_base_path + "/Ode_to_the_West_Wind1.txt"); - EXPECT_FALSE(status.ok()); - status = _broker->exist(_broker_base_path + "/Ode_to_the_West_Wind1.txt"); - EXPECT_EQ(NOT_FOUND, status.code()); -} - -TEST_F(StorageBackendTest, broker_direct_upload) { - Status status = - _broker->direct_upload(_broker_base_path + "/Ode_to_the_West_Wind.txt", _content); - EXPECT_TRUE(status.ok()); - status = _broker->exist(_broker_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - std::string orig_md5sum; - FileUtils::md5sum(_test_file, &orig_md5sum); - status = _broker->download(_broker_base_path + "/Ode_to_the_West_Wind.txt", - _test_file + ".download"); - EXPECT_TRUE(status.ok()); - std::string download_md5sum; - FileUtils::md5sum(_test_file + ".download", &download_md5sum); - EXPECT_EQ(orig_md5sum, download_md5sum); -} - -TEST_F(StorageBackendTest, broker_download) { - Status status = _broker->upload(_test_file, _broker_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - std::string orig_md5sum; - FileUtils::md5sum(_test_file, &orig_md5sum); - status = _broker->download(_broker_base_path + "/Ode_to_the_West_Wind.txt", - _test_file + ".download"); - EXPECT_TRUE(status.ok()); - std::string download_md5sum; - FileUtils::md5sum(_test_file + ".download", &download_md5sum); - EXPECT_EQ(orig_md5sum, download_md5sum); - status = _broker->download(_broker_base_path + "/Ode_to_the_West_Wind.txt.not_found", - _test_file + ".download"); - EXPECT_FALSE(status.ok()); - status = _broker->download(_broker_base_path + "/Ode_to_the_West_Wind.txt.not_found", - "/not_permitted.download"); - EXPECT_FALSE(status.ok()); -} - -TEST_F(StorageBackendTest, broker_rename) { - Status status = - _broker->direct_upload(_broker_base_path + "/Ode_to_the_West_Wind.txt", _content); - EXPECT_TRUE(status.ok()); - status = _broker->rename(_broker_base_path + "/Ode_to_the_West_Wind.txt", - _broker_base_path + "/Ode_to_the_West_Wind.txt.new"); - EXPECT_TRUE(status.ok()); - // rm by broker old file may exist for a few moment - // status = _broker->exist(_broker_base_path + "/Ode_to_the_West_Wind.txt"); - // EXPECT_TRUE(status.code() == NOT_FOUND); - status = _broker->exist(_broker_base_path + "/Ode_to_the_West_Wind.txt.new"); - EXPECT_TRUE(status.ok()); -} - -TEST_F(StorageBackendTest, broker_list) { - Status status = - _broker->direct_upload(_broker_base_path + "/Ode_to_the_West_Wind.md5", _content); - EXPECT_TRUE(status.ok()); - status = _broker->direct_upload(_broker_base_path + "/Ode_to_the_West_Wind1.md5", _content); - EXPECT_TRUE(status.ok()); - status = _broker->direct_upload(_broker_base_path + "/Ode_to_the_West_Wind2.md5", _content); - EXPECT_TRUE(status.ok()); - std::map files; - status = _broker->list(_broker_base_path, true, false, &files); - EXPECT_TRUE(status.ok()); - EXPECT_TRUE(files.find("Ode_to_the_West_Wind") != files.end()); - EXPECT_TRUE(files.find("Ode_to_the_West_Wind1") != files.end()); - EXPECT_TRUE(files.find("Ode_to_the_West_Wind2") != files.end()); - EXPECT_EQ(3, files.size()); -} - -TEST_F(StorageBackendTest, broker_rm) { - Status status = - _broker->direct_upload(_broker_base_path + "/Ode_to_the_West_Wind.txt", _content); - EXPECT_TRUE(status.ok()); - status = _broker->exist(_broker_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - status = _broker->rm(_broker_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - status = _broker->exist(_broker_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.code() == NOT_FOUND); -} - -} // end namespace doris diff --git a/be/test/util/hdfs_storage_backend_test.cpp b/be/test/util/hdfs_storage_backend_test.cpp deleted file mode 100644 index 42960152f1..0000000000 --- a/be/test/util/hdfs_storage_backend_test.cpp +++ /dev/null @@ -1,172 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/hdfs_storage_backend.h" - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "util/file_utils.h" -#include "util/storage_backend.h" - -namespace doris { -using namespace ErrorCode; -static const std::string fs_name = "hdfs://127.0.0.1:9000"; // An invalid address -static const std::string user = "test"; -static const std::string base_path = "/user/test"; - -#define HDFSStorageBackendTest DISABLED_HDFSStorageBackendTest - -class HDFSStorageBackendTest : public testing::Test { -public: - HDFSStorageBackendTest() : _properties({{FS_KEY, fs_name}, {USER, user}}) { - _fs.reset(new HDFSStorageBackend(_properties)); - _base_path = base_path + "/" + gen_uuid(); - } - virtual ~HDFSStorageBackendTest() {} - -protected: - virtual void SetUp() { - _test_file = "/tmp/" + gen_uuid(); - std::ofstream out(_test_file); - out << _content; - out.close(); - } - - virtual void TearDown() { remove(_test_file.c_str()); } - - std::string gen_uuid() { - auto id = boost::uuids::random_generator()(); - return boost::lexical_cast(id); - } - - std::unique_ptr _fs; - std::map _properties; - std::string _test_file; - std::string _base_path; - std::string _content = - "O wild West Wind, thou breath of Autumn's being\n" - "Thou, from whose unseen presence the leaves dead\n" - "Are driven, like ghosts from an enchanter fleeing,\n" - "Yellow, and black, and pale, and hectic red,\n" - "Pestilence-stricken multitudes:O thou\n" - "Who chariotest to their dark wintry bed\n" - "The winged seeds, where they lie cold and low,\n" - "Each like a corpse within its grave, until\n" - "Thine azure sister of the Spring shall blow\n" - "Her clarion o'er the dreaming earth, and fill\n" - "(Driving sweet buds like flocks to feed in air)\n" - "With living hues and odors plain and hill:\n" - "Wild Spirit, which art moving everywhere;\n" - "Destroyer and preserver; hear, oh, hear!"; -}; - -TEST_F(HDFSStorageBackendTest, hdfs_upload) { - Status status = _fs->upload(_test_file, _base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - status = _fs->exist(_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - std::string orig_md5sum; - FileUtils::md5sum(_test_file, &orig_md5sum); - status = _fs->download(_base_path + "/Ode_to_the_West_Wind.txt", _test_file + ".download"); - EXPECT_TRUE(status.ok()); - std::string download_md5sum; - FileUtils::md5sum(_test_file + ".download", &download_md5sum); - EXPECT_EQ(orig_md5sum, download_md5sum); - status = _fs->upload(_test_file + "_not_found", _base_path + "/Ode_to_the_West_Wind1.txt"); - EXPECT_FALSE(status.ok()); - status = _fs->exist(_base_path + "/Ode_to_the_West_Wind1.txt"); - EXPECT_EQ(NOT_FOUND, status.code()); -} - -TEST_F(HDFSStorageBackendTest, hdfs_direct_upload) { - Status status = _fs->direct_upload(_base_path + "/Ode_to_the_West_Wind.txt", _content); - EXPECT_TRUE(status.ok()); - status = _fs->exist(_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - std::string orig_md5sum; - FileUtils::md5sum(_test_file, &orig_md5sum); - status = _fs->download(_base_path + "/Ode_to_the_West_Wind.txt", _test_file + ".download"); - EXPECT_TRUE(status.ok()); - std::string download_md5sum; - FileUtils::md5sum(_test_file + ".download", &download_md5sum); - EXPECT_EQ(orig_md5sum, download_md5sum); -} - -TEST_F(HDFSStorageBackendTest, hdfs_download) { - Status status = _fs->upload(_test_file, _base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - std::string orig_md5sum; - FileUtils::md5sum(_test_file, &orig_md5sum); - status = _fs->download(_base_path + "/Ode_to_the_West_Wind.txt", _test_file + ".download"); - EXPECT_TRUE(status.ok()); - std::string download_md5sum; - FileUtils::md5sum(_test_file + ".download", &download_md5sum); - EXPECT_EQ(orig_md5sum, download_md5sum); - status = _fs->download(_base_path + "/Ode_to_the_West_Wind.txt.not_found", - _test_file + ".download"); - EXPECT_FALSE(status.ok()); - status = _fs->download(_base_path + "/Ode_to_the_West_Wind.txt.not_found", - "/not_permitted.download"); - EXPECT_FALSE(status.ok()); -} - -TEST_F(HDFSStorageBackendTest, hdfs_rename) { - Status status = _fs->direct_upload(_base_path + "/Ode_to_the_West_Wind.txt", _content); - EXPECT_TRUE(status.ok()); - status = _fs->rename(_base_path + "/Ode_to_the_West_Wind.txt", - _base_path + "/Ode_to_the_West_Wind.txt.new"); - EXPECT_TRUE(status.ok()); - status = _fs->exist(_base_path + "/Ode_to_the_West_Wind.txt.new"); - EXPECT_TRUE(status.ok()); -} - -TEST_F(HDFSStorageBackendTest, hdfs_list) { - Status status = _fs->direct_upload(_base_path + "/Ode_to_the_West_Wind.md5", _content); - EXPECT_TRUE(status.ok()); - status = _fs->direct_upload(_base_path + "/Ode_to_the_West_Wind1.md5", _content); - EXPECT_TRUE(status.ok()); - status = _fs->direct_upload(_base_path + "/Ode_to_the_West_Wind2.md5", _content); - EXPECT_TRUE(status.ok()); - std::map files; - status = _fs->list(_base_path, true, false, &files); - EXPECT_TRUE(status.ok()); - EXPECT_TRUE(files.find("Ode_to_the_West_Wind") != files.end()); - EXPECT_TRUE(files.find("Ode_to_the_West_Wind1") != files.end()); - EXPECT_TRUE(files.find("Ode_to_the_West_Wind2") != files.end()); - EXPECT_EQ(3, files.size()); -} - -TEST_F(HDFSStorageBackendTest, hdfs_rm) { - Status status = _fs->direct_upload(_base_path + "/Ode_to_the_West_Wind.txt", _content); - EXPECT_TRUE(status.ok()); - status = _fs->exist(_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - status = _fs->rm(_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - status = _fs->exist(_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.code() == NOT_FOUND); -} - -} // namespace doris \ No newline at end of file diff --git a/be/test/util/s3_storage_backend_test.cpp b/be/test/util/s3_storage_backend_test.cpp deleted file mode 100644 index 8e3c343701..0000000000 --- a/be/test/util/s3_storage_backend_test.cpp +++ /dev/null @@ -1,190 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "util/s3_storage_backend.h" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "util/file_utils.h" -#include "util/storage_backend.h" - -namespace doris { -using namespace ErrorCode; -static const std::string AK = ""; -static const std::string SK = ""; -static const std::string ENDPOINT = "http://s3.bj.bcebos.com"; -static const std::string USE_PATH_STYLE = "false"; -static const std::string REGION = "bj"; -static const std::string BUCKET = "s3://cmy-repo/"; - -#define S3StorageBackendTest DISABLED_S3StorageBackendTest - -class S3StorageBackendTest : public testing::Test { -public: - S3StorageBackendTest() - : _aws_properties({{"AWS_ACCESS_KEY", AK}, - {"AWS_SECRET_KEY", SK}, - {"AWS_ENDPOINT", ENDPOINT}, - {"USE_PATH_STYLE", USE_PATH_STYLE}, - {"AWS_REGION", REGION}}) { - _s3.reset(new S3StorageBackend(_aws_properties)); - _s3_base_path = BUCKET + "s3/" + gen_uuid(); - } - virtual ~S3StorageBackendTest() {} - -protected: - virtual void SetUp() { - _test_file = "/tmp/" + gen_uuid(); - std::ofstream out(_test_file); - out << _content; - out.close(); - } - virtual void TearDown() { - remove(_test_file.c_str()); - _s3->rm(_s3_base_path); - } - std::string gen_uuid() { - auto id = boost::uuids::random_generator()(); - return boost::lexical_cast(id); - } - std::unique_ptr _s3; - std::map _aws_properties; - std::string _test_file; - std::string _s3_base_path; - std::string _content = - "O wild West Wind, thou breath of Autumn's being\n" - "Thou, from whose unseen presence the leaves dead\n" - "Are driven, like ghosts from an enchanter fleeing,\n" - "Yellow, and black, and pale, and hectic red,\n" - "Pestilence-stricken multitudes:O thou\n" - "Who chariotest to their dark wintry bed\n" - "The winged seeds, where they lie cold and low,\n" - "Each like a corpse within its grave, until\n" - "Thine azure sister of the Spring shall blow\n" - "Her clarion o'er the dreaming earth, and fill\n" - "(Driving sweet buds like flocks to feed in air)\n" - "With living hues and odors plain and hill:\n" - "Wild Spirit, which art moving everywhere;\n" - "Destroyer and preserver; hear, oh, hear!"; -}; - -TEST_F(S3StorageBackendTest, s3_upload) { - Status status = _s3->upload(_test_file, _s3_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - status = _s3->exist(_s3_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - std::string orig_md5sum; - FileUtils::md5sum(_test_file, &orig_md5sum); - status = _s3->download(_s3_base_path + "/Ode_to_the_West_Wind.txt", _test_file + ".download"); - EXPECT_TRUE(status.ok()); - std::string download_md5sum; - FileUtils::md5sum(_test_file + ".download", &download_md5sum); - EXPECT_EQ(orig_md5sum, download_md5sum); - status = _s3->upload(_test_file + "_not_found", _s3_base_path + "/Ode_to_the_West_Wind1.txt"); - EXPECT_FALSE(status.ok()); - status = _s3->exist(_s3_base_path + "/Ode_to_the_West_Wind1.txt"); - EXPECT_TRUE(status.code() == NOT_FOUND); -} - -TEST_F(S3StorageBackendTest, s3_direct_upload) { - Status status = _s3->direct_upload(_s3_base_path + "/Ode_to_the_West_Wind.txt", _content); - EXPECT_TRUE(status.ok()); - status = _s3->exist(_s3_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - std::string orig_md5sum; - FileUtils::md5sum(_test_file, &orig_md5sum); - status = _s3->download(_s3_base_path + "/Ode_to_the_West_Wind.txt", _test_file + ".download"); - EXPECT_TRUE(status.ok()); - std::string download_md5sum; - FileUtils::md5sum(_test_file + ".download", &download_md5sum); - EXPECT_EQ(orig_md5sum, download_md5sum); -} - -TEST_F(S3StorageBackendTest, s3_download) { - Status status = _s3->upload(_test_file, _s3_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - std::string orig_md5sum; - FileUtils::md5sum(_test_file, &orig_md5sum); - status = _s3->download(_s3_base_path + "/Ode_to_the_West_Wind.txt", _test_file + ".download"); - EXPECT_TRUE(status.ok()); - std::string download_md5sum; - FileUtils::md5sum(_test_file + ".download", &download_md5sum); - EXPECT_EQ(orig_md5sum, download_md5sum); - status = _s3->download(_s3_base_path + "/Ode_to_the_West_Wind.txt.not_found", - _test_file + ".download"); - EXPECT_FALSE(status.ok()); - status = _s3->download(_s3_base_path + "/Ode_to_the_West_Wind.txt.not_found", - "/not_permitted.download"); - EXPECT_FALSE(status.ok()); -} - -TEST_F(S3StorageBackendTest, s3_rename) { - Status status = _s3->direct_upload(_s3_base_path + "/Ode_to_the_West_Wind.txt", _content); - EXPECT_TRUE(status.ok()); - status = _s3->rename(_s3_base_path + "/Ode_to_the_West_Wind.txt", - _s3_base_path + "/Ode_to_the_West_Wind.txt.new"); - EXPECT_TRUE(status.ok()); - status = _s3->exist(_s3_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.code() == NOT_FOUND); - status = _s3->exist(_s3_base_path + "/Ode_to_the_West_Wind.txt.new"); - EXPECT_TRUE(status.ok()); -} - -TEST_F(S3StorageBackendTest, s3_list) { - Status status = _s3->direct_upload(_s3_base_path + "/Ode_to_the_West_Wind.md5", _content); - EXPECT_TRUE(status.ok()); - status = _s3->direct_upload(_s3_base_path + "/Ode_to_the_West_Wind1.md5", _content); - EXPECT_TRUE(status.ok()); - status = _s3->direct_upload(_s3_base_path + "/Ode_to_the_West_Wind2.md5", _content); - EXPECT_TRUE(status.ok()); - std::map files; - status = _s3->list(_s3_base_path, true, false, &files); - EXPECT_TRUE(status.ok()); - EXPECT_TRUE(files.find("Ode_to_the_West_Wind") != files.end()); - EXPECT_TRUE(files.find("Ode_to_the_West_Wind1") != files.end()); - EXPECT_TRUE(files.find("Ode_to_the_West_Wind2") != files.end()); - EXPECT_EQ(3, files.size()); -} - -TEST_F(S3StorageBackendTest, s3_rm) { - Status status = _s3->direct_upload(_s3_base_path + "/Ode_to_the_West_Wind.txt", _content); - EXPECT_TRUE(status.ok()); - status = _s3->exist(_s3_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - status = _s3->rm(_s3_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.ok()); - status = _s3->exist(_s3_base_path + "/Ode_to_the_West_Wind.txt"); - EXPECT_TRUE(status.code() == NOT_FOUND); -} - -TEST_F(S3StorageBackendTest, s3_mkdir) { - Status status = _s3->mkdir(_s3_base_path + "/dir"); - EXPECT_TRUE(status.ok()); - status = _s3->exist(_s3_base_path + "/dir"); - EXPECT_TRUE(status.code() == NOT_FOUND); -} - -} // end namespace doris diff --git a/be/test/vec/exec/parquet/parquet_reader_test.cpp b/be/test/vec/exec/parquet/parquet_reader_test.cpp index 4ca7e55ecd..ec4eb12fed 100644 --- a/be/test/vec/exec/parquet/parquet_reader_test.cpp +++ b/be/test/vec/exec/parquet/parquet_reader_test.cpp @@ -91,8 +91,7 @@ TEST_F(ParquetReaderTest, normal) { auto slot_descs = desc_tbl->get_tuple_descriptor(0)->slots(); io::FileSystemSPtr local_fs = io::LocalFileSystem::create(""); io::FileReaderSPtr reader; - local_fs->open_file("./be/test/exec/test_data/parquet_scanner/type-decoder.parquet", &reader, - nullptr); + local_fs->open_file("./be/test/exec/test_data/parquet_scanner/type-decoder.parquet", &reader); cctz::time_zone ctz; TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); diff --git a/be/test/vec/exec/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/parquet/parquet_thrift_test.cpp index 64e93c3a3e..fd507b8e07 100644 --- a/be/test/vec/exec/parquet/parquet_thrift_test.cpp +++ b/be/test/vec/exec/parquet/parquet_thrift_test.cpp @@ -23,7 +23,7 @@ #include #include "exec/schema_scanner.h" -#include "io/buffered_reader.h" +#include "io/fs/buffered_reader.h" #include "io/fs/local_file_system.h" #include "olap/iterators.h" #include "runtime/descriptors.h" @@ -51,7 +51,7 @@ TEST_F(ParquetThriftReaderTest, normal) { io::FileSystemSPtr local_fs = io::LocalFileSystem::create(""); io::FileReaderSPtr reader; auto st = local_fs->open_file("./be/test/exec/test_data/parquet_scanner/localfile.parquet", - &reader, nullptr); + &reader); EXPECT_TRUE(st.ok()); std::shared_ptr meta_data; @@ -83,7 +83,7 @@ TEST_F(ParquetThriftReaderTest, complex_nested_file) { io::FileSystemSPtr local_fs = io::LocalFileSystem::create(""); io::FileReaderSPtr reader; auto st = local_fs->open_file("./be/test/exec/test_data/parquet_scanner/hive-complex.parquet", - &reader, nullptr); + &reader); EXPECT_TRUE(st.ok()); std::shared_ptr metadata; @@ -158,7 +158,7 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column ? chunk_meta.dictionary_page_offset : chunk_meta.data_page_offset; size_t chunk_size = chunk_meta.total_compressed_size; - BufferedFileStreamReader stream_reader(file_reader, start_offset, chunk_size, 1024); + io::BufferedFileStreamReader stream_reader(file_reader, start_offset, chunk_size, 1024); cctz::time_zone ctz; TimezoneUtils::find_cctz_time_zone(TimezoneUtils::default_time_zone, ctz); @@ -353,7 +353,7 @@ static void read_parquet_data_and_check(const std::string& parquet_file, io::FileSystemSPtr local_fs = io::LocalFileSystem::create(""); io::FileReaderSPtr reader; - auto st = local_fs->open_file(parquet_file, &reader, nullptr); + auto st = local_fs->open_file(parquet_file, &reader); EXPECT_TRUE(st.ok()); std::unique_ptr block; @@ -393,14 +393,13 @@ static void read_parquet_data_and_check(const std::string& parquet_file, } io::FileReaderSPtr result; - auto rst = local_fs->open_file(result_file, &result, nullptr); + auto rst = local_fs->open_file(result_file, &result); EXPECT_TRUE(rst.ok()); uint8_t result_buf[result->size() + 1]; result_buf[result->size()] = '\0'; size_t bytes_read; Slice res(result_buf, result->size()); - IOContext io_ctx; - result->read_at(0, res, io_ctx, &bytes_read); + result->read_at(0, res, &bytes_read); ASSERT_STREQ(block->dump_data(0, rows).c_str(), reinterpret_cast(result_buf)); } @@ -475,7 +474,7 @@ TEST_F(ParquetThriftReaderTest, group_reader) { io::FileSystemSPtr local_fs = io::LocalFileSystem::create(""); io::FileReaderSPtr file_reader; auto st = local_fs->open_file("./be/test/exec/test_data/parquet_scanner/type-decoder.parquet", - &file_reader, nullptr); + &file_reader); EXPECT_TRUE(st.ok()); // prepare metadata @@ -513,14 +512,13 @@ TEST_F(ParquetThriftReaderTest, group_reader) { io::FileReaderSPtr result; auto rst = local_fs->open_file("./be/test/exec/test_data/parquet_scanner/group-reader.txt", - &result, nullptr); + &result); EXPECT_TRUE(rst.ok()); uint8_t result_buf[result->size() + 1]; result_buf[result->size()] = '\0'; size_t bytes_read; Slice res(result_buf, result->size()); - IOContext io_ctx; - result->read_at(0, res, io_ctx, &bytes_read); + result->read_at(0, res, &bytes_read); ASSERT_STREQ(block.dump_data(0, 10).c_str(), reinterpret_cast(result_buf)); } } // namespace vectorized diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index dc0d8ec701..c25ce7b957 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -612,7 +612,9 @@ public class BackupJob extends AbstractJob { } long signature = env.getNextId(); UploadTask task = new UploadTask(null, beId, signature, jobId, dbId, srcToDest, - brokers.get(0), repo.getStorage().getProperties(), repo.getStorage().getStorageType()); + brokers.get(0), repo.getStorage().getProperties(), repo.getStorage().getStorageType(), + repo.getLocation()); + LOG.info("yy debug upload location: " + repo.getLocation()); batchTask.addTask(task); unfinishedTaskIds.put(signature, beId); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 1fd3de6352..84fdbda91a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1377,7 +1377,7 @@ public class RestoreJob extends AbstractJob { long signature = env.getNextId(); DownloadTask task = new DownloadTask(null, beId, signature, jobId, dbId, srcToDest, brokerAddrs.get(0), repo.getStorage().getProperties(), - repo.getStorage().getStorageType()); + repo.getStorage().getStorageType(), repo.getLocation()); batchTask.addTask(task); unfinishedSignatureToId.put(signature, beId); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/DownloadTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/DownloadTask.java index 46fd73315c..133614e964 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/DownloadTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/DownloadTask.java @@ -33,16 +33,18 @@ public class DownloadTask extends AgentTask { private FsBroker brokerAddr; private Map brokerProperties; private StorageBackend.StorageType storageType; + private String location; public DownloadTask(TResourceInfo resourceInfo, long backendId, long signature, long jobId, long dbId, Map srcToDestPath, FsBroker brokerAddr, Map brokerProperties, - StorageBackend.StorageType storageType) { + StorageBackend.StorageType storageType, String location) { super(resourceInfo, backendId, TTaskType.DOWNLOAD, dbId, -1, -1, -1, -1, signature); this.jobId = jobId; this.srcToDestPath = srcToDestPath; this.brokerAddr = brokerAddr; this.brokerProperties = brokerProperties; this.storageType = storageType; + this.location = location; } public long getJobId() { @@ -66,6 +68,7 @@ public class DownloadTask extends AgentTask { TDownloadReq req = new TDownloadReq(jobId, srcToDestPath, address); req.setBrokerProp(brokerProperties); req.setStorageBackend(storageType.toThrift()); + req.setLocation(location); return req; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/UploadTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/UploadTask.java index bcbb9a58d9..a66a6b2b48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/UploadTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/UploadTask.java @@ -34,16 +34,18 @@ public class UploadTask extends AgentTask { private FsBroker broker; private Map brokerProperties; private StorageBackend.StorageType storageType; + private String location; public UploadTask(TResourceInfo resourceInfo, long backendId, long signature, long jobId, Long dbId, Map srcToDestPath, FsBroker broker, Map brokerProperties, - StorageBackend.StorageType storageType) { + StorageBackend.StorageType storageType, String location) { super(resourceInfo, backendId, TTaskType.UPLOAD, dbId, -1, -1, -1, -1, signature); this.jobId = jobId; this.srcToDestPath = srcToDestPath; this.broker = broker; this.brokerProperties = brokerProperties; this.storageType = storageType; + this.location = location; } public long getJobId() { @@ -67,6 +69,7 @@ public class UploadTask extends AgentTask { TUploadReq request = new TUploadReq(jobId, srcToDestPath, address); request.setBrokerProp(brokerProperties); request.setStorageBackend(storageType.toThrift()); + request.setLocation(location); return request; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java index 9b0afea6ac..50d39e0ab6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupHandlerTest.java @@ -272,7 +272,7 @@ public class BackupHandlerTest { // handleFinishedSnapshotUploadTask Map srcToDestPath = Maps.newHashMap(); UploadTask uploadTask = new UploadTask(null, 0, 0, backupJob.getJobId(), CatalogMocker.TEST_DB_ID, - srcToDestPath, null, null, StorageBackend.StorageType.BROKER); + srcToDestPath, null, null, StorageBackend.StorageType.BROKER, ""); request = new TFinishTaskRequest(); Map> tabletFiles = Maps.newHashMap(); request.setTabletFiles(tabletFiles); @@ -320,7 +320,7 @@ public class BackupHandlerTest { // handleDownloadSnapshotTask DownloadTask downloadTask = new DownloadTask(null, 0, 0, restoreJob.getJobId(), CatalogMocker.TEST_DB_ID, - srcToDestPath, null, null, StorageBackend.StorageType.BROKER); + srcToDestPath, null, null, StorageBackend.StorageType.BROKER, ""); request = new TFinishTaskRequest(); List downloadedTabletIds = Lists.newArrayList(); request.setDownloadedTabletIds(downloadedTabletIds); diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 88b7ec5c75..1bc35c84ee 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -274,7 +274,7 @@ struct TUploadReq { 3: required Types.TNetworkAddress broker_addr 4: optional map broker_prop 5: optional Types.TStorageBackendType storage_backend = Types.TStorageBackendType.BROKER - + 6: optional string location // root path } struct TDownloadReq { @@ -283,6 +283,7 @@ struct TDownloadReq { 3: required Types.TNetworkAddress broker_addr 4: optional map broker_prop 5: optional Types.TStorageBackendType storage_backend = Types.TStorageBackendType.BROKER + 6: optional string location // root path } struct TSnapshotRequest {