database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](file-system)(step-1) refactor file sysmte on BE and remove storage_backend (#17586)

Browse Source 
See #17764 for details
I have tested:
- Unit test for local/s3/hdfs/broker file system: be/test/io/fs/file_system_test.cpp
- Outfile to local/s3/hdfs/broker.
- Load from local/s3/hdfs/broker.
- Query file on local/s3/hdfs/broker file system, with table value function and catalog.
- Backup/Restore with local/s3/hdfs/broker file system

Not test:
- cold & host data separation case.
This commit is contained in:
Mingyu Chen
2023-03-21 21:08:38 +08:00
committed by GitHub
parent 82716ec99d
commit cb79e42e5c
171 changed files with 3523 additions and 5288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
be/src/olap/utils.cpp
View File

@ -33,7 +33,6 @@
#include <string>
#include <vector>
#include "olap/file_helper.h"
#include "util/file_utils.h"
#ifdef DORIS_WITH_LZO
@ -47,6 +46,9 @@
#include "common/status.h"
#include "env/env.h"
#include "gutil/strings/substitute.h"
#include "io/fs/file_reader.h"
#include "io/fs/file_writer.h"
#include "io/fs/local_file_system.h"
#include "olap/olap_common.h"
#include "olap/olap_define.h"
#include "util/errno.h"
@ -449,13 +451,7 @@ Status read_write_test_file(const string& test_file_path) {
return Status::Error<IO_ERROR>();
}
}
Status res = Status::OK();
FileHandler file_handler;
if ((res = file_handler.open_with_mode(test_file_path.c_str(), O_RDWR | O_CREAT | O_SYNC,
S_IRUSR | S_IWUSR)) != Status::OK()) {
LOG(WARNING) << "fail to create test file. path=" << test_file_path;
return res;
}
const size_t TEST_FILE_BUF_SIZE = 4096;
const size_t DIRECT_IO_ALIGNMENT = 512;
char* write_test_buff = nullptr;
@ -476,30 +472,24 @@ Status read_write_test_file(const string& test_file_path) {
int32_t tmp_value = rand_r(&rand_seed);
write_test_buff[i] = static_cast<char>(tmp_value);
}
if (!(res = file_handler.pwrite(write_buff.get(), TEST_FILE_BUF_SIZE, SEEK_SET))) {
LOG(WARNING) << "fail to write test file. [file_name=" << test_file_path << "]";
return res;
}
if ((res = file_handler.pread(read_buff.get(), TEST_FILE_BUF_SIZE, SEEK_SET)) != Status::OK()) {
LOG(WARNING) << "fail to read test file. [file_name=" << test_file_path << "]";
return res;
}
// write file
io::FileWriterPtr file_writer;
RETURN_IF_ERROR(io::global_local_filesystem()->create_file(test_file_path, &file_writer));
RETURN_IF_ERROR(file_writer->append({write_buff.get(), TEST_FILE_BUF_SIZE}));
RETURN_IF_ERROR(file_writer->close());
// read file
io::FileReaderSPtr file_reader;
RETURN_IF_ERROR(io::global_local_filesystem()->open_file(test_file_path, &file_reader));
size_t bytes_read = 0;
RETURN_IF_ERROR(file_reader->read_at(0, {read_buff.get(), TEST_FILE_BUF_SIZE}, &bytes_read));
if (memcmp(write_buff.get(), read_buff.get(), TEST_FILE_BUF_SIZE) != 0) {
LOG(WARNING) << "the test file write_buf and read_buf not equal, [file_name = "
<< test_file_path << "]";
return Status::Error<TEST_FILE_ERROR>();
}
if ((res = file_handler.close()) != Status::OK()) {
LOG(WARNING) << "fail to close test file. [file_name=" << test_file_path << "]";
return res;
}
if (remove(test_file_path.c_str()) != 0) {
char errmsg[64];
VLOG_NOTICE << "fail to delete test file. [err='" << strerror_r(errno, errmsg, 64)
<< "' path='" << test_file_path << "']";
return Status::Error<IO_ERROR>();
}
return res;
// delete file
return io::global_local_filesystem()->delete_file(test_file_path);
}
bool check_datapath_rw(const string& path) {