database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[bug](cooldown) Fix the issue of unused remote files not being deleted (#19785)

Browse Source
This commit is contained in:
plat1ko
2023-06-12 21:05:09 +08:00
committed by GitHub
parent 1433544c56
commit 283c55720d
3 changed files with 10 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
be/src/io/fs/s3_file_system.cpp
View File

@ -320,13 +320,11 @@ Status S3FileSystem::list_impl(const Path& dir, bool only_file, std::vector<File
}
for (const auto& obj : outcome.GetResult().GetContents()) {
std::string key = obj.GetKey();
bool is_dir = (key.at(key.size() - 1) == '/');
bool is_dir = (key.back() == '/');
if (only_file && is_dir) {
continue;
}
FileInfo file_info;
// note: if full path is s3://bucket/path/to/file.txt
// obj.GetKey() will be /path/to/file.txt
file_info.file_name = obj.GetKey().substr(prefix.size());
file_info.file_size = obj.GetSize();
file_info.is_file = !is_dir;

16
be/src/olap/tablet.cpp
View File

@ -2485,23 +2485,23 @@ void Tablet::remove_unused_remote_files() {
if (auto it = buffer.find(id); LIKELY(it != buffer.end())) {
auto& fs = it->second.first;
auto& files = it->second.second;
std::vector<io::Path> paths;
paths.reserve(files.size());
// delete unused files
LOG(INFO) << "delete unused files. root_path=" << fs->root_path()
<< " tablet_id=" << id;
io::Path dir("data/" + std::to_string(id));
io::Path dir = remote_tablet_path(id);
for (auto& file : files) {
auto delete_path = dir / io::Path(file.file_name);
LOG(INFO) << "delete unused file: " << delete_path.native();
auto file_path = dir / file.file_name;
LOG(INFO) << "delete unused file: " << file_path.native();
paths.push_back(std::move(file_path));
}
std::vector<io::Path> file_names;
for (auto& info : files) {
file_names.emplace_back(info.file_name);
}
st = fs->batch_delete(file_names);
st = fs->batch_delete(paths);
if (!st.ok()) {
LOG(WARNING) << "failed to delete unused files, tablet_id=" << id << " : "
<< st;
}
buffer.erase(it);
}
}
};

2
be/src/olap/tablet_manager.cpp
View File

@ -856,7 +856,7 @@ Status TabletManager::load_tablet_from_dir(DataDir* store, TTabletId tablet_id,
}
TabletMetaSharedPtr tablet_meta(new TabletMeta());
if (tablet_meta->create_from_file(header_path) != Status::OK()) {
if (!tablet_meta->create_from_file(header_path).ok()) {
LOG(WARNING) << "fail to load tablet_meta. file_path=" << header_path;
return Status::Error<ENGINE_LOAD_INDEX_TABLE_ERROR>();
}