database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[log](gc) add log for unused rowsets gc (#30854)

Browse Source
This commit is contained in:
zhannngchen
2024-02-06 17:37:13 +08:00
committed by yiguolei
parent b39cee9c16
commit 5b343911e8
2 changed files with 20 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
be/src/olap/data_dir.cpp
View File

@ -736,7 +736,7 @@ void DataDir::_perform_path_gc_by_rowset(const std::vector<std::string>& tablet_
bool is_valid = doris::TabletManager::get_tablet_id_and_schema_hash_from_path(
path, &tablet_id, &schema_hash);
if (!is_valid || tablet_id < 1 || schema_hash < 1) [[unlikely]] {
LOG(WARNING) << "unknown path:" << path;
LOG(WARNING) << "[path gc] unknown path:" << path;
continue;
}
@ -757,7 +757,7 @@ void DataDir::_perform_path_gc_by_rowset(const std::vector<std::string>& tablet_
std::vector<io::FileInfo> files;
auto st = io::global_local_filesystem()->list(path, true, &files, &exists);
if (!st.ok()) [[unlikely]] {
LOG(WARNING) << "fail to list tablet path " << path << " : " << st;
LOG(WARNING) << "[path gc] fail to list tablet path " << path << " : " << st;
continue;
}
@ -786,10 +786,10 @@ void DataDir::_perform_path_gc_by_rowset(const std::vector<std::string>& tablet_
auto reclaim_rowset_file = [](const std::string& path) {
auto st = io::global_local_filesystem()->delete_file(path);
if (!st.ok()) [[unlikely]] {
LOG(WARNING) << "failed to delete garbage rowset file: " << st;
LOG(WARNING) << "[path gc] failed to delete garbage rowset file: " << st;
return;
}
LOG(INFO) << "delete garbage path: " << path; // Audit log
LOG(INFO) << "[path gc] delete garbage path: " << path; // Audit log
};
auto should_reclaim = [&, this](const RowsetId& rowset_id) {

16
be/src/olap/storage_engine.cpp
View File

@ -1074,8 +1074,12 @@ void StorageEngine::_parse_default_rowset_type() {
}
void StorageEngine::start_delete_unused_rowset() {
LOG(INFO) << "start to delete unused rowset, size: " << _unused_rowsets.size();
std::vector<RowsetSharedPtr> unused_rowsets_copy;
unused_rowsets_copy.reserve(_unused_rowsets.size());
auto due_to_use_count = 0;
auto due_to_not_delete_file = 0;
auto due_to_delayed_expired_ts = 0;
{
std::lock_guard<std::mutex> lock(_gc_mutex);
for (auto it = _unused_rowsets.begin(); it != _unused_rowsets.end();) {
@ -1092,9 +1096,20 @@ void StorageEngine::start_delete_unused_rowset() {
it = _unused_rowsets.erase(it);
} else {
++it;
if (rs.use_count() != 1) {
++due_to_use_count;
} else if (!rs->need_delete_file()) {
++due_to_not_delete_file;
} else {
++due_to_delayed_expired_ts;
}
}
}
}
LOG(INFO) << "collected " << unused_rowsets_copy.size() << " unused rowsets to remove, skipped "
<< due_to_use_count << " rowsets due to use count > 1, skipped "
<< due_to_not_delete_file << " rowsets due to don't need to delete file, skipped "
<< due_to_delayed_expired_ts << " rowsets due to delayed expired timestamp.";
for (auto&& rs : unused_rowsets_copy) {
VLOG_NOTICE << "start to remove rowset:" << rs->rowset_id()
<< ", version:" << rs->version();
@ -1107,6 +1122,7 @@ void StorageEngine::start_delete_unused_rowset() {
Status status = rs->remove();
VLOG_NOTICE << "remove rowset:" << rs->rowset_id() << " finished. status:" << status;
}
LOG(INFO) << "removed all collected unused rowsets";
}
void StorageEngine::add_unused_rowset(RowsetSharedPtr rowset) {