From f8fd8a3d17c7fb6fc07ab7d4901019fcc9b39d1b Mon Sep 17 00:00:00 2001
From: yujun <yu.jun.reach@gmail.com>
Date: Fri, 8 Sep 2023 18:13:22 +0800
Subject: [PATCH] [fix](trash) fix clean trash not working (#23936)

When executing admin clean trash, if the backend daemon clean thread is cleaning trash, then SQL command will return immediately. But for the backend daemon thread, it doesn't clean all the trashes, it clean only the expired trashes.
Also if there's lots of trashes, the daemon clean thread will busy handling trashes for a long time.
---
 be/src/olap/olap_server.cpp    | 4 ++++
 be/src/olap/storage_engine.cpp | 5 ++++-
 be/src/olap/storage_engine.h   | 2 ++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index e9219d84ac..89d7e67f09 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -296,6 +296,10 @@ void StorageEngine::_garbage_sweeper_thread_callback() {
 
         // start clean trash and update usage.
         Status res = start_trash_sweep(&usage);
+        if (res.ok() && _need_clean_trash.exchange(false, std::memory_order_relaxed)) {
+            res = start_trash_sweep(&usage, true);
+        }
+
         if (!res.ok()) {
             LOG(WARNING) << "one or more errors occur when sweep trash."
                          << "see previous message for detail. err code=" << res;
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 07d980fd02..b425076fc0 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -634,10 +634,13 @@ Status StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) {
     std::unique_lock<std::mutex> l(_trash_sweep_lock, std::defer_lock);
     if (!l.try_lock()) {
         LOG(INFO) << "trash and snapshot sweep is running.";
+        if (ignore_guard) {
+            _need_clean_trash.store(true, std::memory_order_relaxed);
+        }
         return res;
     }
 
-    LOG(INFO) << "start trash and snapshot sweep.";
+    LOG(INFO) << "start trash and snapshot sweep. is_clean=" << ignore_guard;
 
     const int32_t snapshot_expire = config::snapshot_expire_time_sec;
     const int32_t trash_expire = config::trash_file_expire_time_sec;
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index f738da02f4..02e6ec5f7b 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -484,6 +484,8 @@ private:
 
     bool _clear_segment_cache = false;
 
+    std::atomic<bool> _need_clean_trash {false};
+
     DISALLOW_COPY_AND_ASSIGN(StorageEngine);
 };