FIX: handle more thread pool edge cases (#30392)

* Split `shutdown` into two separate methods for better control:
  - `shutdown` - signals threads to stop accepting new work
  - `wait_for_termination` - waits for threads to finish (with optional timeout)

* Add tracking of busy threads via `@busy_threads` Set
* Make idle_time parameter optional with 30-second default
* Improve thread spawning logic:
  - Spawn initial thread immediately when work is posted
  - Spawn additional threads when all threads are busy and work is queued
* Fix race condition in work distribution
* Add busy thread count to stats output
* Add test coverage for zero min_threads configuration

This commit makes the ThreadPool more reliable, easier to use, and adds 
better visibility into its internal state.

---------

Co-authored-by: Alan Guo Xiang Tan <gxtan1990@gmail.com>

spec/lib/scheduler/thread_pool_spec.rb

@@ -9,7 +9,10 @@ RSpec.describe Scheduler::ThreadPool, type: :multisite do
     described_class.new(min_threads: min_threads, max_threads: max_threads, idle_time: idle_time)
   end
 
-  after { pool.shutdown(timeout: 1) }
+  after do
+    pool.shutdown
+    pool.wait_for_termination(timeout: 1)
+  end
 
   describe "initialization" do
     it "creates the minimum number of threads and validates parameters" do
@@ -70,7 +73,12 @@ RSpec.describe Scheduler::ThreadPool, type: :multisite do
         end
       end
 
+      # we spin up threads in the thread loop, so it can take
+      # a bit of time to react to work pressure
+      wait_for { pool.stats[:thread_count] == max_threads }
+
       expect(pool.stats[:thread_count]).to eq(max_threads)
+
       (max_threads + 1).times { blocker_queue << :continue }
 
       results = Array.new(max_threads + 1) { completion_queue.pop }
@@ -119,7 +127,8 @@ RSpec.describe Scheduler::ThreadPool, type: :multisite do
         results2 = Array.new(3) { completion_queue2.pop }
       end
 
-      pool.shutdown(timeout: 1)
+      pool.shutdown
+      pool.wait_for_termination(timeout: 1)
 
       expect(results1.size).to eq(3)
       expect(results1.sort).to eq([0, 1, 2])
@@ -165,6 +174,24 @@ RSpec.describe Scheduler::ThreadPool, type: :multisite do
     end
   end
 
+  describe "when thread pool has zero min threads" do
+    it "can quickly process and can be cleanly terminated" do
+      # setting idle time to 1000 to ensure that there are maximal delays waiting
+      # for jobs
+      pool = Scheduler::ThreadPool.new(min_threads: 0, max_threads: 5, idle_time: 1000)
+
+      done = Queue.new
+      pool.post { done << :done }
+
+      # should happen in less than 1 second
+      Timeout.timeout(1) { expect(done.pop).to eq(:done) }
+
+      pool.shutdown
+      pool.wait_for_termination
+      expect(pool.stats[:thread_count]).to eq(0)
+    end
+  end
+
   describe "stress test" do
     it "handles multiple task submissions correctly" do
       completion_queue = Queue.new