FIX: Apply crawler rate limits to cached requests (#27174)

This commit moves the logic for crawler rate limits out of the application controller and into the request tracker middleware. The reason for this move is to apply rate limits to all crawler requests instead of just the requests that make it to the application controller. Some requests are served early from the middleware stack without reaching the Rails app for performance reasons (e.g. `AnonymousCache`) which results in crawlers getting 200 responses even though they've reached their limits and should be getting 429 responses.

Internal topic: t/128810.

lib/middleware/request_tracker.rb

@@ -267,6 +267,20 @@ class Middleware::RequestTracker
       end
       return 429, headers, [message]
     end
+
+    if !cookie
+      if error_details = check_crawler_limits(env)
+        available_in, error_code = error_details
+        message = "Too many crawling requests. Error code: #{error_code}."
+        headers = {
+          "Content-Type" => "text/plain",
+          "Retry-After" => available_in.to_s,
+          "Discourse-Rate-Limit-Error-Code" => error_code,
+        }
+        return 429, headers, [message]
+      end
+    end
+
     env["discourse.request_tracker"] = self
 
     MethodProfiler.start
@@ -443,4 +457,30 @@ class Middleware::RequestTracker
       end
     end
   end
+
+  def check_crawler_limits(env)
+    slow_down_agents = SiteSetting.slow_down_crawler_user_agents
+    return if slow_down_agents.blank?
+
+    user_agent = env["HTTP_USER_AGENT"]&.downcase
+    return if user_agent.blank?
+
+    return if !CrawlerDetection.crawler?(user_agent)
+
+    slow_down_agents
+      .downcase
+      .split("|")
+      .each do |crawler|
+        if user_agent.include?(crawler)
+          key = "#{crawler}_crawler_rate_limit"
+          limiter =
+            RateLimiter.new(nil, key, 1, SiteSetting.slow_down_crawler_rate, error_code: key)
+          limiter.performed!
+          break
+        end
+      end
+    nil
+  rescue RateLimiter::LimitExceeded => e
+    [e.available_in, e.error_code]
+  end
 end