FIX: move crawler blocking into anon cache

This refinement of previous fix moves the crawler blocking into
anonymous cache

This ensures we never poison the cache incorrectly when blocking crawlers

lib/middleware/anonymous_cache.rb

@@ -21,6 +21,13 @@ module Middleware
         @request = Rack::Request.new(@env)
       end
 
+      def blocked_crawler?
+        @request.get? &&
+        !@request.xhr? &&
+        !@request.path.ends_with?('robots.txt') &&
+        CrawlerDetection.is_blocked_crawler?(@request.env['HTTP_USER_AGENT'])
+      end
+
       def is_mobile=(val)
         @is_mobile = val ? :true : :false
       end
@@ -188,6 +195,11 @@ module Middleware
       helper = Helper.new(env)
       force_anon = false
 
+      if helper.blocked_crawler?
+        env["discourse.request_tracker.skip"] = true
+        return [403, {}, "Crawler is not allowed!"]
+      end
+
       if helper.should_force_anonymous?
         force_anon = env["DISCOURSE_FORCE_ANON"] = true
         helper.force_anonymous!