FIX: move crawler blocking to app controller

We need access to site settings in multisite, we do not have access
yet if we attempt to get them in request tracker middleware

lib/middleware/request_tracker.rb

@@ -175,12 +175,6 @@ class Middleware::RequestTracker
       return result
     end
 
-    # if block_crawler(request)
-    #   log_request = false
-    #   result = [403, { 'Content-Type' => 'text/plain' }, ["Crawler is not allowed."]]
-    #   return result
-    # end
-
     env["discourse.request_tracker"] = self
     MethodProfiler.start
     result = @app.call(env)
@@ -287,13 +281,6 @@ class Middleware::RequestTracker
     end
   end
 
-  def block_crawler(request)
-    request.get? &&
-      !request.xhr? &&
-      !request.path.ends_with?('robots.txt') &&
-      CrawlerDetection.is_blocked_crawler?(request.env['HTTP_USER_AGENT'])
-  end
-
   def log_later(data, host)
     Scheduler::Defer.later("Track view", _db = nil) do
       self.class.log_request_on_site(data, host)