FIX: Apply onebox blocked domain checks on every redirect (#16150)

The `blocked onebox domains` setting lets site owners change what sites
are allowed to be oneboxed. When a link is entered into a post,
Discourse checks the domain of the link against that setting and blocks
the onebox if the domain is blocked. But if there's a chain of
redirects, then only the final destination website is checked against
the site setting.

This commit amends that behavior so that every website in the redirect
chain is checked against the site setting, and if anything is blocked
the original link doesn't onebox at all in the post. The
`Discourse-No-Onebox` header is also checked in every response and the
onebox is blocked if the header is set to "1".

Additionally, Discourse will now include the `Discourse-No-Onebox`
header with every response if the site requires login to access content.
This is done to signal to a Discourse instance that it shouldn't attempt
to onebox other Discourse instances if they're login-only. Non-Discourse
websites can also use include that header if they don't wish to have
Discourse onebox their content.

Internal ticket: t59305.

lib/final_destination.rb

@@ -74,6 +74,7 @@ class FinalDestination
     @preserve_fragment_url = @preserve_fragment_url_hosts.any? { |host| hostname_matches?(host) }
     @validate_uri = @opts.fetch(:validate_uri) { true }
     @user_agent = @force_custom_user_agent_hosts.any? { |host| hostname_matches?(host) } ? Onebox.options.user_agent : @default_user_agent
+    @stop_at_blocked_pages = @opts[:stop_at_blocked_pages]
   end
 
   def self.connection_timeout
@@ -140,9 +141,12 @@ class FinalDestination
       uri = URI(uri.to_s)
     end
 
-    return nil unless validate_uri
+    return if !validate_uri
+    return if @stop_at_blocked_pages && blocked_domain?(uri)
 
-    result, (location, cookie) = safe_get(uri, &blk)
+    result, headers_subset = safe_get(uri, &blk)
+    cookie = headers_subset.set_cookie
+    location = headers_subset.location
 
     if result == :redirect && (redirects == 0 || !location)
       return nil
@@ -222,6 +226,13 @@ class FinalDestination
       response_block: request_validator
     )
 
+    if @stop_at_blocked_pages
+      if blocked_domain?(@uri) || response.headers['Discourse-No-Onebox'] == "1"
+        @status = :blocked_page
+        return
+      end
+    end
+
     location = nil
     response_headers = nil
     response_status = response.status.to_i
@@ -253,6 +264,18 @@ class FinalDestination
     when 103, 400, 405, 406, 409, 500, 501
       response_status, small_headers = small_get(request_headers)
 
+      if @stop_at_blocked_pages
+        # this may seem weird, but the #to_hash method of the response object
+        # of ruby's net/http lib returns a hash where each value is an array.
+        # small_headers here is like that so our no onebox header value is an
+        # array if it's set. Also the hash keys are always lower-cased.
+        dont_onebox = small_headers["discourse-no-onebox"]&.join("") == "1"
+        if dont_onebox || blocked_domain?(@uri)
+          @status = :blocked_page
+          return
+        end
+      end
+
       if response_status == 200
         @status = :resolved
         return @uri
@@ -425,6 +448,7 @@ class FinalDestination
   def safe_get(uri)
     result = nil
     unsafe_close = false
+    headers_subset = Struct.new(:location, :set_cookie).new
 
     safe_session(uri) do |http|
       headers = request_headers.merge(
@@ -435,8 +459,19 @@ class FinalDestination
       req = Net::HTTP::Get.new(uri.request_uri, headers)
 
       http.request(req) do |resp|
+        headers_subset.set_cookie = resp['Set-Cookie']
+
+        if @stop_at_blocked_pages
+          dont_onebox = resp["Discourse-No-Onebox"] == "1"
+          if dont_onebox
+            result = :blocked, headers_subset
+            next
+          end
+        end
+
         if Net::HTTPRedirection === resp
-          result = :redirect, [resp['location'], resp['Set-Cookie']]
+          headers_subset.location = resp['location']
+          result = :redirect, headers_subset
         end
 
         if Net::HTTPSuccess === resp
@@ -460,7 +495,7 @@ class FinalDestination
               raise StandardError
             end
           end
-          result = :ok
+          result = :ok, headers_subset
         else
           catch(:done) do
             yield resp, nil, nil
@@ -471,7 +506,7 @@ class FinalDestination
 
     result
   rescue StandardError
-    unsafe_close ? :ok : raise
+    unsafe_close ? [:ok, headers_subset] : raise
   end
 
   def safe_session(uri)
@@ -505,4 +540,8 @@ class FinalDestination
 
     uri(complete_url)
   end
+
+  def blocked_domain?(uri)
+    Onebox::DomainChecker.is_blocked?(uri.hostname)
+  end
 end