DEV: ensure Rails application default headers are present in responses (#31619)

Follow up from https://github.com/discourse/discourse/pull/31559.

We expect some standard headers to be added from
`Rails.application.config.action_dispatch.default_headers` for
responses, however these were found to be removed in some error paths. 
For more detail on this behaviour, refer to https://github.com/discourse/discourse/pull/31619#issuecomment-2699644232.

This PR adds those headers back if they aren't there, with the caveats
that we don't add headers that are irrelevant for non-HTML responses,
and neither do we add X-Frame-Options which is intentionally removed for
embeddables.

lib/middleware/default_headers.rb

@@ -2,15 +2,29 @@
 
 module Middleware
   class DefaultHeaders
+    HTML_ONLY_HEADERS = Set.new(%w[X-XSS-Protection])
+    EXCLUDED_HEADERS = Set.new(%w[X-Frame-Options])
+
     def initialize(app, settings = {})
       @app = app
     end
 
     def call(env)
       status, headers, body = @app.call(env)
+      is_html_response = html_response?(headers)
+
+      default_headers =
+        Rails.application.config.action_dispatch.default_headers.to_h.except(*EXCLUDED_HEADERS)
+
+      default_headers.each do |header_name, value|
+        next if !is_html_response && HTML_ONLY_HEADERS.include?(header_name)
+
+        headers[header_name] ||= value
+      end
+
       headers[
         "Cross-Origin-Opener-Policy"
-      ] = SiteSetting.cross_origin_opener_policy_header if html_response?(headers) &&
+      ] = SiteSetting.cross_origin_opener_policy_header if is_html_response &&
         headers["Cross-Origin-Opener-Policy"].nil?
 
       [status, headers, body]

spec/requests/default_headers_spec.rb

@@ -1,5 +1,23 @@
 # frozen_string_literal: true
 RSpec.describe Middleware::DefaultHeaders do
+  let(:mock_default_headers) do
+    {
+      "X-XSS-Protection" => "0",
+      "X-Content-Type-Options" => "nosniff",
+      "X-Permitted-Cross-Domain-Policies" => "none",
+      "Referrer-Policy" => "strict-origin-when-cross-origin",
+    }
+  end
+
+  let(:html_only_headers) { described_class::HTML_ONLY_HEADERS }
+  let(:universal_headers) { Set.new(mock_default_headers.keys) - html_only_headers }
+
+  before do
+    allow(Rails.application.config.action_dispatch).to receive(:default_headers).and_return(
+      mock_default_headers,
+    )
+  end
+
   context "when a public exception(like RoutingError) is raised" do
     context "when requesting an HTML page" do
       let(:html_path) { "/nonexistent" }
@@ -10,15 +28,24 @@ RSpec.describe Middleware::DefaultHeaders do
         expect(response.headers).to have_key("Cross-Origin-Opener-Policy")
         expect(response.headers["Cross-Origin-Opener-Policy"]).to eq("same-origin-allow-popups")
       end
+
+      it "sets all default Rails headers for HTML responses" do
+        get html_path
+
+        mock_default_headers.each { |name, value| expect(response.headers[name]).to eq(value) }
+      end
     end
 
     context "when requesting a JSON response for an invalid URL" do
       let(:json_path) { "/nonexistent.json" }
 
-      it "does not include the Cross-Origin-Opener-Policy header" do
-        SiteSetting.bootstrap_error_pages = true
-        SiteSetting.cross_origin_opener_policy_header = "same-origin"
+      it "adds only universal default headers to non-HTML responses" do
         get json_path
+
+        universal_headers.each do |name|
+          expect(response.headers[name]).to eq(mock_default_headers[name])
+        end
+        html_only_headers.each { |name| expect(response.headers[name]).to be_nil }
         expect(response.headers["Cross-Origin-Opener-Policy"]).to be_nil
       end
     end
@@ -33,7 +60,7 @@ RSpec.describe Middleware::DefaultHeaders do
 
     after { Rails.logger = @old_logger }
 
-    it "should not raise a 500 (nor should it log a warning) for bad params" do
+    it "adds default headers to the response" do
       bad_str = (+"d\xDE").force_encoding("utf-8")
       expect(bad_str.valid_encoding?).to eq(false)
 
@@ -45,6 +72,7 @@ RSpec.describe Middleware::DefaultHeaders do
       expect(response.status).to eq(400)
       expect(response.headers).to have_key("Cross-Origin-Opener-Policy")
       expect(response.headers["Cross-Origin-Opener-Policy"]).to eq("same-origin-allow-popups")
+      mock_default_headers.each { |name, value| expect(response.headers[name]).to eq(value) }
     end
   end
 end