diff --git a/app/controllers/robots_txt_controller.rb b/app/controllers/robots_txt_controller.rb
index c3498b11644..4b80c0ba3f8 100644
--- a/app/controllers/robots_txt_controller.rb
+++ b/app/controllers/robots_txt_controller.rb
@@ -21,7 +21,7 @@ class RobotsTxtController < ApplicationController
     /*?*api_key*
   ]
 
-  DISALLOWED_WITH_HEADER_PATHS = %w[/badges /u/ /my /search /tag/*/l /g /t/*/*.rss /c/*.rss]
+  DISALLOWED_WITH_HEADER_PATHS = %w[/badges /my /search /tag/*/l /g /t/*/*.rss /c/*.rss]
 
   def index
     if (overridden = SiteSetting.overridden_robots_txt.dup).present?
diff --git a/db/migrate/20250115031117_remove_user_profile_from_overridden_robots.rb b/db/migrate/20250115031117_remove_user_profile_from_overridden_robots.rb
new file mode 100644
index 00000000000..fa9f8fc56ea
--- /dev/null
+++ b/db/migrate/20250115031117_remove_user_profile_from_overridden_robots.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+class RemoveUserProfileFromOverriddenRobots < ActiveRecord::Migration[7.2]
+  def up
+    all_agent_user_disallow =
+      "User-agent: *\nDisallow: /admin/\nDisallow: /auth/\nDisallow: /assets/browser-update*.js\nDisallow: /email/\nDisallow: /session\nDisallow: /user-api-key\nDisallow: /*?api_key*\nDisallow: /*?*api_key*\nDisallow: /badges\nDisallow: /u/"
+    all_agent_user_disallow_removed = all_agent_user_disallow.gsub("\nDisallow: /u/", "")
+    if select_value(
+         "SELECT value FROM site_settings WHERE name = 'overridden_robots_txt' AND value LIKE '%#{all_agent_user_disallow}%'",
+       )
+      execute <<~SQL
+        UPDATE site_settings
+        SET value = REPLACE(value, '#{all_agent_user_disallow}', '#{all_agent_user_disallow_removed}')
+        WHERE name = 'overridden_robots_txt'
+      SQL
+    end
+  end
+
+  def down
+    # raise ActiveRecord::IrreversibleMigration
+  end
+end