github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-30 07:11:34 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

prefix the robots.txt rules with the directory when using subfolder

Browse Source
This commit is contained in:
Régis Hanol
2018-04-11 22:05:02 +02:00
parent 3c8b43bb01
commit df7970a6f6
4 changed files with 49 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
app/controllers/robots_txt_controller.rb
View File

@ -2,14 +2,46 @@ class RobotsTxtController < ApplicationController
layout false
skip_before_action :preload_json, :check_xhr, :redirect_to_login_if_required
# NOTE: order is important!
DISALLOWED_PATHS ||= %w{
/auth/cas
/auth/facebook/callback
/auth/twitter/callback
/auth/google/callback
/auth/yahoo/callback
/auth/github/callback
/auth/cas/callback
/assets/browser-update*.js
/users/
/u/
/badges/
/search
/search/
/tags
/tags/
/email/
/session
/session/
/admin
/admin/
/user-api-key
/user-api-key/
/*?api_key*
/*?*api_key*
/groups
/groups/
/t/*/*.rss
/tags/*.rss
/c/*.rss
}
def index
if SiteSetting.allow_index_in_robots_txt
path = :index
@crawler_delayed_agents = []
SiteSetting.slow_down_crawler_user_agents.split('|').each do |agent|
@crawler_delayed_agents << [agent, SiteSetting.slow_down_crawler_rate]
end
@crawler_delayed_agents = SiteSetting.slow_down_crawler_user_agents.split('|').map { |agent|
[agent, SiteSetting.slow_down_crawler_rate]
}
if SiteSetting.whitelisted_crawler_user_agents.present?
@allowed_user_agents = SiteSetting.whitelisted_crawler_user_agents.split('|')