github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-21 18:12:32 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

when writing cache, remove elements from the user agents list. also return a message and content type when blocking a crawler.

Browse Source
This commit is contained in:
Neil Lalonde
2018-03-27 13:44:14 -04:00
parent a84bb81ab5
commit 4d12ff2e8a
3 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/models/web_crawler_request.rb
View File

@ -30,9 +30,9 @@ class WebCrawlerRequest < ActiveRecord::Base
self.last_flush = Time.now.utc
date = date.to_date
ua_list_key = user_agent_list_key(date)
$redis.smembers(user_agent_list_key(date)).each do |user_agent, _|
while user_agent = $redis.spop(ua_list_key)
val = get_and_reset(redis_key(user_agent, date))
next if val == 0
@ -57,7 +57,7 @@ class WebCrawlerRequest < ActiveRecord::Base
$redis.del redis_key(user_agent, date)
end
$redis.del list_key
$redis.del(list_key)
end
protected