github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-06-03 19:39:30 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

FEATURE: Apply rate limits per user instead of IP for trusted users (#14706)

Browse Source 
Currently, Discourse rate limits all incoming requests by the IP address they
originate from regardless of the user making the request. This can be
frustrating if there are multiple users using Discourse simultaneously while
sharing the same IP address (e.g. employees in an office).

This commit implements a new feature to make Discourse apply rate limits by
user id rather than IP address for users at or higher than the configured trust
level (1 is the default).

For example, let's say a Discourse instance is configured to allow 200 requests
per minute per IP address, and we have 10 users at trust level 4 using
Discourse simultaneously from the same IP address. Before this feature, the 10
users could only make a total of 200 requests per minute before they got rate
limited. But with the new feature, each user is allowed to make 200 requests
per minute because the rate limits are applied on user id rather than the IP
address.

The minimum trust level for applying user-id-based rate limits can be
configured by the `skip_per_ip_rate_limit_trust_level` global setting. The
default is 1, but it can be changed by either adding the
`DISCOURSE_SKIP_PER_IP_RATE_LIMIT_TRUST_LEVEL` environment variable with the
desired value to your `app.yml`, or changing the setting's value in the
`discourse.conf` file.

Requests made with API keys are still rate limited by IP address and the
relevant global settings that control API keys rate limits.

Before this commit, Discourse's auth cookie (`_t`) was simply a 32 characters
string that Discourse used to lookup the current user from the database and the
cookie contained no additional information about the user. However, we had to
change the cookie content in this commit so we could identify the user from the
cookie without making a database query before the rate limits logic and avoid
introducing a bottleneck on busy sites.

Besides the 32 characters auth token, the cookie now includes the user id,
trust level and the cookie's generation date, and we encrypt/sign the cookie to
prevent tampering.

Internal ticket number: t54739.
This commit is contained in:
Osama Sayegh
2021-11-17 23:27:30 +03:00
committed by GitHub
parent 9be69b603c
commit b86127ad12
27 changed files with 973 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
lib/middleware/anonymous_cache.rb
View File

@ -49,9 +49,9 @@ module Middleware
ACCEPT_ENCODING = "HTTP_ACCEPT_ENCODING"
DISCOURSE_RENDER = "HTTP_DISCOURSE_RENDER"
def initialize(env)
def initialize(env, request = nil)
@env = env
@request = Rack::Request.new(@env)
@request = request || Rack::Request.new(@env)
end
def blocked_crawler?

234
lib/middleware/request_tracker.rb
View File

@ -4,7 +4,6 @@ require 'method_profiler'
require 'middleware/anonymous_cache'
class Middleware::RequestTracker
@@detailed_request_loggers = nil
@@ip_skipper = nil
@ -56,6 +55,10 @@ class Middleware::RequestTracker
@@ip_skipper = blk
end
def self.ip_skipper
@@ip_skipper
end
def initialize(app, settings = {})
@app = app
end
@ -92,23 +95,25 @@ class Middleware::RequestTracker
end
end
def self.get_data(env, result, timing)
def self.get_data(env, result, timing, request = nil)
status, headers = result
status = status.to_i
helper = Middleware::AnonymousCache::Helper.new(env)
request = Rack::Request.new(env)
request ||= Rack::Request.new(env)
helper = Middleware::AnonymousCache::Helper.new(env, request)
env_track_view = env["HTTP_DISCOURSE_TRACK_VIEW"]
track_view = status == 200
track_view &&= env_track_view != "0" && env_track_view != "false"
track_view &&= env_track_view || (request.get? && !request.xhr? && headers["Content-Type"] =~ /text\/html/)
track_view = !!track_view
has_auth_cookie = Auth::DefaultCurrentUserProvider.find_v0_auth_cookie(request).present?
has_auth_cookie ||= Auth::DefaultCurrentUserProvider.find_v1_auth_cookie(env).present?
h = {
status: status,
is_crawler: helper.is_crawler?,
has_auth_cookie: helper.has_auth_cookie?,
has_auth_cookie: has_auth_cookie,
is_background: !!(request.path =~ /^\/message-bus\// || request.path =~ /\/topics\/timings/),
is_mobile: helper.is_mobile?,
track_view: track_view,
@ -132,9 +137,9 @@ class Middleware::RequestTracker
h
end
def log_request_info(env, result, info)
def log_request_info(env, result, info, request = nil)
# we got to skip this on error ... its just logging
data = self.class.get_data(env, result, info) rescue nil
data = self.class.get_data(env, result, info, request) rescue nil
if data
if result && (headers = result[1])
@ -165,7 +170,7 @@ class Middleware::RequestTracker
def call(env)
result = nil
log_request = true
info = nil
# doing this as early as possible so we have an
# accurate counter
@ -173,14 +178,20 @@ class Middleware::RequestTracker
request = Rack::Request.new(env)
if available_in = rate_limit(request)
return [
429,
{ "Retry-After" => available_in.to_s },
["Slow down, too many requests from this IP address"]
]
cookie = find_auth_cookie(env)
if error_details = rate_limit(request, cookie)
available_in, error_code = error_details
message = <<~TEXT
Slow down, too many requests from this IP address.
Please retry again in #{available_in} seconds.
Error code: #{error_code}.
TEXT
headers = {
"Retry-After" => available_in.to_s,
"Discourse-Rate-Limit-Error-Code" => error_code
}
return [429, headers, [message]]
end
env["discourse.request_tracker"] = self
MethodProfiler.start
@ -222,93 +233,8 @@ class Middleware::RequestTracker
end
end
end
log_request_info(env, result, info) unless !log_request || env["discourse.request_tracker.skip"]
end
def is_private_ip?(ip)
ip = IPAddr.new(ip) rescue nil
!!(ip && (ip.private? || ip.loopback?))
end
def rate_limit(request)
if (
GlobalSetting.max_reqs_per_ip_mode == "block" ||
GlobalSetting.max_reqs_per_ip_mode == "warn" ||
GlobalSetting.max_reqs_per_ip_mode == "warn+block"
)
ip = request.ip
if !GlobalSetting.max_reqs_rate_limit_on_private
return false if is_private_ip?(ip)
end
return false if @@ip_skipper&.call(ip)
return false if STATIC_IP_SKIPPER&.any? { |entry| entry.include?(ip) }
limiter10 = RateLimiter.new(
nil,
"global_ip_limit_10_#{ip}",
GlobalSetting.max_reqs_per_ip_per_10_seconds,
10,
global: true,
aggressive: true
)
limiter60 = RateLimiter.new(
nil,
"global_ip_limit_60_#{ip}",
GlobalSetting.max_reqs_per_ip_per_minute,
60,
global: true,
aggressive: true
)
limiter_assets10 = RateLimiter.new(
nil,
"global_ip_limit_10_assets_#{ip}",
GlobalSetting.max_asset_reqs_per_ip_per_10_seconds,
10,
global: true
)
request.env['DISCOURSE_RATE_LIMITERS'] = [limiter10, limiter60]
request.env['DISCOURSE_ASSET_RATE_LIMITERS'] = [limiter_assets10]
warn = GlobalSetting.max_reqs_per_ip_mode == "warn" || GlobalSetting.max_reqs_per_ip_mode == "warn+block"
if !limiter_assets10.can_perform?
if warn
Discourse.warn("Global asset IP rate limit exceeded for #{ip}: 10 second rate limit", uri: request.env["REQUEST_URI"])
end
if GlobalSetting.max_reqs_per_ip_mode != "warn"
return limiter_assets10.seconds_to_wait(Time.now.to_i)
else
return false
end
end
begin
type = 10
limiter10.performed!
type = 60
limiter60.performed!
false
rescue RateLimiter::LimitExceeded => e
if warn
Discourse.warn("Global IP rate limit exceeded for #{ip}: #{type} second rate limit", uri: request.env["REQUEST_URI"])
if GlobalSetting.max_reqs_per_ip_mode != "warn"
e.available_in
else
false
end
else
e.available_in
end
end
if !env["discourse.request_tracker.skip"]
log_request_info(env, result, info, request)
end
end
@ -319,4 +245,108 @@ class Middleware::RequestTracker
end
end
end
def find_auth_cookie(env)
min_allowed_timestamp = Time.now.to_i - (UserAuthToken::ROTATE_TIME_MINS + 1) * 60
cookie = Auth::DefaultCurrentUserProvider.find_v1_auth_cookie(env)
if cookie && cookie[:issued_at] >= min_allowed_timestamp
cookie
end
end
def is_private_ip?(ip)
ip = IPAddr.new(ip)
!!(ip && (ip.private? || ip.loopback?))
rescue IPAddr::AddressFamilyError, IPAddr::InvalidAddressError
false
end
def rate_limit(request, cookie)
warn = GlobalSetting.max_reqs_per_ip_mode == "warn" ||
GlobalSetting.max_reqs_per_ip_mode == "warn+block"
block = GlobalSetting.max_reqs_per_ip_mode == "block" ||
GlobalSetting.max_reqs_per_ip_mode == "warn+block"
return if !block && !warn
ip = request.ip
if !GlobalSetting.max_reqs_rate_limit_on_private
return if is_private_ip?(ip)
end
return if @@ip_skipper&.call(ip)
return if STATIC_IP_SKIPPER&.any? { |entry| entry.include?(ip) }
ip_or_id = ip
limit_on_id = false
if cookie && cookie[:user_id] && cookie[:trust_level] && cookie[:trust_level] >= GlobalSetting.skip_per_ip_rate_limit_trust_level
ip_or_id = cookie[:user_id]
limit_on_id = true
end
limiter10 = RateLimiter.new(
nil,
"global_ip_limit_10_#{ip_or_id}",
GlobalSetting.max_reqs_per_ip_per_10_seconds,
10,
global: !limit_on_id,
aggressive: true,
error_code: limit_on_id ? "id_10_secs_limit" : "ip_10_secs_limit"
)
limiter60 = RateLimiter.new(
nil,
"global_ip_limit_60_#{ip_or_id}",
GlobalSetting.max_reqs_per_ip_per_minute,
60,
global: !limit_on_id,
error_code: limit_on_id ? "id_60_secs_limit" : "ip_60_secs_limit",
aggressive: true
)
limiter_assets10 = RateLimiter.new(
nil,
"global_ip_limit_10_assets_#{ip_or_id}",
GlobalSetting.max_asset_reqs_per_ip_per_10_seconds,
10,
error_code: limit_on_id ? "id_assets_10_secs_limit" : "ip_assets_10_secs_limit",
global: !limit_on_id
)
request.env['DISCOURSE_RATE_LIMITERS'] = [limiter10, limiter60]
request.env['DISCOURSE_ASSET_RATE_LIMITERS'] = [limiter_assets10]
if !limiter_assets10.can_perform?
if warn
Discourse.warn("Global asset IP rate limit exceeded for #{ip}: 10 second rate limit", uri: request.env["REQUEST_URI"])
end
if block
return [
limiter_assets10.seconds_to_wait(Time.now.to_i),
limiter_assets10.error_code
]
end
end
begin
type = 10
limiter10.performed!
type = 60
limiter60.performed!
nil
rescue RateLimiter::LimitExceeded => e
if warn
Discourse.warn("Global IP rate limit exceeded for #{ip}: #{type} second rate limit", uri: request.env["REQUEST_URI"])
end
if block
[e.available_in, e.error_code]
else
nil
end
end
end
end