github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-23 23:31:18 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

SECURITY: Expand and improve SSRF Protections (#18815)

Browse Source 
See https://github.com/discourse/discourse/security/advisories/GHSA-rcc5-28r3-23rr

Co-authored-by: OsamaSayegh <asooomaasoooma90@gmail.com>
Co-authored-by: Daniel Waterworth <me@danielwaterworth.com>
This commit is contained in:
David Taylor
2022-11-01 16:33:17 +00:00
committed by GitHub
parent 695b44269b
commit 68b4fe4cf8
42 changed files with 1164 additions and 443 deletions
Download Patch File Download Diff File Expand all files Collapse all files

241
spec/lib/final_destination_spec.rb
View File

@ -10,26 +10,6 @@ RSpec.describe FinalDestination do
force_get_hosts: ['https://force.get.com', 'https://*.ihaveawildcard.com/'],
preserve_fragment_url_hosts: ['https://eviltrout.com'],
# avoid IP lookups in test
lookup_ip: lambda do |host|
case host
when 'eviltrout.com' then '52.84.143.152'
when 'particularly.eviltrout.com' then '52.84.143.152'
when 'codinghorror.com' then '91.146.108.148'
when 'discourse.org' then '104.25.152.10'
when 'some_thing.example.com' then '104.25.152.10'
when 'private-host.com' then '192.168.10.1'
when 'internal-ipv6.com' then '2001:abc:de:01:3:3d0:6a65:c2bf'
when 'ignore-me.com' then '53.84.143.152'
when 'force.get.com' then '22.102.29.40'
when 'any-subdomain.ihaveawildcard.com' then '104.25.152.11'
when 'wikipedia.com' then '1.2.3.4'
else
_as_ip = IPAddr.new(host)
host
end
end
}
end
@ -54,15 +34,32 @@ RSpec.describe FinalDestination do
}
end
def fd_stub_request(method, url)
uri = URI.parse(url)
host = uri.hostname
ip = "1.2.3.4"
# In Excon we pass the IP in the URL, so we need to stub
# that version as well
uri.hostname = "HOSTNAME_PLACEHOLDER"
matcher = Regexp.escape(uri.to_s).sub(
"HOSTNAME_PLACEHOLDER",
"(#{Regexp.escape(host)}|#{Regexp.escape(ip)})"
)
stub_request(method, /\A#{matcher}\z/).with(headers: { "Host" => host })
end
def canonical_follow(from, dest)
stub_request(:get, from).to_return(
fd_stub_request(:get, from).to_return(
status: 200,
body: "<head><link rel=\"canonical\" href=\"#{dest}\"></head>"
)
end
def redirect_response(from, dest)
stub_request(:head, from).to_return(
fd_stub_request(:head, from).to_return(
status: 302,
headers: { "Location" => dest }
)
@ -90,6 +87,16 @@ RSpec.describe FinalDestination do
expect(fd('asdf').resolve).to be_nil
end
it "returns nil for unresolvable url" do
FinalDestination::SSRFDetector.stubs(:lookup_ips).raises(SocketError)
expect(fd("https://example.com").resolve).to eq(nil)
end
it "returns nil for url timeout" do
FinalDestination::SSRFDetector.stubs(:lookup_ips).raises(Timeout::Error)
expect(fd("https://example.com").resolve).to eq(nil)
end
it "returns nil when read timeouts" do
Excon.expects(:public_send).raises(Excon::Errors::Timeout)
@ -98,7 +105,7 @@ RSpec.describe FinalDestination do
context "without redirects" do
before do
stub_request(:head, "https://eviltrout.com").to_return(doc_response)
fd_stub_request(:head, "https://eviltrout.com/").to_return(doc_response)
end
it "returns the final url" do
@ -118,7 +125,7 @@ RSpec.describe FinalDestination do
context "with underscores in URLs" do
before do
stub_request(:head, 'https://some_thing.example.com').to_return(doc_response)
fd_stub_request(:head, 'https://some_thing.example.com').to_return(doc_response)
end
it "doesn't raise errors with underscores in urls" do
@ -133,7 +140,7 @@ RSpec.describe FinalDestination do
before do
redirect_response("https://eviltrout.com", "https://codinghorror.com/blog")
redirect_response("https://codinghorror.com/blog", "https://discourse.org")
stub_request(:head, "https://discourse.org").to_return(doc_response)
fd_stub_request(:head, "https://discourse.org").to_return(doc_response)
end
it "returns the final url" do
@ -148,7 +155,7 @@ RSpec.describe FinalDestination do
before do
redirect_response("https://eviltrout.com", "https://codinghorror.com/blog")
redirect_response("https://codinghorror.com/blog", "https://discourse.org")
stub_request(:head, "https://discourse.org").to_return(doc_response)
fd_stub_request(:head, "https://discourse.org").to_return(doc_response)
end
it "returns the final url" do
@ -162,7 +169,8 @@ RSpec.describe FinalDestination do
context "with a redirect to an internal IP" do
before do
redirect_response("https://eviltrout.com", "https://private-host.com")
stub_request(:head, "https://private-host.com").to_return(doc_response)
FinalDestination::SSRFDetector.stubs(:lookup_and_filter_ips).with("eviltrout.com").returns(["1.2.3.4"])
FinalDestination::SSRFDetector.stubs(:lookup_and_filter_ips).with("private-host.com").raises(FinalDestination::SSRFDetector::DisallowedIpError)
end
it "returns the final url" do
@ -188,14 +196,14 @@ RSpec.describe FinalDestination do
it 'raises error when response is too big' do
stub_const(described_class, "MAX_REQUEST_SIZE_BYTES", 1) do
stub_request(:get, "https://codinghorror.com/blog").to_return(body_response)
fd_stub_request(:get, "https://codinghorror.com/blog").to_return(body_response)
final = FinalDestination.new('https://codinghorror.com/blog', opts.merge(follow_canonical: true))
expect { final.resolve }.to raise_error(Excon::Errors::ExpectationFailed, "response size too big: https://codinghorror.com/blog")
end
end
it 'raises error when response is too slow' do
stub_request(:get, "https://codinghorror.com/blog").to_return(lambda { |request| freeze_time(11.seconds.from_now) ; body_response })
fd_stub_request(:get, "https://codinghorror.com/blog").to_return(lambda { |request| freeze_time(11.seconds.from_now) ; body_response })
final = FinalDestination.new('https://codinghorror.com/blog', opts.merge(follow_canonical: true))
expect { final.resolve }.to raise_error(Excon::Errors::ExpectationFailed, "connect timeout reached: https://codinghorror.com/blog")
end
@ -203,7 +211,7 @@ RSpec.describe FinalDestination do
context 'when following canonical links' do
it 'resolves the canonical link as the final destination' do
canonical_follow("https://eviltrout.com", "https://codinghorror.com/blog")
stub_request(:head, "https://codinghorror.com/blog").to_return(doc_response)
fd_stub_request(:head, "https://codinghorror.com/blog").to_return(doc_response)
final = FinalDestination.new('https://eviltrout.com', opts.merge(follow_canonical: true))
@ -216,7 +224,7 @@ RSpec.describe FinalDestination do
host = "https://codinghorror.com"
canonical_follow("#{host}/blog", "/blog/canonical")
stub_request(:head, "#{host}/blog/canonical").to_return(doc_response)
fd_stub_request(:head, "#{host}/blog/canonical").to_return(doc_response)
final = FinalDestination.new("#{host}/blog", opts.merge(follow_canonical: true))
@ -228,7 +236,7 @@ RSpec.describe FinalDestination do
it 'resolves the canonical link when the URL is relative and does not start with the / symbol' do
host = "https://codinghorror.com"
canonical_follow("#{host}/blog", "blog/canonical")
stub_request(:head, "#{host}/blog/canonical").to_return(doc_response)
fd_stub_request(:head, "#{host}/blog/canonical").to_return(doc_response)
final = FinalDestination.new("#{host}/blog", opts.merge(follow_canonical: true))
@ -259,65 +267,71 @@ RSpec.describe FinalDestination do
end
context "when forcing GET" do
before do
stub_request(:head, 'https://force.get.com/posts?page=4')
stub_request(:get, 'https://force.get.com/posts?page=4')
stub_request(:get, 'https://any-subdomain.ihaveawildcard.com/some/other/content')
stub_request(:head, 'https://eviltrout.com/posts?page=2')
stub_request(:get, 'https://eviltrout.com/posts?page=2')
stub_request(:head, 'https://particularly.eviltrout.com/has/a/secret/plan')
stub_request(:get, 'https://particularly.eviltrout.com/has/a/secret/plan')
end
it "will do a GET when forced" do
final = FinalDestination.new('https://force.get.com/posts?page=4', opts)
expect(final.resolve.to_s).to eq('https://force.get.com/posts?page=4')
url = 'https://force.get.com/posts?page=4'
get_stub = fd_stub_request(:get, url)
head_stub = fd_stub_request(:head, url)
final = FinalDestination.new(url, opts)
expect(final.resolve.to_s).to eq(url)
expect(final.status).to eq(:resolved)
expect(WebMock).to have_requested(:get, 'https://force.get.com/posts?page=4')
expect(WebMock).to_not have_requested(:head, 'https://force.get.com/posts?page=4')
expect(get_stub).to have_been_requested
expect(head_stub).to_not have_been_requested
end
it "will do a HEAD if not forced" do
final = FinalDestination.new('https://eviltrout.com/posts?page=2', opts)
expect(final.resolve.to_s).to eq('https://eviltrout.com/posts?page=2')
url = 'https://eviltrout.com/posts?page=2'
get_stub = fd_stub_request(:get, url)
head_stub = fd_stub_request(:head, url)
final = FinalDestination.new(url, opts)
expect(final.resolve.to_s).to eq(url)
expect(final.status).to eq(:resolved)
expect(WebMock).to_not have_requested(:get, 'https://eviltrout.com/posts?page=2')
expect(WebMock).to have_requested(:head, 'https://eviltrout.com/posts?page=2')
expect(get_stub).to_not have_been_requested
expect(head_stub).to have_been_requested
end
it "will do a GET when forced on a wildcard subdomain" do
final = FinalDestination.new('https://any-subdomain.ihaveawildcard.com/some/other/content', opts)
expect(final.resolve.to_s).to eq('https://any-subdomain.ihaveawildcard.com/some/other/content')
url = 'https://any-subdomain.ihaveawildcard.com/some/other/content'
get_stub = fd_stub_request(:get, url)
head_stub = fd_stub_request(:head, url)
final = FinalDestination.new(url, opts)
expect(final.resolve.to_s).to eq(url)
expect(final.status).to eq(:resolved)
expect(WebMock).to have_requested(:get, 'https://any-subdomain.ihaveawildcard.com/some/other/content')
expect(WebMock).to_not have_requested(:head, 'https://any-subdomain.ihaveawildcard.com/some/other/content')
expect(get_stub).to have_been_requested
expect(head_stub).to_not have_been_requested
end
it "will do a HEAD if on a subdomain of a forced get domain without a wildcard" do
final = FinalDestination.new('https://particularly.eviltrout.com/has/a/secret/plan', opts)
expect(final.resolve.to_s).to eq('https://particularly.eviltrout.com/has/a/secret/plan')
url = 'https://particularly.eviltrout.com/has/a/secret/plan'
get_stub = fd_stub_request(:get, url)
head_stub = fd_stub_request(:head, url)
final = FinalDestination.new(url, opts)
expect(final.resolve.to_s).to eq(url)
expect(final.status).to eq(:resolved)
expect(WebMock).to_not have_requested(:get, 'https://particularly.eviltrout.com/has/a/secret/plan')
expect(WebMock).to have_requested(:head, 'https://particularly.eviltrout.com/has/a/secret/plan')
expect(get_stub).to_not have_been_requested
expect(head_stub).to have_been_requested
end
end
context "when HEAD not supported" do
before do
stub_request(:get, 'https://eviltrout.com').to_return(
fd_stub_request(:get, 'https://eviltrout.com').to_return(
status: 301,
headers: {
"Location" => 'https://discourse.org',
'Set-Cookie' => 'evil=trout'
}
)
stub_request(:head, 'https://discourse.org')
fd_stub_request(:head, 'https://discourse.org')
end
context "when the status code is 405" do
before do
stub_request(:head, 'https://eviltrout.com').to_return(status: 405)
fd_stub_request(:head, 'https://eviltrout.com').to_return(status: 405)
end
it "will try a GET" do
@ -330,7 +344,7 @@ RSpec.describe FinalDestination do
context "when the status code is 501" do
before do
stub_request(:head, 'https://eviltrout.com').to_return(status: 501)
fd_stub_request(:head, 'https://eviltrout.com').to_return(status: 501)
end
it "will try a GET" do
@ -342,9 +356,9 @@ RSpec.describe FinalDestination do
end
it "correctly extracts cookies during GET" do
stub_request(:head, "https://eviltrout.com").to_return(status: 405)
fd_stub_request(:head, "https://eviltrout.com").to_return(status: 405)
stub_request(:get, "https://eviltrout.com")
fd_stub_request(:get, "https://eviltrout.com")
.to_return(status: 302, body: "" , headers: {
"Location" => "https://eviltrout.com",
"Set-Cookie" => ["foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com",
@ -352,7 +366,7 @@ RSpec.describe FinalDestination do
"baz=2; expires=Tue, 19-Feb-2019 10:14:24 GMT; path=/; domain=eviltrout.com"]
})
stub_request(:head, "https://eviltrout.com")
fd_stub_request(:head, "https://eviltrout.com")
.with(headers: { "Cookie" => "bar=1; baz=2; foo=219ffwef9w0f" })
final = FinalDestination.new("https://eviltrout.com", opts)
@ -363,13 +377,13 @@ RSpec.describe FinalDestination do
end
it "should use the correct format for cookies when there is only one cookie" do
stub_request(:head, "https://eviltrout.com")
fd_stub_request(:head, "https://eviltrout.com")
.to_return(status: 302, headers: {
"Location" => "https://eviltrout.com",
"Set-Cookie" => "foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com"
})
stub_request(:head, "https://eviltrout.com")
fd_stub_request(:head, "https://eviltrout.com")
.with(headers: { "Cookie" => "foo=219ffwef9w0f" })
final = FinalDestination.new("https://eviltrout.com", opts)
@ -379,7 +393,7 @@ RSpec.describe FinalDestination do
end
it "should use the correct format for cookies when there are multiple cookies" do
stub_request(:head, "https://eviltrout.com")
fd_stub_request(:head, "https://eviltrout.com")
.to_return(status: 302, headers: {
"Location" => "https://eviltrout.com",
"Set-Cookie" => ["foo=219ffwef9w0f; expires=Mon, 19-Feb-2018 10:44:24 GMT; path=/; domain=eviltrout.com",
@ -387,7 +401,7 @@ RSpec.describe FinalDestination do
"baz=2; expires=Tue, 19-Feb-2019 10:14:24 GMT; path=/; domain=eviltrout.com"]
})
stub_request(:head, "https://eviltrout.com")
fd_stub_request(:head, "https://eviltrout.com")
.with(headers: { "Cookie" => "bar=1; baz=2; foo=219ffwef9w0f" })
final = FinalDestination.new("https://eviltrout.com", opts)
@ -401,7 +415,7 @@ RSpec.describe FinalDestination do
upstream_url = "https://eviltrout.com/upstream/lib/code/foobar.rb"
redirect_response(origin_url, upstream_url)
stub_request(:head, upstream_url).to_return(doc_response)
fd_stub_request(:head, upstream_url).to_return(doc_response)
final = FinalDestination.new("#{origin_url}#L154-L205", opts)
expect(final.resolve.to_s).to eq("#{upstream_url}#L154-L205")
@ -410,7 +424,7 @@ RSpec.describe FinalDestination do
context "with content_type" do
before do
stub_request(:head, "https://eviltrout.com/this/is/an/image").to_return(image_response)
fd_stub_request(:head, "https://eviltrout.com/this/is/an/image").to_return(image_response)
end
it "returns a content_type" do
@ -489,15 +503,6 @@ RSpec.describe FinalDestination do
end
end
describe '.validate_uri' do
context "with host lookups" do
it "works for various hosts" do
expect(fd('https://private-host.com').validate_uri).to eq(false)
expect(fd('https://eviltrout.com:443').validate_uri).to eq(true)
end
end
end
describe ".validate_url_format" do
it "supports http urls" do
expect(fd('http://eviltrout.com').validate_uri_format).to eq(true)
@ -535,89 +540,19 @@ RSpec.describe FinalDestination do
end
end
describe ".is_dest_valid" do
it "returns false for a valid ipv4" do
expect(fd("https://52.84.143.67").is_dest_valid?).to eq(true)
expect(fd("https://104.25.153.10").is_dest_valid?).to eq(true)
end
it "returns false for short ip" do
lookup = lambda do |host|
# How IPs are looked up for single digits
if host == "0"
"0.0.0.0"
elsif host == "1"
"0.0.0.1"
end
end
expect(FinalDestination.new('https://0/logo.png', lookup_ip: lookup).is_dest_valid?).to eq(false)
expect(FinalDestination.new('https://1/logo.png', lookup_ip: lookup).is_dest_valid?).to eq(false)
end
it "returns false for private ipv4" do
expect(fd("https://127.0.0.1").is_dest_valid?).to eq(false)
expect(fd("https://192.168.1.3").is_dest_valid?).to eq(false)
expect(fd("https://10.0.0.5").is_dest_valid?).to eq(false)
expect(fd("https://172.16.0.1").is_dest_valid?).to eq(false)
end
it "returns false for IPV6 via site settings" do
SiteSetting.blocked_ip_blocks = '2001:abc:de::/48|2002:abc:de::/48'
expect(fd('https://[2001:abc:de:01:0:3f0:6a65:c2bf]').is_dest_valid?).to eq(false)
expect(fd('https://[2002:abc:de:01:0:3f0:6a65:c2bf]').is_dest_valid?).to eq(false)
expect(fd('https://internal-ipv6.com').is_dest_valid?).to eq(false)
expect(fd('https://[2003:abc:de:01:0:3f0:6a65:c2bf]').is_dest_valid?).to eq(true)
end
it "ignores invalid ranges" do
SiteSetting.blocked_ip_blocks = '2001:abc:de::/48|eviltrout'
expect(fd('https://[2001:abc:de:01:0:3f0:6a65:c2bf]').is_dest_valid?).to eq(false)
end
it "returns true for public ipv6" do
expect(fd("https://[2001:470:1:3a8::251]").is_dest_valid?).to eq(true)
end
it "returns false for private ipv6" do
expect(fd("https://[fdd7:b450:d4d1:6b44::1]").is_dest_valid?).to eq(false)
end
it "returns true for the base uri" do
SiteSetting.force_hostname = "final-test.example.com"
expect(fd("https://final-test.example.com/onebox").is_dest_valid?).to eq(true)
end
it "returns true for the S3 CDN url" do
SiteSetting.enable_s3_uploads = true
SiteSetting.s3_cdn_url = "https://s3.example.com"
expect(fd("https://s3.example.com/some/thing").is_dest_valid?).to eq(true)
end
it "returns true for the CDN url" do
GlobalSetting.stubs(:cdn_url).returns("https://cdn.example.com/discourse")
expect(fd("https://cdn.example.com/some/asset").is_dest_valid?).to eq(true)
end
it 'supports allowlisting via a site setting' do
SiteSetting.allowed_internal_hosts = 'private-host.com'
expect(fd("https://private-host.com/some/url").is_dest_valid?).to eq(true)
end
end
describe "https cache" do
it 'will cache https lookups' do
FinalDestination.clear_https_cache!("wikipedia.com")
stub_request(:head, "http://wikipedia.com/image.png")
fd_stub_request(:head, "http://wikipedia.com/image.png")
.to_return(status: 302, body: "", headers: { location: 'https://wikipedia.com/image.png' })
stub_request(:head, "https://wikipedia.com/image.png")
fd_stub_request(:head, "https://wikipedia.com/image.png")
fd('http://wikipedia.com/image.png').resolve
stub_request(:head, "https://wikipedia.com/image2.png")
fd_stub_request(:head, "https://wikipedia.com/image2.png")
fd('http://wikipedia.com/image2.png').resolve
end