FEATURE: Add support for secure media (#7888)

This PR introduces a new secure media setting. When enabled, it prevent unathorized access to media uploads (files of type image, video and audio). When the `login_required` setting is enabled, then all media uploads will be protected from unauthorized (anonymous) access. When `login_required`is disabled, only media in private messages will be protected from unauthorized access. 

A few notes: 

- the `prevent_anons_from_downloading_files` setting no longer applies to audio and video uploads
- the `secure_media` setting can only be enabled if S3 uploads are already enabled and configured
- upload records have a new column, `secure`, which is a boolean `true/false` of the upload's secure status
- when creating a public post with an upload that has already been uploaded and is marked as secure, the post creator will raise an error
- when enabling or disabling the setting on a site with existing uploads, the rake task `uploads:ensure_correct_acl` should be used to update all uploads' secure status and their ACL on S3
This commit is contained in:
Penar Musaraj
2019-11-17 20:25:42 -05:00
committed by Martin Brennan
parent 56b19ba740
commit 102909edb3
40 changed files with 1157 additions and 153 deletions

View File

@ -812,8 +812,8 @@ describe CookedPostProcessor do
it "is always allowed to crawl our own images" do
store = stub
Discourse.expects(:store).returns(store).at_least_once
store.expects(:has_been_uploaded?).returns(true)
Discourse.expects(:store).returns(store)
FastImage.expects(:size).returns([100, 200])
expect(cpp.get_size("http://foo.bar/image2.png")).to eq([100, 200])
end
@ -1104,40 +1104,175 @@ describe CookedPostProcessor do
HTML
end
it "uses the right CDN when uploads are on S3" do
Rails.configuration.action_controller.stubs(:asset_host).returns("https://local.cdn.com")
context "s3_uploads" do
before do
Rails.configuration.action_controller.stubs(:asset_host).returns("https://local.cdn.com")
SiteSetting.s3_upload_bucket = "some-bucket-on-s3"
SiteSetting.s3_access_key_id = "s3-access-key-id"
SiteSetting.s3_secret_access_key = "s3-secret-access-key"
SiteSetting.s3_cdn_url = "https://s3.cdn.com"
SiteSetting.enable_s3_uploads = true
SiteSetting.s3_upload_bucket = "some-bucket-on-s3"
SiteSetting.s3_access_key_id = "s3-access-key-id"
SiteSetting.s3_secret_access_key = "s3-secret-access-key"
SiteSetting.s3_cdn_url = "https://s3.cdn.com"
SiteSetting.enable_s3_uploads = true
SiteSetting.authorized_extensions = "png|jpg|gif|mov|ogg|"
uploaded_file = file_from_fixtures("smallest.png")
upload_sha1 = Digest::SHA1.hexdigest(File.read(uploaded_file))
uploaded_file = file_from_fixtures("smallest.png")
upload_sha1 = Digest::SHA1.hexdigest(File.read(uploaded_file))
upload.update!(
original_filename: "smallest.png",
width: 10,
height: 20,
sha1: upload_sha1,
extension: "png",
)
end
upload.update!(
original_filename: "smallest.png",
width: 10,
height: 20,
sha1: upload_sha1,
extension: "png",
)
it "uses the right CDN when uploads are on S3" do
stored_path = Discourse.store.get_path_for_upload(upload)
upload.update_column(:url, "#{SiteSetting.Upload.absolute_base_url}/#{stored_path}")
stored_path = Discourse.store.get_path_for_upload(upload)
upload.update_column(:url, "#{SiteSetting.Upload.absolute_base_url}/#{stored_path}")
the_post = Fabricate(:post, raw: %Q{This post has a local emoji :+1: and an external upload\n\n![smallest.png|10x20](#{upload.short_url})})
the_post = Fabricate(:post, raw: %Q{This post has a local emoji :+1: and an external upload\n\n![smallest.png|10x20](#{upload.short_url})})
cpp = CookedPostProcessor.new(the_post)
cpp.optimize_urls
cpp = CookedPostProcessor.new(the_post)
cpp.optimize_urls
expect(cpp.html).to match_html <<~HTML
<p>This post has a local emoji <img src="https://local.cdn.com/images/emoji/twitter/+1.png?v=#{Emoji::EMOJI_VERSION}" title=":+1:" class="emoji" alt=":+1:"> and an external upload</p>
<p><img src="https://s3.cdn.com/#{stored_path}" alt="smallest.png" data-base62-sha1="#{upload.base62_sha1}" width="10" height="20"></p>
HTML
end
expect(cpp.html).to match_html <<~HTML
<p>This post has a local emoji <img src="https://local.cdn.com/images/emoji/twitter/+1.png?v=#{Emoji::EMOJI_VERSION}" title=":+1:" class="emoji" alt=":+1:"> and an external upload</p>
<p><img src="https://s3.cdn.com/#{stored_path}" alt="smallest.png" data-base62-sha1="#{upload.base62_sha1}" width="10" height="20"></p>
HTML
it "doesn't use CDN for secure media" do
SiteSetting.secure_media = true
stored_path = Discourse.store.get_path_for_upload(upload)
upload.update_column(:url, "#{SiteSetting.Upload.absolute_base_url}/#{stored_path}")
upload.update_column(:secure, true)
the_post = Fabricate(:post, raw: %Q{This post has a local emoji :+1: and an external upload\n\n![smallest.png|10x20](#{upload.short_url})})
cpp = CookedPostProcessor.new(the_post)
cpp.optimize_urls
expect(cpp.html).to match_html <<~HTML
<p>This post has a local emoji <img src="https://local.cdn.com/images/emoji/twitter/+1.png?v=#{Emoji::EMOJI_VERSION}" title=":+1:" class="emoji" alt=":+1:"> and an external upload</p>
<p><img src="/secure-media-uploads/#{stored_path}" alt="smallest.png" data-base62-sha1="#{upload.base62_sha1}" width="10" height="20"></p>
HTML
end
context "media uploads" do
fab!(:image_upload) { Fabricate(:upload) }
fab!(:audio_upload) { Fabricate(:upload, extension: "ogg") }
fab!(:video_upload) { Fabricate(:upload, extension: "mov") }
before do
video_upload.update!(url: "#{SiteSetting.s3_cdn_url}/#{Discourse.store.get_path_for_upload(video_upload)}")
stub_request(:head, video_upload.url)
end
it "ignores prevent_anons_from_downloading_files and oneboxes video uploads" do
SiteSetting.prevent_anons_from_downloading_files = true
the_post = Fabricate(:post, raw: "This post has an S3 video onebox:\n#{video_upload.url}")
cpp = CookedPostProcessor.new(the_post)
cpp.post_process_oneboxes
expect(cpp.html).to match_html <<~HTML
<p>This post has an S3 video onebox:<br></p>
<div class="onebox video-onebox">
<video width="100%" height="100%" controls="">
<source src="#{video_upload.url}">
<a href="#{video_upload.url}" rel="nofollow ugc noopener">#{video_upload.url}</a>
</source>
</video>
</div>
HTML
end
it "oneboxes video using secure url when secure_media is enabled" do
SiteSetting.login_required = true
SiteSetting.secure_media = true
video_upload.update_column(:secure, true)
the_post = Fabricate(:post, raw: "This post has an S3 video onebox:\n#{video_upload.url}")
cpp = CookedPostProcessor.new(the_post)
cpp.post_process_oneboxes
secure_url = video_upload.url.sub(SiteSetting.s3_cdn_url, "#{Discourse.base_url}/secure-media-uploads")
expect(cpp.html).to match_html <<~HTML
<p>This post has an S3 video onebox:<br>
<div class="onebox video-onebox">
<video width="100%" height="100%" controls="">
<source src="#{secure_url}">
<a href="#{secure_url}">#{secure_url}</a>
</source>
</video>
</div>
</p>
HTML
end
it "oneboxes only audio/video and not images when secure_media is enabled" do
SiteSetting.login_required = true
SiteSetting.secure_media = true
video_upload.update_column(:secure, true)
audio_upload.update!(
url: "#{SiteSetting.s3_cdn_url}/#{Discourse.store.get_path_for_upload(audio_upload)}",
secure: true
)
image_upload.update!(
url: "#{SiteSetting.s3_cdn_url}/#{Discourse.store.get_path_for_upload(image_upload)}",
secure: true
)
stub_request(:head, audio_upload.url)
stub_request(:get, image_upload.url)
raw = <<~RAW
This post has a video upload.
#{video_upload.url}
This post has an audio upload.
#{audio_upload.url}
And an image upload.
![logo.png](upload://#{image_upload.base62_sha1}.#{image_upload.extension})
RAW
the_post = Fabricate(:post, raw: raw)
cpp = CookedPostProcessor.new(the_post)
cpp.post_process_oneboxes
secure_video_url = video_upload.url.sub(SiteSetting.s3_cdn_url, "#{Discourse.base_url}/secure-media-uploads")
secure_audio_url = audio_upload.url.sub(SiteSetting.s3_cdn_url, "#{Discourse.base_url}/secure-media-uploads")
expect(cpp.html).to match_html <<~HTML
<p>This post has a video upload.<br></p>
<div class="onebox video-onebox">
<video width="100%" height="100%" controls="">
<source src="#{secure_video_url}">
<a href="#{secure_video_url}">#{secure_video_url}</a>
</source>
</video>
</div>
<p>This post has an audio upload.<br>
<audio controls><source src="#{secure_audio_url}"><a href="#{secure_audio_url}">#{secure_audio_url}</a></source></audio>
</p>
<p>And an image upload.<br>
<img src="#{image_upload.url}" alt="#{image_upload.original_filename}" data-base62-sha1="#{image_upload.base62_sha1}"></p>
HTML
end
end
end
end
end

View File

@ -199,4 +199,19 @@ describe Email::Styles do
end
end
context "replace_relative_urls" do
it "replaces secure media within a link with a placeholder" do
frag = html_fragment("<a href=\"#{Discourse.base_url}\/secure-media-uploads/original/1X/testimage.png\"><img src=\"/secure-media-uploads/original/1X/testimage.png\"></a>")
expect(frag.at('p.secure-media-notice')).to be_present
expect(frag.at('img')).not_to be_present
expect(frag.at('a')).not_to be_present
end
it "replaces secure images with a placeholder" do
frag = html_fragment("<img src=\"/secure-media-uploads/original/1X/testimage.png\">")
expect(frag.at('p.secure-media-notice')).to be_present
expect(frag.at('img')).not_to be_present
end
end
end

View File

@ -86,5 +86,18 @@ RSpec.describe FileStore::BaseStore do
expect(file.class).to eq(File)
end
it "should return the file when secure media are enabled" do
SiteSetting.login_required = true
SiteSetting.secure_media = true
stub_request(:head, "https://s3-upload-bucket.s3.amazonaws.com/")
signed_url = Discourse.store.signed_url_for_path(upload_s3.url)
stub_request(:get, signed_url).to_return(status: 200, body: "Hello world")
file = store.download(upload_s3)
expect(file.class).to eq(File)
end
end
end

View File

@ -43,16 +43,16 @@ describe FileStore::S3Store do
let(:s3_object) { stub }
let(:etag) { "etag" }
before do
s3_object.stubs(:put).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
end
describe "#store_upload" do
it "returns an absolute schemaless url" do
store.expects(:get_depth_for).with(upload.id).returns(0)
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.png").returns(s3_object)
s3_object.expects(:put).with(
acl: "public-read",
cache_control: "max-age=31556952, public, immutable",
content_type: "image/png",
body: uploaded_file).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
expect(store.store_upload(uploaded_file, upload)).to eq(
"//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/original/1X/#{upload.sha1}.png"
@ -62,6 +62,7 @@ describe FileStore::S3Store do
describe "when s3_upload_bucket includes folders path" do
before do
s3_object.stubs(:put).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
SiteSetting.s3_upload_bucket = "s3-upload-bucket/discourse-uploads"
end
@ -78,28 +79,36 @@ describe FileStore::S3Store do
end
end
describe "when private uploads are enabled" do
it "returns signed URL for eligible private upload" do
describe "when secure uploads are enabled" do
it "saves secure attachment using private ACL" do
SiteSetting.prevent_anons_from_downloading_files = true
SiteSetting.authorized_extensions = "pdf|png|jpg|gif"
upload.update!(original_filename: "small.pdf", extension: "pdf")
upload.update!(original_filename: "small.pdf", extension: "pdf", secure: true)
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.pdf").returns(s3_object).at_least_once
s3_object.expects(:presigned_url).with(:get, expires_in: S3Helper::DOWNLOAD_URL_EXPIRES_AFTER_SECONDS)
s3_helper.expects(:s3_bucket).returns(s3_bucket)
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.pdf").returns(s3_object)
s3_object.expects(:put).with(
acl: "private",
cache_control: "max-age=31556952, public, immutable",
content_type: "application/pdf",
content_disposition: "attachment; filename=\"#{upload.original_filename}\"",
body: uploaded_file).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
expect(store.store_upload(uploaded_file, upload)).to eq(
"//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/original/1X/#{upload.sha1}.pdf"
)
expect(store.url_for(upload)).not_to eq(upload.url)
end
it "returns regular URL for ineligible private upload" do
it "saves image upload using public ACL" do
SiteSetting.prevent_anons_from_downloading_files = true
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.png").returns(s3_object).at_least_once
s3_object.expects(:put).with(
acl: "public-read",
cache_control: "max-age=31556952, public, immutable",
content_type: "image/png",
body: uploaded_file).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
expect(store.store_upload(uploaded_file, upload)).to eq(
"//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/original/1X/#{upload.sha1}.png"
@ -111,6 +120,10 @@ describe FileStore::S3Store do
end
describe "#store_optimized_image" do
before do
s3_object.stubs(:put).returns(Aws::S3::Types::PutObjectOutput.new(etag: "\"#{etag}\""))
end
it "returns an absolute schemaless url" do
store.expects(:get_depth_for).with(optimized_image.upload.id).returns(0)
s3_helper.expects(:s3_bucket).returns(s3_bucket)
@ -355,23 +368,27 @@ describe FileStore::S3Store do
include_context "s3 helpers"
let(:s3_object) { stub }
before do
SiteSetting.authorized_extensions = "pdf|png"
end
describe ".update_upload_ACL" do
it "sets acl to private when private uploads are enabled" do
SiteSetting.prevent_anons_from_downloading_files = true
it "sets acl to public by default" do
upload.update!(original_filename: "small.pdf", extension: "pdf")
s3_helper.expects(:s3_bucket).returns(s3_bucket)
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.png").returns(s3_object)
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.pdf").returns(s3_object)
s3_object.expects(:acl).returns(s3_object)
s3_object.expects(:put).with(acl: "private").returns(s3_object)
s3_object.expects(:put).with(acl: "public-read").returns(s3_object)
expect(store.update_upload_ACL(upload)).to be_truthy
end
it "sets acl to public when private uploads are disabled" do
SiteSetting.prevent_anons_from_downloading_files = false
it "sets acl to private when upload is marked secure" do
upload.update!(original_filename: "small.pdf", extension: "pdf", secure: true)
s3_helper.expects(:s3_bucket).returns(s3_bucket)
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.png").returns(s3_object)
s3_bucket.expects(:object).with("original/1X/#{upload.sha1}.pdf").returns(s3_object)
s3_object.expects(:acl).returns(s3_object)
s3_object.expects(:put).with(acl: "public-read").returns(s3_object)
s3_object.expects(:put).with(acl: "private").returns(s3_object)
expect(store.update_upload_ACL(upload)).to be_truthy
end
@ -421,4 +438,21 @@ describe FileStore::S3Store do
end
end
describe ".signed_url_for_path" do
include_context "s3 helpers"
let(:s3_object) { stub }
it "returns signed URL for a given path" do
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
s3_bucket.expects(:object).with("special/optimized/file.png").returns(s3_object)
opts = {
expires_in: S3Helper::DOWNLOAD_URL_EXPIRES_AFTER_SECONDS
}
s3_object.expects(:presigned_url).with(:get, opts)
expect(store.signed_url_for_path("special/optimized/file.png")).not_to eq(upload.url)
end
end
end

View File

@ -1388,4 +1388,63 @@ describe PostCreator do
end
end
end
context "secure media uploads" do
fab!(:image_upload) { Fabricate(:upload, secure: true) }
fab!(:user2) { Fabricate(:user) }
fab!(:public_topic) { Fabricate(:topic) }
before do
SiteSetting.enable_s3_uploads = true
SiteSetting.authorized_extensions = "png|jpg|gif|mp4"
SiteSetting.s3_upload_bucket = "s3-upload-bucket"
SiteSetting.s3_access_key_id = "some key"
SiteSetting.s3_secret_access_key = "some secret key"
SiteSetting.secure_media = true
stub_request(:head, "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/")
stub_request(
:put,
"https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{image_upload.sha1}.#{image_upload.extension}?acl"
)
end
it "does not allow a secure image to be used in a public topic" do
public_post = PostCreator.create(
user,
topic_id: public_topic.id,
raw: "A public post with an image.\n![](#{image_upload.short_path})"
)
expect(public_post.errors.count).to be(1)
expect(public_post.errors.full_messages).to include(I18n.t('secure_upload_not_allowed_in_public_topic', upload_filenames: image_upload.original_filename))
# secure upload CAN be used in another PM
pm = PostCreator.create(
user,
title: 'this is another private message',
raw: "with an upload: \n![](#{image_upload.short_path})",
archetype: Archetype.private_message,
target_usernames: [user2.username].join(',')
)
expect(pm.errors).to be_blank
end
it "does not allow a secure video to be used in a public topic" do
video_upload = Fabricate(:upload_s3, extension: 'mp4', original_filename: "video.mp4", secure: true)
public_post = PostCreator.create(
user,
topic_id: public_topic.id,
raw: "A public post with a video onebox:\n#{video_upload.url}"
)
expect(public_post.errors.count).to be(1)
expect(public_post.errors.full_messages).to include(I18n.t('secure_upload_not_allowed_in_public_topic', upload_filenames: video_upload.original_filename))
end
end
end

View File

@ -810,6 +810,50 @@ describe PrettyText do
html = "<p>Check out this video – <iframe src='https://player.vimeo.com/video/329875646' data-original-href='https://vimeo.com/329875646/> <script>alert(1)</script>'></iframe>.</p>"
expect(PrettyText.format_for_email(html, post)).to match(Regexp.escape("https://vimeo.com/329875646/%3E%20%3Cscript%3Ealert(1)%3C/script%3E"))
end
describe "#strip_secure_media" do
before do
SiteSetting.s3_upload_bucket = "some-bucket-on-s3"
SiteSetting.s3_access_key_id = "s3-access-key-id"
SiteSetting.s3_secret_access_key = "s3-secret-access-key"
SiteSetting.s3_cdn_url = "https://s3.cdn.com"
SiteSetting.enable_s3_uploads = true
SiteSetting.secure_media = true
SiteSetting.login_required = true
end
it "replaces secure video content" do
html = <<~HTML
<video width="100%" height="100%" controls="">
<source src="#{base_url}/secure-media-uploads/original/1X/some-video.mp4">
<a href="#{base_url}/secure-media-uploads/original/1X/some-video.mp4">Video label</a>
</source>
</video>
HTML
md = PrettyText.format_for_email(html, post)
expect(md).not_to include('<video')
expect(md.to_s).to match(I18n.t("emails.secure_media_placeholder"))
expect(md.to_s).not_to match(SiteSetting.Upload.s3_cdn_url)
end
it "replaces secure audio content" do
html = <<~HTML
<audio controls>
<source src="#{base_url}/secure-media-uploads/original/1X/some-audio.mp3">
<a href="#{base_url}/secure-media-uploads/original/1X/some-audio.mp3">Audio label</a>
</source>
</audio>
HTML
md = PrettyText.format_for_email(html, post)
expect(md).not_to include('<video')
expect(md.to_s).to match(I18n.t("emails.secure_media_placeholder"))
expect(md.to_s).not_to match(SiteSetting.Upload.s3_cdn_url)
end
end
end
it 'Is smart about linebreaks and IMG tags' do