FEATURE: Add support for secure media (#7888)

This PR introduces a new secure media setting. When enabled, it prevent unathorized access to media uploads (files of type image, video and audio). When the `login_required` setting is enabled, then all media uploads will be protected from unauthorized (anonymous) access. When `login_required`is disabled, only media in private messages will be protected from unauthorized access. 

A few notes: 

- the `prevent_anons_from_downloading_files` setting no longer applies to audio and video uploads
- the `secure_media` setting can only be enabled if S3 uploads are already enabled and configured
- upload records have a new column, `secure`, which is a boolean `true/false` of the upload's secure status
- when creating a public post with an upload that has already been uploaded and is marked as secure, the post creator will raise an error
- when enabling or disabling the setting on a site with existing uploads, the rake task `uploads:ensure_correct_acl` should be used to update all uploads' secure status and their ACL on S3

spec/models/post_spec.rb

@@ -1245,7 +1245,7 @@ describe Post do
     expect(post.revisions.pluck(:number)).to eq([1, 2])
   end
 
-  describe '#link_post_uploads' do
+  describe 'uploads' do
     fab!(:video_upload) { Fabricate(:upload, extension: "mp4") }
     fab!(:image_upload) { Fabricate(:upload) }
     fab!(:audio_upload) { Fabricate(:upload, extension: "ogg") }
@@ -1257,7 +1257,7 @@ describe Post do
     let(:video_url) { "#{base_url}#{video_upload.url}" }
     let(:audio_url) { "#{base_url}#{audio_upload.url}" }
 
-    let(:raw) do
+    let(:raw_multiple) do
       <<~RAW
       <a href="#{attachment_upload.url}">Link</a>
       [test|attachment](#{attachment_upload_2.short_url})
@@ -1276,36 +1276,121 @@ describe Post do
       RAW
     end
 
-    let(:post) { Fabricate(:post, raw: raw) }
+    let(:post) { Fabricate(:post, raw: raw_multiple) }
 
-    it "finds all the uploads in the post" do
-      post.custom_fields[Post::DOWNLOADED_IMAGES] = {
-        "/uploads/default/original/1X/1/1234567890123456.csv": attachment_upload.id
-      }
+    context "#link_post_uploads" do
+      it "finds all the uploads in the post" do
+        post.custom_fields[Post::DOWNLOADED_IMAGES] = {
+          "/uploads/default/original/1X/1/1234567890123456.csv": attachment_upload.id
+        }
 
-      post.save_custom_fields
-      post.link_post_uploads
+        post.save_custom_fields
+        post.link_post_uploads
 
-      expect(PostUpload.where(post: post).pluck(:upload_id)).to contain_exactly(
-        video_upload.id,
-        image_upload.id,
-        audio_upload.id,
-        attachment_upload.id,
-        attachment_upload_2.id,
-        attachment_upload_3.id
-      )
+        expect(PostUpload.where(post: post).pluck(:upload_id)).to contain_exactly(
+          video_upload.id,
+          image_upload.id,
+          audio_upload.id,
+          attachment_upload.id,
+          attachment_upload_2.id,
+          attachment_upload_3.id
+        )
+      end
+
+      it "cleans the reverse index up for the current post" do
+        post.link_post_uploads
+
+        post_uploads_ids = post.post_uploads.pluck(:id)
+
+        post.link_post_uploads
+
+        expect(post.reload.post_uploads.pluck(:id)).to_not contain_exactly(
+          post_uploads_ids
+        )
+      end
     end
 
-    it "cleans the reverse index up for the current post" do
-      post.link_post_uploads
+    context '#update_uploads_secure_status' do
+      fab!(:user) { Fabricate(:user, trust_level: 0) }
 
-      post_uploads_ids = post.post_uploads.pluck(:id)
+      let(:raw) do
+        <<~RAW
+        <a href="#{attachment_upload.url}">Link</a>
+        <img src="#{image_upload.url}">
+        RAW
+      end
 
-      post.link_post_uploads
+      before do
+        SiteSetting.authorized_extensions = "pdf|png|jpg|csv"
+        SiteSetting.enable_s3_uploads = true
+        SiteSetting.s3_upload_bucket = "s3-upload-bucket"
+        SiteSetting.s3_access_key_id = "some key"
+        SiteSetting.s3_secret_access_key = "some secret key"
+        SiteSetting.secure_media = true
+        attachment_upload.update!(original_filename: "hello.csv")
 
-      expect(post.reload.post_uploads.pluck(:id)).to_not contain_exactly(
-        post_uploads_ids
-      )
+        stub_request(:head, "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/")
+
+        stub_request(
+          :put,
+          "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{attachment_upload.sha1}.#{attachment_upload.extension}?acl"
+        )
+
+        stub_request(
+          :put,
+          "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{image_upload.sha1}.#{image_upload.extension}?acl"
+        )
+      end
+
+      it "marks image uploads as secure in PMs when secure_media is ON" do
+        post = Fabricate(:post, raw: raw, user: user, topic: Fabricate(:private_message_topic, user: user))
+        post.link_post_uploads
+        post.update_uploads_secure_status
+
+        expect(PostUpload.where(post: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
+          [attachment_upload.id, false],
+          [image_upload.id, true]
+        )
+      end
+
+      it "marks image uploads as not secure in PMs when when secure_media is ON" do
+        SiteSetting.secure_media = false
+        post = Fabricate(:post, raw: raw, user: user, topic: Fabricate(:private_message_topic, user: user))
+        post.link_post_uploads
+        post.update_uploads_secure_status
+
+        expect(PostUpload.where(post: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
+          [attachment_upload.id, false],
+          [image_upload.id, false]
+        )
+      end
+
+      it "marks attachments as secure when relevant setting is enabled" do
+        SiteSetting.prevent_anons_from_downloading_files = true
+        post = Fabricate(:post, raw: raw, user: user, topic: Fabricate(:topic, user: user))
+        post.link_post_uploads
+        post.update_uploads_secure_status
+
+        expect(PostUpload.where(post: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
+          [attachment_upload.id, true],
+          [image_upload.id, false]
+        )
+      end
+
+      it "does not mark an upload as secure if it has already been used in a public topic" do
+        post = Fabricate(:post, raw: raw, user: user, topic: Fabricate(:topic, user: user))
+        post.link_post_uploads
+        post.update_uploads_secure_status
+
+        pm = Fabricate(:post, raw: raw, user: user, topic: Fabricate(:private_message_topic, user: user))
+        pm.link_post_uploads
+        pm.update_uploads_secure_status
+
+        expect(PostUpload.where(post: pm).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
+          [attachment_upload.id, false],
+          [image_upload.id, false]
+        )
+      end
     end
   end
 

spec/models/topic_converter_spec.rb

@@ -102,6 +102,47 @@ describe TopicConverter do
         expect(Notification.exists?(user_notification.id)).to eq(false)
         expect(Notification.exists?(admin_notification.id)).to eq(true)
       end
+
+      context "secure uploads" do
+        fab!(:image_upload) { Fabricate(:upload) }
+        fab!(:public_topic) { Fabricate(:topic, user: author) }
+
+        before do
+          SiteSetting.enable_s3_uploads = true
+          SiteSetting.s3_upload_bucket = "s3-upload-bucket"
+          SiteSetting.s3_access_key_id = "some key"
+          SiteSetting.s3_secret_access_key = "some secret key"
+          SiteSetting.secure_media = true
+
+          stub_request(:head, "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/")
+
+          stub_request(
+            :put,
+            "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{image_upload.sha1}.#{image_upload.extension}?acl"
+          )
+        end
+
+        it "converts regular uploads to secure when making a public post a PM" do
+          public_reply = Fabricate(:post, raw: "<img src='#{image_upload.url}'>", user: other_user, topic: public_topic)
+          public_reply.link_post_uploads
+          public_reply.update_uploads_secure_status
+
+          expect(public_reply.uploads[0].secure).to eq(false)
+          public_topic.convert_to_private_message(admin)
+          expect(public_topic.reload.posts.find(public_reply.id).uploads[0].secure).to eq(true)
+        end
+
+        it "converts secure uploads back to public" do
+          first_post
+          second_post = Fabricate(:post, raw: "<img src='#{image_upload.url}'>", user: other_user, topic: private_message)
+          second_post.link_post_uploads
+          second_post.update_uploads_secure_status
+
+          expect(second_post.uploads[0].secure).to eq(true)
+          private_message.convert_to_public_topic(admin)
+          expect(private_message.reload.posts.find(second_post.id).uploads[0].secure).to eq(false)
+        end
+      end
     end
   end
 

spec/models/upload_spec.rb

@@ -288,6 +288,84 @@ describe Upload do
     end
   end
 
+  describe '.update_secure_status' do
+    it 'marks a local upload as not secure with default settings' do
+      upload.update!(secure: true)
+      expect { upload.update_secure_status }
+        .to change { upload.secure }
+
+      expect(upload.secure).to eq(false)
+    end
+
+    it 'marks a local attachment as secure if prevent_anons_from_downloading_files is enabled' do
+      SiteSetting.prevent_anons_from_downloading_files = true
+      SiteSetting.authorized_extensions = "pdf"
+      upload.update!(original_filename: "small.pdf", extension: "pdf")
+
+      expect { upload.update_secure_status }
+        .to change { upload.secure }
+
+      expect(upload.secure).to eq(true)
+    end
+
+    it 'marks a local attachment as not secure if prevent_anons_from_downloading_files is disabled' do
+      SiteSetting.prevent_anons_from_downloading_files = false
+      SiteSetting.authorized_extensions = "pdf"
+      upload.update!(original_filename: "small.pdf", extension: "pdf", secure: true)
+
+      expect { upload.update_secure_status }
+        .to change { upload.secure }
+
+      expect(upload.secure).to eq(false)
+    end
+
+    it 'does not change secure status of a non-attachment when prevent_anons_from_downloading_files is enabled' do
+      SiteSetting.prevent_anons_from_downloading_files = true
+      SiteSetting.authorized_extensions = "mp4"
+      upload.update!(original_filename: "small.mp4", extension: "mp4")
+
+      expect { upload.update_secure_status }
+        .not_to change { upload.secure }
+
+      expect(upload.secure).to eq(false)
+    end
+
+    context "secure media enabled" do
+      before do
+        SiteSetting.enable_s3_uploads = true
+        SiteSetting.s3_upload_bucket = "s3-upload-bucket"
+        SiteSetting.s3_access_key_id = "some key"
+        SiteSetting.s3_secret_access_key = "some secret key"
+        SiteSetting.secure_media = true
+
+        stub_request(:head, "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/")
+
+        stub_request(
+          :put,
+          "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{upload.sha1}.#{upload.extension}?acl"
+        )
+      end
+
+      it 'marks an image upload as not secure when not associated with a post' do
+        upload.update!(secure: true)
+        expect { upload.update_secure_status }
+          .to change { upload.secure }
+
+        expect(upload.secure).to eq(false)
+      end
+
+      it 'marks an image upload as secure if login_required is enabled' do
+        SiteSetting.login_required = true
+        upload.update!(secure: false)
+
+        expect { upload.update_secure_status }
+          .to change { upload.secure }
+
+        expect(upload.secure).to eq(true)
+      end
+    end
+  end
+
   describe '.reset_unknown_extensions!' do
     it 'should reset the extension of uploads when it is "unknown"' do
       upload1 = Fabricate(:upload, extension: "unknown")