FEATURE: Add support for secure media (#7888)

This PR introduces a new secure media setting. When enabled, it prevent unathorized access to media uploads (files of type image, video and audio). When the `login_required` setting is enabled, then all media uploads will be protected from unauthorized (anonymous) access. When `login_required`is disabled, only media in private messages will be protected from unauthorized access. 

A few notes: 

- the `prevent_anons_from_downloading_files` setting no longer applies to audio and video uploads
- the `secure_media` setting can only be enabled if S3 uploads are already enabled and configured
- upload records have a new column, `secure`, which is a boolean `true/false` of the upload's secure status
- when creating a public post with an upload that has already been uploaded and is marked as secure, the post creator will raise an error
- when enabling or disabling the setting on a site with existing uploads, the rake task `uploads:ensure_correct_acl` should be used to update all uploads' secure status and their ACL on S3

lib/file_store/base_store.rb

@@ -54,6 +54,10 @@ module FileStore
       not_implemented
     end
 
+    def s3_upload_host
+      not_implemented
+    end
+
     def external?
       not_implemented
     end
@@ -77,7 +81,11 @@ module FileStore
 
         if !file
           max_file_size_kb = [SiteSetting.max_image_size_kb, SiteSetting.max_attachment_size_kb].max.kilobytes
-          url = Discourse.store.cdn_url(upload.url)
+
+          url = upload.secure? ?
+            Discourse.store.signed_url_for_path(upload.url) :
+            Discourse.store.cdn_url(upload.url)
+
           url = SiteSetting.scheme + ":" + url if url =~ /^\/\//
           file = FileHelper.download(
             url,

lib/file_store/s3_store.rb

@@ -21,13 +21,13 @@ module FileStore
 
     def store_upload(file, upload, content_type = nil)
       path = get_path_for_upload(upload)
-      url, upload.etag = store_file(file, path, filename: upload.original_filename, content_type: content_type, cache_locally: true, private: upload.private?)
+      url, upload.etag = store_file(file, path, filename: upload.original_filename, content_type: content_type, cache_locally: true, private_acl: upload.secure?)
       url
     end
 
-    def store_optimized_image(file, optimized_image, content_type = nil)
+    def store_optimized_image(file, optimized_image, content_type = nil, secure: false)
       path = get_path_for_optimized_image(optimized_image)
-      url, optimized_image.etag = store_file(file, path, content_type: content_type)
+      url, optimized_image.etag = store_file(file, path, content_type: content_type, private_acl: secure)
       url
     end
 
@@ -42,12 +42,12 @@ module FileStore
       # cache file locally when needed
       cache_file(file, File.basename(path)) if opts[:cache_locally]
       options = {
-        acl: opts[:private] ? "private" : "public-read",
+        acl: opts[:private_acl] ? "private" : "public-read",
         cache_control: 'max-age=31556952, public, immutable',
         content_type: opts[:content_type].presence || MiniMime.lookup_by_filename(filename)&.content_type
       }
       # add a "content disposition" header for "attachments"
-      options[:content_disposition] = "attachment; filename=\"#{filename}\"" unless FileHelper.is_supported_image?(filename)
+      options[:content_disposition] = "attachment; filename=\"#{filename}\"" unless FileHelper.is_supported_media?(filename)
 
       path.prepend(File.join(upload_path, "/")) if Rails.configuration.multisite
 
@@ -88,6 +88,10 @@ module FileStore
       @absolute_base_url ||= SiteSetting.Upload.absolute_base_url
     end
 
+    def s3_upload_host
+      SiteSetting.Upload.s3_cdn_url.present? ? SiteSetting.Upload.s3_cdn_url : "https:#{absolute_base_url}"
+    end
+
     def external?
       true
     end
@@ -111,22 +115,9 @@ module FileStore
     end
 
     def url_for(upload, force_download: false)
-      if upload.private? || force_download
-        opts = { expires_in: S3Helper::DOWNLOAD_URL_EXPIRES_AFTER_SECONDS }
-
-        if force_download
-          opts[:response_content_disposition] = ActionDispatch::Http::ContentDisposition.format(
-            disposition: "attachment", filename: upload.original_filename
-          )
-        end
-
-        obj = @s3_helper.object(get_upload_key(upload))
-        url = obj.presigned_url(:get, opts)
-      else
-        url = upload.url
-      end
-
-      url
+      upload.secure? || force_download ?
+        presigned_url(get_upload_key(upload), force_download: force_download, filename: upload.original_filename) :
+        upload.url
     end
 
     def cdn_url(url)
@@ -136,6 +127,11 @@ module FileStore
       url.sub(File.join("#{schema}#{absolute_base_url}", folder), File.join(SiteSetting.Upload.s3_cdn_url, "/"))
     end
 
+    def signed_url_for_path(path)
+      key = path.sub(absolute_base_url + "/", "")
+      presigned_url(key)
+    end
+
     def cache_avatar(avatar, user_id)
       source = avatar.url.sub(absolute_base_url + "/", "")
       destination = avatar_template(avatar, user_id).sub(absolute_base_url + "/", "")
@@ -163,14 +159,15 @@ module FileStore
     end
 
     def update_upload_ACL(upload)
-      private_uploads = SiteSetting.prevent_anons_from_downloading_files
       key = get_upload_key(upload)
+      update_ACL(key, upload.secure?)
 
-      begin
-        @s3_helper.object(key).acl.put(acl: private_uploads ? "private" : "public-read")
-      rescue Aws::S3::Errors::NoSuchKey
-        Rails.logger.warn("Could not update ACL on upload with key: '#{key}'. Upload is missing.")
+      upload.optimized_images.find_each do |optimized_image|
+        optimized_image_key = get_path_for_optimized_image(optimized_image)
+        update_ACL(optimized_image_key, upload.secure?)
       end
+
+      true
     end
 
     def download_file(upload, destination_path)
@@ -179,6 +176,18 @@ module FileStore
 
     private
 
+    def presigned_url(url, force_download: false, filename: false)
+      opts = { expires_in: S3Helper::DOWNLOAD_URL_EXPIRES_AFTER_SECONDS }
+      if force_download && filename
+        opts[:response_content_disposition] = ActionDispatch::Http::ContentDisposition.format(
+          disposition: "attachment", filename: filename
+        )
+      end
+
+      obj = @s3_helper.object(url)
+      obj.presigned_url(:get, opts)
+    end
+
     def get_upload_key(upload)
       if Rails.configuration.multisite
         File.join(upload_path, "/", get_path_for_upload(upload))
@@ -187,6 +196,14 @@ module FileStore
       end
     end
 
+    def update_ACL(key, secure)
+      begin
+        @s3_helper.object(key).acl.put(acl: secure ? "private" : "public-read")
+      rescue Aws::S3::Errors::NoSuchKey
+        Rails.logger.warn("Could not update ACL on upload with key: '#{key}'. Upload is missing.")
+      end
+    end
+
     def list_missing(model, prefix)
       connection = ActiveRecord::Base.connection.raw_connection
       connection.exec('CREATE TEMP TABLE verified_ids(val integer PRIMARY KEY)')