FEATURE: Add support for secure media (#7888)

This PR introduces a new secure media setting. When enabled, it prevent unathorized access to media uploads (files of type image, video and audio). When the `login_required` setting is enabled, then all media uploads will be protected from unauthorized (anonymous) access. When `login_required`is disabled, only media in private messages will be protected from unauthorized access. 

A few notes: 

- the `prevent_anons_from_downloading_files` setting no longer applies to audio and video uploads
- the `secure_media` setting can only be enabled if S3 uploads are already enabled and configured
- upload records have a new column, `secure`, which is a boolean `true/false` of the upload's secure status
- when creating a public post with an upload that has already been uploaded and is marked as secure, the post creator will raise an error
- when enabling or disabling the setting on a site with existing uploads, the rake task `uploads:ensure_correct_acl` should be used to update all uploads' secure status and their ACL on S3

app/models/post.rb

@@ -300,6 +300,15 @@ class Post < ActiveRecord::Base
     options[:user_id] = post_user.id if post_user
     options[:omit_nofollow] = true if omit_nofollow?
 
+    if self.with_secure_media?
+      each_upload_url do |url|
+        uri = URI.parse(url)
+        if FileHelper.is_supported_media?(File.basename(uri.path))
+          raw = raw.sub(Discourse.store.s3_upload_host, "#{Discourse.base_url}/secure-media-uploads")
+        end
+      end
+    end
+
     cooked = post_analyzer.cook(raw, options)
 
     new_cooked = Plugin::Filter.apply(:after_post_cook, self, cooked)
@@ -492,6 +501,11 @@ class Post < ActiveRecord::Base
     ReviewableFlaggedPost.pending.find_by(target: self)
   end
 
+  def with_secure_media?
+    return false unless SiteSetting.secure_media?
+    topic&.private_message? || SiteSetting.login_required?
+  end
+
   def hide!(post_action_type_id, reason = nil)
     return if hidden?
 
@@ -882,6 +896,13 @@ class Post < ActiveRecord::Base
     end
 
     upload_ids |= Upload.where(id: downloaded_images.values).pluck(:id)
+
+    disallowed_uploads = []
+    if SiteSetting.secure_media? && !topic&.private_message?
+      disallowed_uploads = Upload.where(id: upload_ids, secure: true).pluck(:original_filename)
+    end
+    return disallowed_uploads if disallowed_uploads.count > 0
+
     values = upload_ids.map! { |upload_id| "(#{self.id},#{upload_id})" }.join(",")
 
     PostUpload.transaction do
@@ -893,6 +914,12 @@ class Post < ActiveRecord::Base
     end
   end
 
+  def update_uploads_secure_status
+    if Discourse.store.external?
+      self.uploads.each { |upload| upload.update_secure_status }
+    end
+  end
+
   def downloaded_images
     JSON.parse(self.custom_fields[Post::DOWNLOADED_IMAGES].presence || "{}")
   rescue JSON::ParserError
@@ -909,6 +936,7 @@ class Post < ActiveRecord::Base
     ]
 
     fragments ||= Nokogiri::HTML::fragment(self.cooked)
+
     links = fragments.css("a/@href", "img/@src").map do |media|
       src = media.value
       next if src.blank?

app/models/topic_converter.rb

@@ -30,9 +30,9 @@ class TopicConverter
       )
 
       update_user_stats
+      update_post_uploads_secure_status
       Jobs.enqueue(:topic_action_converter, topic_id: @topic.id)
       Jobs.enqueue(:delete_inaccessible_notifications, topic_id: @topic.id)
-
       watch_topic(topic)
     end
     @topic
@@ -49,6 +49,7 @@ class TopicConverter
       )
 
       add_allowed_users
+      update_post_uploads_secure_status
 
       Jobs.enqueue(:topic_action_converter, topic_id: @topic.id)
       Jobs.enqueue(:delete_inaccessible_notifications, topic_id: @topic.id)
@@ -97,4 +98,11 @@ class TopicConverter
     end
   end
 
+  def update_post_uploads_secure_status
+    @topic.posts.each do |post|
+      next if post.uploads.empty?
+      post.update_uploads_secure_status
+      post.rebake!
+    end
+  end
 end

app/models/topic_link.rb

@@ -176,7 +176,7 @@ class TopicLink < ActiveRecord::Base
     if upload = Upload.get_from_url(url)
       internal = Discourse.store.internal?
       # Store the same URL that will be used in the cooked version of the post
-      url = UrlHelper.cook_url(upload.url)
+      url = UrlHelper.cook_url(upload.url, secure: upload.secure?)
     elsif route = Discourse.route_for(parsed)
       internal = true
 

app/models/upload.rb

@@ -140,11 +140,6 @@ class Upload < ActiveRecord::Base
     !(url =~ /^(https?:)?\/\//)
   end
 
-  def private?
-    return false if self.for_theme || self.for_site_setting
-    SiteSetting.prevent_anons_from_downloading_files && !FileHelper.is_supported_image?(self.original_filename)
-  end
-
   def fix_dimensions!
     return if !FileHelper.is_supported_image?("image.#{extension}")
 
@@ -235,6 +230,34 @@ class Upload < ActiveRecord::Base
     self.posts.where("cooked LIKE '%/_optimized/%'").find_each(&:rebake!)
   end
 
+  def update_secure_status
+    return false if self.for_theme || self.for_site_setting
+    mark_secure = should_be_secure?
+
+    self.update_column("secure", mark_secure)
+    Discourse.store.update_upload_ACL(self) if Discourse.store.external?
+  end
+
+  def should_be_secure?
+    mark_secure = false
+    if FileHelper.is_supported_media?(self.original_filename)
+      if SiteSetting.secure_media?
+        mark_secure = true if SiteSetting.login_required?
+        unless SiteSetting.login_required?
+          # first post associated with upload determines secure status
+          # i.e. an already public upload will stay public even if added to a new PM
+          first_post_with_upload = self.posts.order(sort_order: :asc).first
+          mark_secure = first_post_with_upload ? first_post_with_upload.with_secure_media? : false
+        end
+      else
+        mark_secure = false
+      end
+    else
+      mark_secure = SiteSetting.prevent_anons_from_downloading_files?
+    end
+    mark_secure
+  end
+
   def self.migrate_to_new_scheme(limit: nil)
     problems = []
 
@@ -385,6 +408,7 @@ end
 #  thumbnail_width   :integer
 #  thumbnail_height  :integer
 #  etag              :string
+#  secure            :boolean          default(FALSE), not null
 #
 # Indexes
 #