FEATURE: Add support for secure media (#7888)

This PR introduces a new secure media setting. When enabled, it prevent unathorized access to media uploads (files of type image, video and audio). When the `login_required` setting is enabled, then all media uploads will be protected from unauthorized (anonymous) access. When `login_required`is disabled, only media in private messages will be protected from unauthorized access. 

A few notes: 

- the `prevent_anons_from_downloading_files` setting no longer applies to audio and video uploads
- the `secure_media` setting can only be enabled if S3 uploads are already enabled and configured
- upload records have a new column, `secure`, which is a boolean `true/false` of the upload's secure status
- when creating a public post with an upload that has already been uploaded and is marked as secure, the post creator will raise an error
- when enabling or disabling the setting on a site with existing uploads, the rake task `uploads:ensure_correct_acl` should be used to update all uploads' secure status and their ACL on S3

lib/tasks/topics.rake

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
-def print_status_with_label(label, current, max)
-  print "\r%s%9d / %d (%5.1f%%)" % [label, current, max, ((current.to_f / max.to_f) * 100).round(1)]
-end
+require_dependency "rake_helpers"
 
 def close_old_topics(category)
   topics = Topic.where(closed: false, category_id: category.id)
@@ -23,7 +21,7 @@ def close_old_topics(category)
 
   topics.find_each do |topic|
     topic.update_status("closed", true, Discourse.system_user)
-    print_status_with_label("    closing old topics: ", topics_closed += 1, total)
+    RakeHelpers.print_status_with_label("    closing old topics: ", topics_closed += 1, total)
   end
 end
 
@@ -49,7 +47,7 @@ def apply_auto_close(category)
 
   topics.find_each do |topic|
     topic.inherit_auto_close_from_category
-    print_status_with_label("    applying auto-close to topics: ", topics_closed += 1, total)
+    RakeHelpers.print_status_with_label("    applying auto-close to topics: ", topics_closed += 1, total)
   end
 end
 
@@ -77,7 +75,7 @@ task "topics:watch_all_replied_topics" => :environment do
     t.topic_users.where(posted: true).find_each do |tp|
       tp.update!(notification_level: TopicUser.notification_levels[:watching], notifications_reason_id: TopicUser.notification_reasons[:created_post])
     end
-    print_status(count += 1, total)
+    RakeHelpers.print_status(count += 1, total)
   end
 
   puts "", "Done"
@@ -96,12 +94,8 @@ task "topics:update_fancy_titles" => :environment do
 
   Topic.find_each do |topic|
     topic.fancy_title
-    print_status(count += 1, total)
+    RakeHelpers.print_status(count += 1, total)
   end
 
   puts "", "Done"
 end
-
-def print_status(current, max)
-  print "\r%9d / %d (%5.1f%%)" % [current, max, ((current.to_f / max.to_f) * 100).round(1)]
-end

lib/tasks/uploads.rake

@@ -8,6 +8,8 @@ require "base62"
 #                                    gather                                    #
 ################################################################################
 
+require_dependency "rake_helpers"
+
 task "uploads:gather" => :environment do
   ENV["RAILS_DB"] ? gather_uploads : gather_uploads_for_all_sites
 end
@@ -426,7 +428,7 @@ def migrate_to_s3
           %Q{attachment; filename="#{upload.original_filename}"}
       end
 
-      if upload&.private?
+      if upload.secure
         options[:acl] = "private"
       end
     end
@@ -907,6 +909,108 @@ task "uploads:recover" => :environment do
   end
 end
 
+##
+# Run this task whenever the secure_media or login_required
+# settings are changed for a Discourse instance to update
+# the upload secure flag and S3 upload ACLs.
+task "uploads:ensure_correct_acl" => :environment do
+  RailsMultisite::ConnectionManagement.each_connection do |db|
+    unless Discourse.store.external?
+      puts "This task only works for external storage."
+      exit 1
+    end
+
+    puts "Ensuring correct ACL for uploads in #{db}...", ""
+
+    Upload.transaction do
+      mark_secure_in_loop_because_no_login_required = false
+
+      # First of all only get relevant uploads (supported media).
+      #
+      # Also only get uploads that are not for a theme or a site setting, so only
+      # get post related uploads.
+      uploads_with_supported_media = Upload.includes(:posts, :optimized_images).where(
+        "LOWER(original_filename) SIMILAR TO '%\.(jpg|jpeg|png|gif|svg|ico|mp3|ogg|wav|m4a|mov|mp4|webm|ogv)'"
+      ).joins(:post_uploads)
+
+      puts "There are #{uploads_with_supported_media.count} upload(s) with supported media that could be marked secure.", ""
+
+      # Simply mark all these uploads as secure if login_required because no anons will be able to access them
+      if SiteSetting.login_required?
+        mark_all_as_secure_login_required(uploads_with_supported_media)
+      else
+
+        # If NOT login_required, then we have to go for the other slower flow, where in the loop
+        # we mark the upload as secure if the first post it is used in is with_secure_media?
+        mark_secure_in_loop_because_no_login_required = true
+        puts "Marking posts as secure in the next step because login_required is false."
+      end
+
+      puts "", "Rebaking #{uploads_with_supported_media.count} upload posts and updating ACLs in S3.", ""
+
+      upload_ids_to_mark_as_secure, uploads_skipped_because_of_error = update_acls_and_rebake_upload_posts(
+        uploads_with_supported_media, mark_secure_in_loop_because_no_login_required
+      )
+
+      log_rebake_errors(uploads_skipped_because_of_error)
+      mark_specific_uploads_as_secure_no_login_required(upload_ids_to_mark_as_secure)
+    end
+  end
+  puts "", "Done"
+end
+
+def mark_all_as_secure_login_required(uploads_with_supported_media)
+  puts "Marking #{uploads_with_supported_media.count} upload(s) as secure because login_required is true.", ""
+  uploads_with_supported_media.update_all(secure: true)
+  puts "Finished marking upload(s) as secure."
+end
+
+def log_rebake_errors(uploads_skipped_because_of_error)
+  return if uploads_skipped_because_of_error.empty?
+  puts "Skipped the following uploads due to error:", ""
+  uploads_skipped_because_of_error.each do |message|
+    puts message
+  end
+end
+
+def mark_specific_uploads_as_secure_no_login_required(upload_ids_to_mark_as_secure)
+  return if upload_ids_to_mark_as_secure.empty?
+  puts "Marking #{upload_ids_to_mark_as_secure.length} uploads as secure because their first post contains secure media."
+  Upload.where(id: upload_ids_to_mark_as_secure).update_all(secure: true)
+  puts "Finished marking uploads as secure."
+end
+
+def update_acls_and_rebake_upload_posts(uploads_with_supported_media, mark_secure_in_loop_because_no_login_required)
+  upload_ids_to_mark_as_secure = []
+  uploads_skipped_because_of_error = []
+
+  i = 0
+  uploads_with_supported_media.find_each(batch_size: 50) do |upload_with_supported_media|
+    RakeHelpers.print_status_with_label("Updating ACL for upload.......", i, uploads_with_supported_media.count)
+
+    Discourse.store.update_upload_ACL(upload_with_supported_media)
+
+    RakeHelpers.print_status_with_label("Rebaking posts for upload.....", i, uploads_with_supported_media.count)
+    begin
+      upload_with_supported_media.posts.each { |post| post.rebake! }
+
+      if mark_secure_in_loop_because_no_login_required
+        first_post_with_upload = upload_with_supported_media.posts.order(sort_order: :asc).first
+        mark_secure = first_post_with_upload ? first_post_with_upload.with_secure_media? : false
+        upload_ids_to_mark_as_secure << upload_with_supported_media.id if mark_secure
+      end
+    rescue => e
+      uploads_skipped_because_of_error << "#{upload_with_supported_media.original_filename} (#{upload_with_supported_media.url}) #{e.message}"
+    end
+
+    i += 1
+  end
+  RakeHelpers.print_status_with_label("Rebaking complete!            ", i, uploads_with_supported_media.count)
+  puts ""
+
+  [upload_ids_to_mark_as_secure, uploads_skipped_because_of_error]
+end
+
 def inline_uploads(post)
   replaced = false