FEATURE: allow specifying s3 config via globals

This refactors handling of s3 so it can be specified via GlobalSetting

This means that in a multisite environment you can configure s3 uploads
without actual sites knowing credentials in s3

It is a critical setting for situations where assets are mirrored to s3.

lib/tasks/s3.rake

@@ -11,15 +11,12 @@ def gzip_s3_path(path)
 end
 
 def should_skip?(path)
-  return true if ENV['FORCE_S3_UPLOADS']
-  @existing_assets ||= Set.new(helper.list.map(&:key))
-  @existing_assets.include?('assets/' + path)
+  return false if ENV['FORCE_S3_UPLOADS']
+  @existing_assets ||= Set.new(helper.list("assets/").map(&:key))
+  @existing_assets.include?(path)
 end
 
-def upload_asset(helper, path, recurse: true, content_type: nil, fullpath: nil, content_encoding: nil)
-  fullpath ||= (Rails.root + "public/assets/#{path}").to_s
-
-  content_type ||= MiniMime.lookup_by_filename(path).content_type
+def upload(path, remote_path, content_type, content_encoding = nil)
 
   options = {
     cache_control: 'max-age=31556952, public, immutable',
@@ -32,50 +29,46 @@ def upload_asset(helper, path, recurse: true, content_type: nil, fullpath: nil,
     options[:content_encoding] = content_encoding
   end
 
-  if should_skip?(path)
-    puts "Skipping: #{path}"
+  if should_skip?(remote_path)
+    puts "Skipping: #{remote_path}"
   else
-    puts "Uploading: #{path}"
-    helper.upload(fullpath, path, options)
+    puts "Uploading: #{remote_path}"
+    helper.upload(path, remote_path, options)
   end
+end
 
-  if recurse
-    if File.exist?(fullpath + ".br")
-      brotli_path = brotli_s3_path(path)
-      upload_asset(helper, brotli_path,
-        fullpath: fullpath + ".br",
-        recurse: false,
-        content_type: content_type,
-        content_encoding: 'br'
-      )
-    end
-
-    if File.exist?(fullpath + ".gz")
-      gzip_path = gzip_s3_path(path)
-      upload_asset(helper, gzip_path,
-        fullpath: fullpath + ".gz",
-        recurse: false,
-        content_type: content_type,
-        content_encoding: 'gzip'
-      )
-    end
-
-    if File.exist?(fullpath + ".map")
-      upload_asset(helper, path + ".map", recurse: false, content_type: 'application/json')
-    end
-  end
+def helper
+  @helper ||= S3Helper.new(GlobalSetting.s3_bucket.downcase, '', S3Helper.s3_options(GlobalSetting))
 end
 
 def assets
   cached = Rails.application.assets&.cached
   manifest = Sprockets::Manifest.new(cached, Rails.root + 'public/assets', Rails.application.config.assets.manifest)
 
-  raise Discourse::SiteSettingMissing.new("s3_upload_bucket") if SiteSetting.s3_upload_bucket.blank?
-  manifest.assets
-end
+  results = []
 
-def helper
-  @helper ||= S3Helper.new(SiteSetting.s3_upload_bucket.downcase + '/assets')
+  manifest.assets.each do |_, path|
+    fullpath = (Rails.root + "public/assets/#{path}").to_s
+
+    content_type = MiniMime.lookup_by_filename(fullpath).content_type
+
+    asset_path = "assets/#{path}"
+    results << [fullpath, asset_path, content_type]
+
+    if File.exist?(fullpath + '.br')
+      results << [fullpath + '.br', brotli_s3_path(asset_path), content_type, 'br']
+    end
+
+    if File.exist?(fullpath + '.gz')
+      results << [fullpath + '.gz', gzip_s3_path(asset_path), content_type, 'gzip']
+    end
+
+    if File.exist?(fullpath + '.map')
+      results << [fullpath + '.map', asset_path + '.map', 'application/json']
+    end
+  end
+
+  results
 end
 
 def in_manifest
@@ -102,13 +95,23 @@ def in_manifest
   Set.new(found)
 end
 
+def ensure_s3_configured!
+  unless GlobalSetting.use_s3?
+    STDERR.puts "ERROR: Ensure S3 is configured in config/discourse.conf of environment vars"
+    exit 1
+  end
+end
+
 task 's3:upload_assets' => :environment do
-  assets.each do |name, fingerprint|
-    upload_asset(helper, fingerprint)
+  ensure_s3_configured!
+
+  assets.each do |asset|
+    upload(*asset)
   end
 end
 
 task 's3:expire_missing_assets' => :environment do
+  ensure_s3_configured!
   keep = in_manifest
 
   count = 0

lib/tasks/uploads.rake

@@ -114,13 +114,13 @@ def migrate_from_s3
   require "file_store/s3_store"
 
   # make sure S3 is disabled
-  if SiteSetting.enable_s3_uploads
+  if SiteSetting.Uploads.enable_s3_uploads
     puts "You must disable S3 uploads before running that task."
     return
   end
 
   # make sure S3 bucket is set
-  if SiteSetting.s3_upload_bucket.blank?
+  if SiteSetting.Upload.s3_upload_bucket.blank?
     puts "The S3 upload bucket must be set before running that task."
     return
   end
@@ -188,14 +188,14 @@ end
 
 def migrate_to_s3
   # make sure s3 is enabled
-  if !SiteSetting.enable_s3_uploads
+  if !SiteSetting.Upload.enable_s3_uploads
     puts "You must enable s3 uploads before running that task"
     return
   end
 
   db = RailsMultisite::ConnectionManagement.current_db
 
-  puts "Migrating uploads to S3 (#{SiteSetting.s3_upload_bucket}) for '#{db}'..."
+  puts "Migrating uploads to S3 (#{SiteSetting.Upload.s3_upload_bucket}) for '#{db}'..."
 
   # will throw an exception if the bucket is missing
   s3 = FileStore::S3Store.new