FEATURE: rake tasks for uploading assets to S3

This opens the door to serving application.js and so on from s3.

Also updates s3 gem for some tagging support

lib/tasks/s3.rake

@@ -0,0 +1,133 @@
+require_dependency "s3_helper"
+
+def brotli_s3_path(path)
+  ext = File.extname(path)
+  "#{path[0..-ext.length]}br#{ext}"
+end
+
+def gzip_s3_path(path)
+  ext = File.extname(path)
+  "#{path[0..-ext.length]}gz#{ext}"
+end
+
+def should_skip?(path)
+  return true if ENV['FORCE_S3_UPLOADS']
+  @existing_assets ||= Set.new(helper.list.map(&:key))
+  @existing_assets.include?('assets/' + path)
+end
+
+def upload_asset(helper, path, recurse: true, content_type: nil, fullpath: nil, content_encoding: nil)
+  fullpath ||= (Rails.root + "public/assets/#{path}").to_s
+
+  content_type ||= MiniMime.lookup_by_filename(path).content_type
+
+  options = {
+    cache_control: 'max-age=31556952, public, immutable',
+    content_type: content_type,
+    acl: 'public-read',
+    tagging: ''
+  }
+
+  if content_encoding
+    options[:content_encoding] = content_encoding
+  end
+
+  if should_skip?(path)
+    puts "Skipping: #{path}"
+  else
+    puts "Uploading: #{path}"
+    helper.upload(fullpath, path, options)
+  end
+
+  if recurse
+    if File.exist?(fullpath + ".br")
+      brotli_path = brotli_s3_path(path)
+      upload_asset(helper, brotli_path,
+        fullpath: fullpath + ".br",
+        recurse: false,
+        content_type: content_type,
+        content_encoding: 'br'
+      )
+    end
+
+    if File.exist?(fullpath + ".gz")
+      gzip_path = gzip_s3_path(path)
+      upload_asset(helper, gzip_path,
+        fullpath: fullpath + ".gz",
+        recurse: false,
+        content_type: content_type,
+        content_encoding: 'gzip'
+      )
+    end
+
+    if File.exist?(fullpath + ".map")
+      upload_asset(helper, path + ".map", recurse: false, content_type: 'application/json')
+    end
+  end
+end
+
+def assets
+  cached = Rails.application.assets.cached
+  manifest = Sprockets::Manifest.new(cached, Rails.root + 'public/assets', Rails.application.config.assets.manifest)
+
+  raise Discourse::SiteSettingMissing.new("s3_upload_bucket") if SiteSetting.s3_upload_bucket.blank?
+  manifest.assets
+end
+
+def helper
+  @helper ||= S3Helper.new(SiteSetting.s3_upload_bucket.downcase + '/assets')
+end
+
+def in_manifest
+  found = []
+  assets.each do |_, path|
+    fullpath = (Rails.root + "public/assets/#{path}").to_s
+
+    asset_path = "assets/#{path}"
+    found << asset_path
+
+    if File.exist?(fullpath + '.br')
+      found << brotli_s3_path(asset_path)
+    end
+
+    if File.exist?(fullpath + '.gz')
+      found << gzip_s3_path(asset_path)
+    end
+
+    if File.exist?(fullpath + '.map')
+      found << asset_path + '.map'
+    end
+
+  end
+  Set.new(found)
+end
+
+task 's3:upload_assets' => :environment do
+  assets.each do |name, fingerprint|
+    upload_asset(helper, fingerprint)
+  end
+end
+
+task 's3:expire_missing_assets' => :environment do
+  keep = in_manifest
+
+  count = 0
+  puts "Ensuring AWS assets are tagged correctly for removal"
+  helper.list.each do |f|
+    if keep.include?(f.key)
+      helper.tag_file(f.key, old: true)
+      count += 1
+    else
+      # ensure we do not delete this by mistake
+      helper.tag_file(f.key, {})
+    end
+  end
+
+  puts "#{count} assets were flagged for removal in 10 days"
+
+  puts "Ensuring AWS rule exists for purging old assets"
+  #helper.update_lifecycle("delete_old_assets", 10, prefix: 'old=true')
+
+  puts "Waiting on https://github.com/aws/aws-sdk-ruby/issues/1623"
+
+end