github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-22 22:43:33 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Update rake task to backfill etags from s3 inventory

Browse Source
This commit is contained in:
Vinoth Kannan
2019-02-14 05:18:06 +05:30
parent b8d2549922
commit 7b5931013a
4 changed files with 26 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lib/file_store/s3_store.rb
View File

@ -123,11 +123,11 @@ module FileStore
SiteSetting.Upload.s3_upload_bucket.downcase SiteSetting.Upload.s3_upload_bucket.downcase
end end
def list_missing_uploads(skip_optimized: false) def list_missing_uploads(skip_optimized: false, backfill_etags: false)
if SiteSetting.enable_s3_inventory if SiteSetting.enable_s3_inventory
require 's3_inventory' require 's3_inventory'
S3Inventory.new(s3_helper, :upload).list_missing S3Inventory.new(s3_helper, :upload).list_missing(backfill_etags: backfill_etags)
S3Inventory.new(s3_helper, :optimized).list_missing unless skip_optimized S3Inventory.new(s3_helper, :optimized).list_missing(backfill_etags: backfill_etags) unless skip_optimized
else else
list_missing(Upload, "original/") list_missing(Upload, "original/")
list_missing(OptimizedImage, "optimized/") unless skip_optimized list_missing(OptimizedImage, "optimized/") unless skip_optimized

9
lib/s3_inventory.rb
View File

@ -24,7 +24,7 @@ class S3Inventory
end end
end end
def list_missing def list_missing(backfill_etags: false)
if files.blank? if files.blank?
error("Failed to list inventory from S3") error("Failed to list inventory from S3")
return return
@ -46,6 +46,13 @@ class S3Inventory
end end
end end
if backfill_etags
uploads = model.where(etag: nil).joins("LEFT JOIN #{table_name} ON #{model.table_name}.url ILIKE '%' || #{table_name}.key")
uploads.select(:id, :"#{table_name}.etag").find_each do |upload|
model.where(id: upload.id).update_all(etag: upload.etag)
end
end
uploads = (model == Upload) ? model.where("created_at < ?", last_modified) : model uploads = (model == Upload) ? model.where("created_at < ?", last_modified) : model
missing_uploads = uploads.joins("LEFT JOIN #{table_name} ON #{table_name}.etag = #{model.table_name}.etag").where("#{table_name}.etag is NULL") missing_uploads = uploads.joins("LEFT JOIN #{table_name} ON #{table_name}.etag = #{model.table_name}.etag").where("#{table_name}.etag is NULL")

8
lib/tasks/uploads.rake
View File

@ -482,16 +482,16 @@ end
# list all missing uploads and optimized images # list all missing uploads and optimized images
task "uploads:missing" => :environment do task "uploads:missing" => :environment do
if ENV["RAILS_DB"] if ENV["RAILS_DB"]
list_missing_uploads(skip_optimized: ENV['SKIP_OPTIMIZED']) list_missing_uploads(skip_optimized: ENV['SKIP_OPTIMIZED'], backfill_etags: ENV['BACKFILL_ETAGS'])
else else
RailsMultisite::ConnectionManagement.each_connection do |db| RailsMultisite::ConnectionManagement.each_connection do |db|
list_missing_uploads(skip_optimized: ENV['SKIP_OPTIMIZED']) list_missing_uploads(skip_optimized: ENV['SKIP_OPTIMIZED'], backfill_etags: ENV['BACKFILL_ETAGS'])
end end
end end
end end
def list_missing_uploads(skip_optimized: false) def list_missing_uploads(skip_optimized: false, backfill_etags: false)
Discourse.store.list_missing_uploads(skip_optimized: skip_optimized) Discourse.store.list_missing_uploads(skip_optimized: skip_optimized, backfill_etags: backfill_etags)
end end
################################################################################ ################################################################################

11
spec/components/s3_inventory_spec.rb
View File

@ -74,4 +74,15 @@ describe "S3Inventory" do
expect(output).to eq("Downloading inventory file 'Key' to tmp directory...\n#{upload.url}\n1 of 4 uploads are missing\n") expect(output).to eq("Downloading inventory file 'Key' to tmp directory...\n#{upload.url}\n1 of 4 uploads are missing\n")
end end
it "should backfill etags to uploads table correctly" do
Fabricate(:upload, url: "//bucket.amazonaws.com/original/0184537a4f419224404d013414e913a4f56018f2.jpg", created_at: 2.days.ago)
inventory.expects(:decompress_inventory_files)
inventory.expects(:files).returns([{ key: "Key", filename: "#{csv_filename}.gz" }]).at_least(1)
output = capture_stdout do
expect { inventory.list_missing(backfill_etags: true) }.to change { Upload.where(etag: nil).count }.by(-1)
end
end
end end