diff --git a/app/models/post.rb b/app/models/post.rb index 0fcf0f802f8..c579d012ea0 100644 --- a/app/models/post.rb +++ b/app/models/post.rb @@ -955,10 +955,7 @@ class Post < ActiveRecord::Base upload_id = Upload.where(sha1: sha1).pluck(:id).first if sha1.present? upload_id ||= yield(post, src, path, sha1) - if upload_id.present? - attributes = { post_id: post.id, upload_id: upload_id } - PostUpload.create!(attributes) unless PostUpload.exists?(attributes) - else + if upload_id.blank? missing_uploads << src missing_post_uploads[post.id] << src end diff --git a/lib/s3_inventory.rb b/lib/s3_inventory.rb index 4afc545167e..309b2ef61ce 100644 --- a/lib/s3_inventory.rb +++ b/lib/s3_inventory.rb @@ -82,7 +82,7 @@ class S3Inventory def list_missing_post_uploads log "Listing missing post uploads..." - missing = Post.find_missing_uploads(include_local_upload: false) do |_, _, _, sha1| + missing = Post.find_missing_uploads(include_local_upload: false) do |post, _, _, sha1| next if sha1.blank? upload_id = nil @@ -105,6 +105,7 @@ class S3Inventory ) upload.save!(validate: false) upload_id = upload.id + post.link_post_uploads rescue Aws::S3::Errors::NotFound next end diff --git a/spec/components/s3_inventory_spec.rb b/spec/components/s3_inventory_spec.rb index 0b8c66d45e8..a63913a7b74 100644 --- a/spec/components/s3_inventory_spec.rb +++ b/spec/components/s3_inventory_spec.rb @@ -82,8 +82,8 @@ describe "S3Inventory" do it "should backfill etags to uploads table correctly" do files = [ - ["#{Discourse.store.absolute_base_url}/original/0184537a4f419224404d013414e913a4f56018f2.jpg", "defcaac0b4aca535c284e95f30d608d0"], - ["#{Discourse.store.absolute_base_url}/original/0789fbf5490babc68326b9cec90eeb0d6590db05.png", "25c02eaceef4cb779fc17030d33f7f06"] + ["#{Discourse.store.absolute_base_url}/original/1X/0184537a4f419224404d013414e913a4f56018f2.jpg", "defcaac0b4aca535c284e95f30d608d0"], + ["#{Discourse.store.absolute_base_url}/original/1X/0789fbf5490babc68326b9cec90eeb0d6590db05.png", "25c02eaceef4cb779fc17030d33f7f06"] ] files.each { |file| Fabricate(:upload, url: file[0]) } @@ -97,4 +97,29 @@ describe "S3Inventory" do expect(Upload.by_users.order(:url).pluck(:url, :etag)).to eq(files) end + + it "should recover missing uploads correctly" do + freeze_time + + CSV.foreach(csv_filename, headers: false) do |row| + Fabricate(:upload, url: File.join(Discourse.store.absolute_base_url, row[S3Inventory::CSV_KEY_INDEX]), etag: row[S3Inventory::CSV_ETAG_INDEX], created_at: 2.days.ago) + end + + upload = Upload.last + etag = upload.etag + post = Fabricate(:post, raw: "![](#{upload.url})") + post.link_post_uploads + upload.delete + + inventory.expects(:download_inventory_files_to_tmp_directory) + inventory.expects(:decompress_inventory_files) + inventory.expects(:files).returns([{ key: "Key", filename: "#{csv_filename}.gz" }]).times(2) + + output = capture_stdout do + inventory.backfill_etags_and_list_missing + end + + expect(output).to eq("Listing missing post uploads...\n0 post uploads are missing.\n") + expect(post.uploads.first.etag).to eq(etag) + end end diff --git a/spec/fixtures/csv/s3_inventory.csv b/spec/fixtures/csv/s3_inventory.csv index 04325fb7c39..a45103c2a03 100644 --- a/spec/fixtures/csv/s3_inventory.csv +++ b/spec/fixtures/csv/s3_inventory.csv @@ -1,3 +1,3 @@ -"abc","original/0184537a4f419224404d013414e913a4f56018f2.jpg","defcaac0b4aca535c284e95f30d608d0" -"abc","original/050afc0ab01debe8cf48fd2ce50fbbf5eb072815.jpg","0cdc623af39cde0adb382670a6dc702a" -"abc","original/0789fbf5490babc68326b9cec90eeb0d6590db05.png","25c02eaceef4cb779fc17030d33f7f06" +"abc","original/1X/0184537a4f419224404d013414e913a4f56018f2.jpg","defcaac0b4aca535c284e95f30d608d0" +"abc","original/1X/050afc0ab01debe8cf48fd2ce50fbbf5eb072815.jpg","0cdc623af39cde0adb382670a6dc702a" +"abc","original/1X/0789fbf5490babc68326b9cec90eeb0d6590db05.png","25c02eaceef4cb779fc17030d33f7f06"