github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-30 07:11:34 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: Allow site administrators to mark S3 uploads with a missing status (#27222)

Browse Source 
This commit introduces the following changes which allows a site
administrator to mark `Upload` records with the `s3_file_missing`
verification status which will result in the `Upload` record being ignored when
`Discourse.store.list_missing_uploads` is ran on a site where S3 uploads
are enabled and `SiteSetting.enable_s3_inventory` is set to `true`.

1. Introduce `s3_file_missing` to `Upload.verification_statuses`
2. Introduce `Upload.mark_invalid_s3_uploads_as_missing` which updates
   `Upload#verification_status` of all `Upload` records from `invalid_etag` to `s3_file_missing`.
3. Introduce `rake uploads:mark_invalid_s3_uploads_as_missing` Rake task
   which allows a site administrator to change `Upload` records with
`invalid_etag` verification status to the `s3_file_missing`
verificaton_status.
4. Update `S3Inventory` to ignore `Upload` records with the
   `s3_file_missing` verification status.
This commit is contained in:
Alan Guo Xiang Tan
2024-05-30 08:37:38 +08:00
committed by GitHub
parent 2d1ab4c9e3
commit dc55b645b2
5 changed files with 125 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
lib/tasks/uploads.rake
View File

@ -949,7 +949,7 @@ def analyze_missing_s3
puts
end
missing_uploads = Upload.where(verification_status: Upload.verification_statuses[:invalid_etag])
missing_uploads = Upload.with_invalid_etag_verification_status
puts "Total missing uploads: #{missing_uploads.count}, newest is #{missing_uploads.maximum(:created_at)}"
puts "Total problem posts: #{lookup.keys.count} with #{lookup.values.sum { |a| a.length }} missing uploads"
puts "Other missing uploads count: #{other.count}"
@ -991,16 +991,15 @@ def analyze_missing_s3
end
def delete_missing_s3
missing =
Upload.where(verification_status: Upload.verification_statuses[:invalid_etag]).order(
:created_at,
)
missing = Upload.with_invalid_etag_verification_status.order(:created_at)
count = missing.count
if count > 0
puts "The following uploads will be deleted from the database"
missing.each { |upload| puts "#{upload.id} - #{upload.url} - #{upload.created_at}" }
puts "Please confirm you wish to delete #{count} upload records by typing YES"
confirm = STDIN.gets.strip
if confirm == "YES"
missing.destroy_all
puts "#{count} records were deleted"
@ -1011,6 +1010,29 @@ def delete_missing_s3
end
end
task "uploads:mark_invalid_s3_uploads_as_missing" => :environment do
puts "Marking invalid S3 uploads as missing for '#{RailsMultisite::ConnectionManagement.current_db}'..."
invalid_s3_uploads = Upload.with_invalid_etag_verification_status.order(:created_at)
count = invalid_s3_uploads.count
if count > 0
puts "The following uploads will be marked as missing on S3"
invalid_s3_uploads.each { |upload| puts "#{upload.id} - #{upload.url} - #{upload.created_at}" }
puts "Please confirm you wish to mark #{count} upload records as missing by typing YES"
confirm = STDIN.gets.strip
if confirm == "YES"
changed_count = Upload.mark_invalid_s3_uploads_as_missing
puts "#{changed_count} records were marked as missing"
else
STDERR.puts "Aborting"
exit 1
end
else
puts "No uploads found with invalid S3 etag verification status"
end
end
task "uploads:delete_missing_s3" => :environment do
if RailsMultisite::ConnectionManagement.current_db != "default"
delete_missing_s3
@ -1031,7 +1053,7 @@ def fix_missing_s3
Jobs.run_immediately!
puts "Attempting to download missing uploads and recreate"
ids = Upload.where(verification_status: Upload.verification_statuses[:invalid_etag]).pluck(:id)
ids = Upload.with_invalid_etag_verification_status.pluck(:id)
ids.each do |id|
upload = Upload.find_by(id: id)
next if !upload