github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-31 06:58:31 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

FEATURE: Use amazon s3 inventory to manage upload stats (#6867)

Browse Source
This commit is contained in:
Vinoth Kannan
2019-02-01 10:10:48 +05:30
committed by GitHub
parent a1b4d9b061
commit b4f713ca52
14 changed files with 388 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
lib/file_store/s3_store.rb
View File

@ -124,8 +124,14 @@ module FileStore
end
def list_missing_uploads(skip_optimized: false)
list_missing(Upload, "original/")
list_missing(OptimizedImage, "optimized/") unless skip_optimized
if SiteSetting.enable_s3_inventory
require 's3_inventory'
S3Inventory.new(s3_helper, :upload).list_missing
S3Inventory.new(s3_helper, :optimized).list_missing unless skip_optimized
else
list_missing(Upload, "original/")
list_missing(OptimizedImage, "optimized/") unless skip_optimized
end
end
private
@ -140,7 +146,7 @@ module FileStore
verified_ids = []
files.each do |f|
id = model.where("url LIKE '%#{f.key}'").pluck(:id).first if f.size > 0
id = model.where("url LIKE '%#{f.key}' AND etag = '#{f.etag}'").pluck(:id).first
verified_ids << id if id.present?
marker = f.key
end
@ -150,7 +156,7 @@ module FileStore
files = @s3_helper.list(prefix, marker)
end
missing_uploads = model.where("id NOT IN (SELECT val FROM verified_ids)")
missing_uploads = model.joins('LEFT JOIN verified_ids ON verified_ids.val = id').where("verified_ids.val IS NULL")
missing_count = missing_uploads.count
if missing_count > 0