github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-06-07 13:21:15 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

FEATURE: Migrate uploads to S3 during restore

Browse Source
This commit is contained in:
Gerhard Schlager
2019-06-04 15:47:21 +02:00
parent fb121815a5
commit f7a2648694
2 changed files with 24 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
lib/backup_restore/restorer.rb
View File

@ -448,19 +448,38 @@ module BackupRestore
DbHelper.remap("uploads/#{previous_db_name}", "uploads/#{current_db_name}") DbHelper.remap("uploads/#{previous_db_name}", "uploads/#{current_db_name}")
end end
if SiteSetting.Upload.enable_s3_uploads
migrate_to_s3
remove_local_uploads(File.join(public_uploads_path, "uploads/#{current_db_name}"))
end
generate_optimized_images unless optimized_images_exist generate_optimized_images unless optimized_images_exist
end end
end end
end end
def migrate_to_s3
log "Migrating uploads to S3..."
ENV["SKIP_FAILED"] = "1"
ENV["MIGRATE_TO_MULTISITE"] = "1" if Rails.configuration.multisite
Rake::Task["uploads:migrate_to_s3"].invoke
end
def remove_local_uploads(directory)
log "Removing local uploads directory..."
FileUtils.rm_rf(directory) if Dir[directory].present?
rescue => ex
log "Something went wrong while removing the following uploads directory: #{directory}", ex
end
def generate_optimized_images def generate_optimized_images
log 'Optimizing site icons...' log 'Optimizing site icons...'
DB.exec("TRUNCATE TABLE optimized_images")
SiteIconManager.ensure_optimized! SiteIconManager.ensure_optimized!
log 'Posts will be rebaked by a background job in sidekiq. You will see missing images until that has completed.' log 'Posts will be rebaked by a background job in sidekiq. You will see missing images until that has completed.'
log 'You can expedite the process by manually running "rake posts:rebake_uncooked_posts"' log 'You can expedite the process by manually running "rake posts:rebake_uncooked_posts"'
DB.exec("TRUNCATE TABLE optimized_images")
DB.exec(<<~SQL) DB.exec(<<~SQL)
UPDATE posts UPDATE posts
SET baked_version = NULL SET baked_version = NULL

7
lib/tasks/uploads.rake
View File

@ -345,6 +345,7 @@ def migrate_to_s3
end end
bucket_has_folder_path = true if ENV["DISCOURSE_S3_BUCKET"].include? "/" bucket_has_folder_path = true if ENV["DISCOURSE_S3_BUCKET"].include? "/"
public_directory = Rails.root.join("public").to_s
opts = { opts = {
region: ENV["DISCOURSE_S3_REGION"], region: ENV["DISCOURSE_S3_REGION"],
@ -369,7 +370,7 @@ def migrate_to_s3
print " - Listing local files" print " - Listing local files"
local_files = [] local_files = []
IO.popen("cd public && find uploads/#{db}/original -type f").each do |file| IO.popen("cd #{public_directory} && find uploads/#{db}/original -type f").each do |file|
local_files << file.chomp local_files << file.chomp
putc "." if local_files.size % 1000 == 0 putc "." if local_files.size % 1000 == 0
end end
@ -398,7 +399,7 @@ def migrate_to_s3
skip_etag_verify = ENV["SKIP_ETAG_VERIFY"].present? skip_etag_verify = ENV["SKIP_ETAG_VERIFY"].present?
local_files.each do |file| local_files.each do |file|
path = File.join("public", file) path = File.join(public_directory, file)
name = File.basename(path) name = File.basename(path)
etag = Digest::MD5.file(path).hexdigest unless skip_etag_verify etag = Digest::MD5.file(path).hexdigest unless skip_etag_verify
key = file[file.index(prefix)..-1] key = file[file.index(prefix)..-1]
@ -534,7 +535,7 @@ def migrate_to_s3
.where("u.id IS NOT NULL AND u.url LIKE '//%' AND optimized_images.url NOT LIKE '//%'") .where("u.id IS NOT NULL AND u.url LIKE '//%' AND optimized_images.url NOT LIKE '//%'")
.delete_all .delete_all
puts "Flagging all posts containing oneboxes for rebake..." puts "Flagging all posts containing lightboxes for rebake..."
count = Post.where("cooked LIKE '%class=\"lightbox\"%'").update_all(baked_version: nil) count = Post.where("cooked LIKE '%class=\"lightbox\"%'").update_all(baked_version: nil)
puts "#{count} posts were flagged for a rebake" puts "#{count} posts were flagged for a rebake"