github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-06-06 23:07:28 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: Update backup/restore pipeline to avoid cd (#8347)

Browse Source
This commit is contained in:
David Taylor
2019-11-13 15:52:28 +00:00
committed by GitHub
parent d2d846a88e
commit 481efebe76
2 changed files with 52 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
lib/backup_restore/backuper.rb
View File

@ -245,12 +245,11 @@ module BackupRestore
Discourse::Utils.execute_command('tar', '--create', '--file', tar_filename, '--files-from', '/dev/null') Discourse::Utils.execute_command('tar', '--create', '--file', tar_filename, '--files-from', '/dev/null')
log "Archiving data dump..." log "Archiving data dump..."
FileUtils.cd(File.dirname(@dump_filename)) do Discourse::Utils.execute_command(
Discourse::Utils.execute_command( 'tar', '--append', '--dereference', '--file', tar_filename, File.basename(@dump_filename),
'tar', '--append', '--dereference', '--file', tar_filename, File.basename(@dump_filename), failure_message: "Failed to archive data dump.",
failure_message: "Failed to archive data dump." chdir: File.dirname(@dump_filename)
) )
end
add_local_uploads_to_archive(tar_filename) add_local_uploads_to_archive(tar_filename)
add_remote_uploads_to_archive(tar_filename) if SiteSetting.Upload.enable_s3_uploads add_remote_uploads_to_archive(tar_filename) if SiteSetting.Upload.enable_s3_uploads
@ -268,17 +267,16 @@ module BackupRestore
log "Archiving uploads..." log "Archiving uploads..."
upload_directory = "uploads/" + @current_db upload_directory = "uploads/" + @current_db
FileUtils.cd(File.join(Rails.root, "public")) do if File.directory?(File.join(Rails.root, "public", upload_directory))
if File.directory?(upload_directory) exclude_optimized = SiteSetting.include_thumbnails_in_backups ? '' : "--exclude=#{upload_directory}/optimized"
exclude_optimized = SiteSetting.include_thumbnails_in_backups ? '' : "--exclude=#{upload_directory}/optimized"
Discourse::Utils.execute_command( Discourse::Utils.execute_command(
'tar', '--append', '--dereference', exclude_optimized, '--file', tar_filename, upload_directory, 'tar', '--append', '--dereference', exclude_optimized, '--file', tar_filename, upload_directory,
failure_message: "Failed to archive uploads.", success_status_codes: [0, 1] failure_message: "Failed to archive uploads.", success_status_codes: [0, 1],
) chdir: File.join(Rails.root, "public")
else )
log "No local uploads found. Skipping archiving of local uploads..." else
end log "No local uploads found. Skipping archiving of local uploads..."
end end
end end
@ -294,30 +292,29 @@ module BackupRestore
upload_directory = File.join("uploads", @current_db) upload_directory = File.join("uploads", @current_db)
count = 0 count = 0
FileUtils.cd(@tmp_directory) do Upload.find_each do |upload|
Upload.find_each do |upload| next if upload.local?
next if upload.local? filename = File.join(@tmp_directory, upload_directory, store.get_path_for_upload(upload))
filename = File.join(@tmp_directory, upload_directory, store.get_path_for_upload(upload))
begin begin
FileUtils.mkdir_p(File.dirname(filename)) FileUtils.mkdir_p(File.dirname(filename))
store.download_file(upload, filename) store.download_file(upload, filename)
rescue StandardError => ex rescue StandardError => ex
log "Failed to download file with upload ID #{upload.id} from S3", ex log "Failed to download file with upload ID #{upload.id} from S3", ex
end
if File.exists?(filename)
Discourse::Utils.execute_command(
'tar', '--append', '--file', tar_filename, upload_directory,
failure_message: "Failed to add #{upload.original_filename} to archive.", success_status_codes: [0, 1]
)
File.delete(filename)
end
count += 1
log "#{count} files have already been downloaded. Still downloading..." if count % 500 == 0
end end
if File.exists?(filename)
Discourse::Utils.execute_command(
'tar', '--append', '--file', tar_filename, upload_directory,
failure_message: "Failed to add #{upload.original_filename} to archive.", success_status_codes: [0, 1],
chdir: @tmp_directory
)
File.delete(filename)
end
count += 1
log "#{count} files have already been downloaded. Still downloading..." if count % 500 == 0
end end
log "No uploads found on S3. Skipping archiving of uploads stored on S3..." if count == 0 log "No uploads found on S3. Skipping archiving of uploads stored on S3..." if count == 0

35
lib/backup_restore/restorer.rb
View File

@ -432,29 +432,28 @@ module BackupRestore
public_uploads_path = File.join(Rails.root, "public") public_uploads_path = File.join(Rails.root, "public")
FileUtils.cd(public_uploads_path) do FileUtils.mkdir_p(File.join(public_uploads_path, "uploads"))
FileUtils.mkdir_p("uploads")
tmp_uploads_path = Dir.glob(File.join(@tmp_directory, "uploads", "*")).first tmp_uploads_path = Dir.glob(File.join(@tmp_directory, "uploads", "*")).first
return if tmp_uploads_path.blank? return if tmp_uploads_path.blank?
previous_db_name = BackupMetadata.value_for("db_name") || File.basename(tmp_uploads_path) previous_db_name = BackupMetadata.value_for("db_name") || File.basename(tmp_uploads_path)
current_db_name = RailsMultisite::ConnectionManagement.current_db current_db_name = RailsMultisite::ConnectionManagement.current_db
optimized_images_exist = File.exist?(File.join(tmp_uploads_path, 'optimized')) optimized_images_exist = File.exist?(File.join(tmp_uploads_path, 'optimized'))
Discourse::Utils.execute_command( Discourse::Utils.execute_command(
'rsync', '-avp', '--safe-links', "#{tmp_uploads_path}/", "uploads/#{current_db_name}/", 'rsync', '-avp', '--safe-links', "#{tmp_uploads_path}/", "uploads/#{current_db_name}/",
failure_message: "Failed to restore uploads." failure_message: "Failed to restore uploads.",
) chdir: public_uploads_path
)
remap_uploads(previous_db_name, current_db_name) remap_uploads(previous_db_name, current_db_name)
if SiteSetting.Upload.enable_s3_uploads if SiteSetting.Upload.enable_s3_uploads
migrate_to_s3 migrate_to_s3
remove_local_uploads(File.join(public_uploads_path, "uploads/#{current_db_name}")) remove_local_uploads(File.join(public_uploads_path, "uploads/#{current_db_name}"))
end
generate_optimized_images unless optimized_images_exist
end end
generate_optimized_images unless optimized_images_exist
end end
def remap_uploads(previous_db_name, current_db_name) def remap_uploads(previous_db_name, current_db_name)