github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-22 06:41:25 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

FIX: Uploading large files (> 5GB) failed when enable_direct_s3_uploads is enabled (#16724)

Browse Source 
Larger files require a multipart copy.
This commit is contained in:
Gerhard Schlager
2022-06-28 21:30:00 +02:00
committed by GitHub
parent 0bcc478635
commit 9d870f151c
2 changed files with 64 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
lib/s3_helper.rb
View File

@ -111,37 +111,39 @@ class S3Helper
end
destination = get_path_for_s3_upload(destination)
if !Rails.configuration.multisite
options[:copy_source] = File.join(@s3_bucket_name, source)
source_object = if !Rails.configuration.multisite || source.include?(multisite_upload_path) || source.include?(@tombstone_prefix)
s3_bucket.object(source)
elsif @s3_bucket_folder_path
folder, filename = source.split("/", 2)
s3_bucket.object(File.join(folder, multisite_upload_path, filename))
else
if source.include?(multisite_upload_path) || source.include?(@tombstone_prefix)
options[:copy_source] = File.join(@s3_bucket_name, source)
elsif @s3_bucket_folder_path
folder, filename = begin
source.split("/", 2)
end
options[:copy_source] = File.join(@s3_bucket_name, folder, multisite_upload_path, filename)
else
options[:copy_source] = File.join(@s3_bucket_name, multisite_upload_path, source)
end
s3_bucket.object(File.join(multisite_upload_path, source))
end
if source_object.size > FIFTEEN_MEGABYTES
options[:multipart_copy] = true
options[:content_length] = source_object.size
end
destination_object = s3_bucket.object(destination)
# TODO: copy_source is a legacy option here and may become unsupported
# in later versions, we should change to use Aws::S3::Client#copy_object
# at some point.
#
# See https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations/object.rb#L67-L74
#
# ----
#
# Also note, any options for metadata (e.g. content_disposition, content_type)
# will not be applied unless the metadata_directive = "REPLACE" option is passed
# in. If this is not passed in, the source object's metadata will be used.
response = destination_object.copy_from(options)
# Note for small files that do not use multipart copy: Any options for metadata
# (e.g. content_disposition, content_type) will not be applied unless the
# metadata_directive = "REPLACE" option is passed in. If this is not passed in,
# the source object's metadata will be used.
# For larger files it copies the metadata from the source file and merges it
# with values from the copy call.
response = destination_object.copy_from(source_object, options)
[destination, response.copy_object_result.etag.gsub('"', '')]
etag = if response.respond_to?(:copy_object_result)
# small files, regular copy
response.copy_object_result.etag
else
# larger files, multipart copy
response.data.etag
end
[destination, etag.gsub('"', '')]
end
# Several places in the application need certain CORS rules to exist