github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-06-04 20:20:59 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: Apply syntax_tree formatting to lib/*

Browse Source
This commit is contained in:
David Taylor
2023-01-09 12:10:19 +00:00
parent b0fda61a8e
commit 6417173082
507 changed files with 16550 additions and 12627 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
lib/file_store/s3_store.rb
View File

@ -7,61 +7,64 @@ require "s3_helper"
require "file_helper"
module FileStore
class S3Store < BaseStore
TOMBSTONE_PREFIX ||= "tombstone/"
delegate :abort_multipart, :presign_multipart_part, :list_multipart_parts,
:complete_multipart, to: :s3_helper
delegate :abort_multipart,
:presign_multipart_part,
:list_multipart_parts,
:complete_multipart,
to: :s3_helper
def initialize(s3_helper = nil)
@s3_helper = s3_helper
end
def s3_helper
@s3_helper ||= S3Helper.new(s3_bucket,
Rails.configuration.multisite ? multisite_tombstone_prefix : TOMBSTONE_PREFIX
)
@s3_helper ||=
S3Helper.new(
s3_bucket,
Rails.configuration.multisite ? multisite_tombstone_prefix : TOMBSTONE_PREFIX,
)
end
def store_upload(file, upload, content_type = nil)
upload.url = nil
path = get_path_for_upload(upload)
url, upload.etag = store_file(
file,
path,
filename: upload.original_filename,
content_type: content_type,
cache_locally: true,
private_acl: upload.secure?
)
url, upload.etag =
store_file(
file,
path,
filename: upload.original_filename,
content_type: content_type,
cache_locally: true,
private_acl: upload.secure?,
)
url
end
def move_existing_stored_upload(
existing_external_upload_key:,
upload: nil,
content_type: nil
)
def move_existing_stored_upload(existing_external_upload_key:, upload: nil, content_type: nil)
upload.url = nil
path = get_path_for_upload(upload)
url, upload.etag = store_file(
nil,
path,
filename: upload.original_filename,
content_type: content_type,
cache_locally: false,
private_acl: upload.secure?,
move_existing: true,
existing_external_upload_key: existing_external_upload_key
)
url, upload.etag =
store_file(
nil,
path,
filename: upload.original_filename,
content_type: content_type,
cache_locally: false,
private_acl: upload.secure?,
move_existing: true,
existing_external_upload_key: existing_external_upload_key,
)
url
end
def store_optimized_image(file, optimized_image, content_type = nil, secure: false)
optimized_image.url = nil
path = get_path_for_optimized_image(optimized_image)
url, optimized_image.etag = store_file(file, path, content_type: content_type, private_acl: secure)
url, optimized_image.etag =
store_file(file, path, content_type: content_type, private_acl: secure)
url
end
@ -85,8 +88,9 @@ module FileStore
cache_file(file, File.basename(path)) if opts[:cache_locally]
options = {
acl: opts[:private_acl] ? "private" : "public-read",
cache_control: 'max-age=31556952, public, immutable',
content_type: opts[:content_type].presence || MiniMime.lookup_by_filename(filename)&.content_type
cache_control: "max-age=31556952, public, immutable",
content_type:
opts[:content_type].presence || MiniMime.lookup_by_filename(filename)&.content_type,
}
# add a "content disposition: attachment" header with the original
@ -96,7 +100,8 @@ module FileStore
# browser.
if !FileHelper.is_inline_image?(filename)
options[:content_disposition] = ActionDispatch::Http::ContentDisposition.format(
disposition: "attachment", filename: filename
disposition: "attachment",
filename: filename,
)
end
@ -106,11 +111,7 @@ module FileStore
if opts[:move_existing] && opts[:existing_external_upload_key]
original_path = opts[:existing_external_upload_key]
options[:apply_metadata_to_destination] = true
path, etag = s3_helper.copy(
original_path,
path,
options: options
)
path, etag = s3_helper.copy(original_path, path, options: options)
delete_file(original_path)
else
path, etag = s3_helper.upload(file, path, options)
@ -142,7 +143,7 @@ module FileStore
begin
parsed_url = URI.parse(UrlHelper.encode(url))
rescue
rescue StandardError
# There are many exceptions possible here including Addressable::URI:: exceptions
# and URI:: exceptions, catch all may seem wide, but it makes no sense to raise ever
# on an invalid url here
@ -169,7 +170,10 @@ module FileStore
s3_cdn_url = URI.parse(SiteSetting.Upload.s3_cdn_url || "")
cdn_hostname = s3_cdn_url.hostname
return true if cdn_hostname.presence && url[cdn_hostname] && (s3_cdn_url.path.blank? || parsed_url.path.starts_with?(s3_cdn_url.path))
if cdn_hostname.presence && url[cdn_hostname] &&
(s3_cdn_url.path.blank? || parsed_url.path.starts_with?(s3_cdn_url.path))
return true
end
false
end
@ -186,7 +190,11 @@ module FileStore
end
def s3_upload_host
SiteSetting.Upload.s3_cdn_url.present? ? SiteSetting.Upload.s3_cdn_url : "https:#{absolute_base_url}"
if SiteSetting.Upload.s3_cdn_url.present?
SiteSetting.Upload.s3_cdn_url
else
"https:#{absolute_base_url}"
end
end
def external?
@ -208,28 +216,45 @@ module FileStore
def path_for(upload)
url = upload&.url
FileStore::LocalStore.new.path_for(upload) if url && url[/^\/[^\/]/]
FileStore::LocalStore.new.path_for(upload) if url && url[%r{^/[^/]}]
end
def url_for(upload, force_download: false)
upload.secure? || force_download ?
presigned_get_url(get_upload_key(upload), force_download: force_download, filename: upload.original_filename) :
if upload.secure? || force_download
presigned_get_url(
get_upload_key(upload),
force_download: force_download,
filename: upload.original_filename,
)
else
upload.url
end
end
def cdn_url(url)
return url if SiteSetting.Upload.s3_cdn_url.blank?
schema = url[/^(https?:)?\/\//, 1]
schema = url[%r{^(https?:)?//}, 1]
folder = s3_bucket_folder_path.nil? ? "" : "#{s3_bucket_folder_path}/"
url.sub(File.join("#{schema}#{absolute_base_url}", folder), File.join(SiteSetting.Upload.s3_cdn_url, "/"))
url.sub(
File.join("#{schema}#{absolute_base_url}", folder),
File.join(SiteSetting.Upload.s3_cdn_url, "/"),
)
end
def signed_url_for_path(path, expires_in: SiteSetting.s3_presigned_get_url_expires_after_seconds, force_download: false)
def signed_url_for_path(
path,
expires_in: SiteSetting.s3_presigned_get_url_expires_after_seconds,
force_download: false
)
key = path.sub(absolute_base_url + "/", "")
presigned_get_url(key, expires_in: expires_in, force_download: force_download)
end
def signed_url_for_temporary_upload(file_name, expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS, metadata: {})
def signed_url_for_temporary_upload(
file_name,
expires_in: S3Helper::UPLOAD_URL_EXPIRES_AFTER_SECONDS,
metadata: {}
)
key = temporary_upload_path(file_name)
s3_helper.presigned_url(
key,
@ -237,16 +262,15 @@ module FileStore
expires_in: expires_in,
opts: {
metadata: metadata,
acl: "private"
}
acl: "private",
},
)
end
def temporary_upload_path(file_name)
folder_prefix = s3_bucket_folder_path.nil? ? upload_path : File.join(s3_bucket_folder_path, upload_path)
FileStore::BaseStore.temporary_upload_path(
file_name, folder_prefix: folder_prefix
)
folder_prefix =
s3_bucket_folder_path.nil? ? upload_path : File.join(s3_bucket_folder_path, upload_path)
FileStore::BaseStore.temporary_upload_path(file_name, folder_prefix: folder_prefix)
end
def object_from_path(path)
@ -264,13 +288,15 @@ module FileStore
end
def s3_bucket
raise Discourse::SiteSettingMissing.new("s3_upload_bucket") if SiteSetting.Upload.s3_upload_bucket.blank?
if SiteSetting.Upload.s3_upload_bucket.blank?
raise Discourse::SiteSettingMissing.new("s3_upload_bucket")
end
SiteSetting.Upload.s3_upload_bucket.downcase
end
def list_missing_uploads(skip_optimized: false)
if SiteSetting.enable_s3_inventory
require 's3_inventory'
require "s3_inventory"
S3Inventory.new(s3_helper, :upload).backfill_etags_and_list_missing
S3Inventory.new(s3_helper, :optimized).backfill_etags_and_list_missing unless skip_optimized
else
@ -326,7 +352,6 @@ module FileStore
s3_options: FileStore::ToS3Migration.s3_options_from_site_settings,
migrate_to_multisite: Rails.configuration.multisite,
).migrate
ensure
FileUtils.rm(public_upload_path) if File.symlink?(public_upload_path)
FileUtils.mv(old_upload_path, public_upload_path) if old_upload_path
@ -349,7 +374,8 @@ module FileStore
if force_download && filename
opts[:response_content_disposition] = ActionDispatch::Http::ContentDisposition.format(
disposition: "attachment", filename: filename
disposition: "attachment",
filename: filename,
)
end
@ -375,11 +401,11 @@ module FileStore
def list_missing(model, prefix)
connection = ActiveRecord::Base.connection.raw_connection
connection.exec('CREATE TEMP TABLE verified_ids(val integer PRIMARY KEY)')
connection.exec("CREATE TEMP TABLE verified_ids(val integer PRIMARY KEY)")
marker = nil
files = s3_helper.list(prefix, marker)
while files.count > 0 do
while files.count > 0
verified_ids = []
files.each do |f|
@ -388,23 +414,25 @@ module FileStore
marker = f.key
end
verified_id_clause = verified_ids.map { |id| "('#{PG::Connection.escape_string(id.to_s)}')" }.join(",")
verified_id_clause =
verified_ids.map { |id| "('#{PG::Connection.escape_string(id.to_s)}')" }.join(",")
connection.exec("INSERT INTO verified_ids VALUES #{verified_id_clause}")
files = s3_helper.list(prefix, marker)
end
missing_uploads = model.joins('LEFT JOIN verified_ids ON verified_ids.val = id').where("verified_ids.val IS NULL")
missing_uploads =
model.joins("LEFT JOIN verified_ids ON verified_ids.val = id").where(
"verified_ids.val IS NULL",
)
missing_count = missing_uploads.count
if missing_count > 0
missing_uploads.find_each do |upload|
puts upload.url
end
missing_uploads.find_each { |upload| puts upload.url }
puts "#{missing_count} of #{model.count} #{model.name.underscore.pluralize} are missing"
end
ensure
connection.exec('DROP TABLE verified_ids') unless connection.nil?
connection.exec("DROP TABLE verified_ids") unless connection.nil?
end
end
end