FIX: Use dualstack S3 endpoint for direct uploads (#29611)

When we added direct S3 uploads to Discourse, which use
presigned URLs, we never took into account the dualstack
endpoints for IPv6 on S3.

This commit fixes the issue by using the dualstack endpoints
for presigned URLs and requests, which are used in the
get-presigned-put and batch-presign-urls endpoints used when
directly uploading to S3.

It also makes regular S3 requests for `put` and so on use
dualstack URLs. It doesn't seem like there is a downside to
doing this, but a bunch of specs needed to be updated to reflect this.
This commit is contained in:
Martin Brennan
2024-11-07 11:06:39 +10:00
committed by GitHub
parent cc01555fce
commit 0568d36133
10 changed files with 90 additions and 27 deletions

View File

@ -874,6 +874,7 @@ RSpec.describe UploadsController do
expect(result["key"]).to include(FileStore::S3Store::TEMPORARY_UPLOAD_PREFIX)
expect(result["url"]).to include(FileStore::S3Store::TEMPORARY_UPLOAD_PREFIX)
expect(result["url"]).to include("Amz-Expires")
expect(result["url"]).to include("dualstack")
end
it "includes accepted metadata in the response when provided" do
@ -1040,7 +1041,7 @@ RSpec.describe UploadsController do
XML
stub_request(
:post,
"https://s3-upload-bucket.s3.us-west-1.amazonaws.com/uploads/default/#{test_bucket_prefix}/temp/28fccf8259bbe75b873a2bd2564b778c/test.png?uploads",
"https://s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/uploads/default/#{test_bucket_prefix}/temp/28fccf8259bbe75b873a2bd2564b778c/test.png?uploads",
).to_return({ status: 200, body: create_multipart_result })
end
@ -1184,7 +1185,7 @@ RSpec.describe UploadsController do
XML
stub_request(
:get,
"https://s3-upload-bucket.#{SiteSetting.enable_s3_transfer_acceleration ? "s3-accelerate" : "s3.us-west-1"}.amazonaws.com/#{external_upload_stub.key}?max-parts=1&uploadId=#{mock_multipart_upload_id}",
"https://s3-upload-bucket.#{SiteSetting.enable_s3_transfer_acceleration ? "s3-accelerate.dualstack" : "s3.dualstack.us-west-1"}.amazonaws.com/#{external_upload_stub.key}?max-parts=1&uploadId=#{mock_multipart_upload_id}",
).to_return({ status: 200, body: list_multipart_result })
end
@ -1268,6 +1269,19 @@ RSpec.describe UploadsController do
)
end
it "uses dualstack endpoint for presigned URLs based on S3 region" do
stub_list_multipart_request
post "/uploads/batch-presign-multipart-parts.json",
params: {
unique_identifier: external_upload_stub.unique_identifier,
part_numbers: [2, 3, 4],
}
expect(response.status).to eq(200)
result = response.parsed_body
expect(result["presigned_urls"]["2"]).to include("dualstack")
end
context "when enable_s3_transfer_acceleration is true" do
before { SiteSetting.enable_s3_transfer_acceleration = true }
@ -1328,7 +1342,7 @@ RSpec.describe UploadsController do
describe "#complete_multipart" do
let(:upload_base_url) do
"https://#{SiteSetting.s3_upload_bucket}.#{SiteSetting.enable_s3_transfer_acceleration ? "s3-accelerate" : "s3.#{SiteSetting.s3_region}"}.amazonaws.com"
"https://#{SiteSetting.s3_upload_bucket}.#{SiteSetting.enable_s3_transfer_acceleration ? "s3-accelerate.dualstack" : "s3.dualstack.#{SiteSetting.s3_region}"}.amazonaws.com"
end
let(:mock_multipart_upload_id) do
"ibZBv_75gd9r8lH_gqXatLdxMVpAlj6CFTR.OwyF3953YdwbcQnMA2BLGn8Lx12fQNICtMw5KyteFeHw.Sjng--"
@ -1549,7 +1563,7 @@ RSpec.describe UploadsController do
describe "#abort_multipart" do
let(:upload_base_url) do
"https://#{SiteSetting.s3_upload_bucket}.#{SiteSetting.enable_s3_transfer_acceleration ? "s3-accelerate" : "s3.#{SiteSetting.s3_region}"}.amazonaws.com"
"https://#{SiteSetting.s3_upload_bucket}.#{SiteSetting.enable_s3_transfer_acceleration ? "s3-accelerate.dualstack" : "s3.dualstack.#{SiteSetting.s3_region}"}.amazonaws.com"
end
let(:mock_multipart_upload_id) do
"ibZBv_75gd9r8lH_gqXatLdxMVpAlj6CFTR.OwyF3953YdwbcQnMA2BLGn8Lx12fQNICtMw5KyteFeHw.Sjng--"