diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index f90df56fb23..227ba7fafaa 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -1757,6 +1757,7 @@ en: force_custom_user_agent_hosts: "Hosts for which to use the custom onebox user agent on all requests. (Especially useful for hosts that limit access by user agent)." max_oneboxes_per_post: "Set the maximum number of oneboxes that can be included in a single post. Oneboxes provide a preview of linked content within the post." facebook_app_access_token: "A token generated from your Facebook app ID and secret. Used to generate Instagram oneboxes." + github_onebox_access_token: "A GitHub access token which is used to generate GitHub oneboxes for private repos, commits, pull requests, issues, and file contents. Without this, only public GitHub URLs will be oneboxed." logo: "The logo image at the top left of your site. Use a wide rectangular image with a height of 120 and an aspect ratio greater than 3:1. If left blank, the site title text will be shown." logo_small: "The small logo image at the top left of your site, seen when scrolling down. Use a square 120 × 120 image. If left blank, a home glyph will be shown." diff --git a/config/site_settings.yml b/config/site_settings.yml index 9285629f24c..d20d8d60ec1 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -2125,6 +2125,9 @@ onebox: cache_onebox_user_agent: default: "" hidden: true + github_onebox_access_token: + default: "" + secret: true spam: add_rel_nofollow_to_user_content: true hide_post_sensitivity: diff --git a/lib/onebox/engine/github_actions_onebox.rb b/lib/onebox/engine/github_actions_onebox.rb index c6309b3c5e7..f2494151c9e 100644 --- a/lib/onebox/engine/github_actions_onebox.rb +++ b/lib/onebox/engine/github_actions_onebox.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "../mixins/github_body" +require_relative "../mixins/github_auth_header" module Onebox module Engine @@ -8,6 +9,7 @@ module Onebox include Engine include LayoutSupport include JSON + include Onebox::Mixins::GithubAuthHeader matches_regexp( %r{^https?://(?:www\.)?(?:(?:\w)+\.)?github\.com/(?.+)/(?.+)/(actions/runs/[[:digit:]]+|pull/[[:digit:]]*/checks\?check_run_id=[[:digit:]]+)}, @@ -63,20 +65,22 @@ module Onebox end def data + result = raw(github_auth_header).clone + status = "unknown" - if raw["status"] == "completed" - if raw["conclusion"] == "success" + if result["status"] == "completed" + if result["conclusion"] == "success" status = "success" - elsif raw["conclusion"] == "failure" + elsif result["conclusion"] == "failure" status = "failure" end - elsif raw["status"] == "in_progress" + elsif result["status"] == "in_progress" status = "pending" end title = if type == :actions_run - raw["head_commit"]["message"].lines.first + result["head_commit"]["message"].lines.first elsif type == :pr_run pr_url = "https://api.github.com/repos/#{match[:org]}/#{match[:repo]}/pulls/#{match[:pr_id]}" @@ -86,8 +90,8 @@ module Onebox { :link => @url, :title => title, - :name => raw["name"], - :run_number => raw["run_number"], + :name => result["name"], + :run_number => result["run_number"], status => true, } end diff --git a/lib/onebox/engine/github_blob_onebox.rb b/lib/onebox/engine/github_blob_onebox.rb index 4b4ed67f271..47c240affbd 100644 --- a/lib/onebox/engine/github_blob_onebox.rb +++ b/lib/onebox/engine/github_blob_onebox.rb @@ -1,10 +1,13 @@ # frozen_string_literal: true require_relative "../mixins/git_blob_onebox" +require_relative "../mixins/github_auth_header" module Onebox module Engine class GithubBlobOnebox + include Onebox::Mixins::GithubAuthHeader + def self.git_regexp %r{^https?://(www\.)?github\.com.*/blob/} end @@ -35,6 +38,10 @@ module Onebox def title Sanitize.fragment(Onebox::Helpers.uri_unencode(link).sub(%r{^https?\://github\.com/}, "")) end + + def auth_headers + github_auth_header + end end end end diff --git a/lib/onebox/engine/github_commit_onebox.rb b/lib/onebox/engine/github_commit_onebox.rb index f7c81a103ab..0954617f4fd 100644 --- a/lib/onebox/engine/github_commit_onebox.rb +++ b/lib/onebox/engine/github_commit_onebox.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "../mixins/github_body" +require_relative "../mixins/github_auth_header" module Onebox module Engine @@ -9,6 +10,7 @@ module Onebox include LayoutSupport include JSON include Onebox::Mixins::GithubBody + include Onebox::Mixins::GithubAuthHeader matches_regexp(%r{^https?://(?:www\.)?(?:(?:\w)+\.)?(github)\.com(?:/)?(?:.)*/commit/}) always_https @@ -33,7 +35,7 @@ module Onebox end def data - result = raw.clone + result = raw(github_auth_header).clone lines = result["commit"]["message"].split("\n") result["title"] = lines.first diff --git a/lib/onebox/engine/github_issue_onebox.rb b/lib/onebox/engine/github_issue_onebox.rb index 145df704f0e..9f57a3e214d 100644 --- a/lib/onebox/engine/github_issue_onebox.rb +++ b/lib/onebox/engine/github_issue_onebox.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "../mixins/github_body" +require_relative "../mixins/github_auth_header" module Onebox module Engine @@ -10,6 +11,7 @@ module Onebox include LayoutSupport include JSON include Onebox::Mixins::GithubBody + include Onebox::Mixins::GithubAuthHeader matches_regexp( %r{^https?://(?:www\.)?(?:(?:\w)+\.)?github\.com/(?.+)/(?.+)/issues/([[:digit:]]+)}, @@ -35,31 +37,32 @@ module Onebox end def data - created_at = Time.parse(raw["created_at"]) - closed_at = Time.parse(raw["closed_at"]) if raw["closed_at"] - body, excerpt = compute_body(raw["body"]) + result = raw(github_auth_header).clone + created_at = Time.parse(result["created_at"]) + closed_at = Time.parse(result["closed_at"]) if result["closed_at"] + body, excerpt = compute_body(result["body"]) ulink = URI(link) labels = - raw["labels"].map do |l| + result["labels"].map do |l| { name: Emoji.codes_to_img(Onebox::Helpers.sanitize(l["name"])) } end { link: @url, - title: raw["title"], + title: result["title"], body: body, excerpt: excerpt, labels: labels, - user: raw["user"], + user: result["user"], created_at: created_at.strftime("%I:%M%p - %d %b %y %Z"), created_at_date: created_at.strftime("%F"), created_at_time: created_at.strftime("%T"), closed_at: closed_at&.strftime("%I:%M%p - %d %b %y %Z"), closed_at_date: closed_at&.strftime("%F"), closed_at_time: closed_at&.strftime("%T"), - closed_by: raw["closed_by"], - avatar: "https://avatars1.githubusercontent.com/u/#{raw["user"]["id"]}?v=2&s=96", + closed_by: result["closed_by"], + avatar: "https://avatars1.githubusercontent.com/u/#{result["user"]["id"]}?v=2&s=96", domain: "#{ulink.host}/#{ulink.path.split("/")[1]}/#{ulink.path.split("/")[2]}", i18n: i18n, } diff --git a/lib/onebox/engine/github_pull_request_onebox.rb b/lib/onebox/engine/github_pull_request_onebox.rb index 6ad192fc2d6..e86d0fe1818 100644 --- a/lib/onebox/engine/github_pull_request_onebox.rb +++ b/lib/onebox/engine/github_pull_request_onebox.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "../mixins/github_body" +require_relative "../mixins/github_auth_header" module Onebox module Engine @@ -9,6 +10,7 @@ module Onebox include LayoutSupport include JSON include Onebox::Mixins::GithubBody + include Onebox::Mixins::GithubAuthHeader GITHUB_COMMENT_REGEX = /(\r\n)/ @@ -27,7 +29,7 @@ module Onebox end def data - result = raw.clone + result = raw(github_auth_header).clone result["link"] = link created_at = Time.parse(result["created_at"]) diff --git a/lib/onebox/engine/gitlab_blob_onebox.rb b/lib/onebox/engine/gitlab_blob_onebox.rb index a3a3eccac76..56071692c14 100644 --- a/lib/onebox/engine/gitlab_blob_onebox.rb +++ b/lib/onebox/engine/gitlab_blob_onebox.rb @@ -33,6 +33,10 @@ module Onebox def title Sanitize.fragment(Onebox::Helpers.uri_unencode(link).sub(%r{^https?\://gitlab\.com/}, "")) end + + def auth_headers + {} + end end end end diff --git a/lib/onebox/engine/json.rb b/lib/onebox/engine/json.rb index 204b09c720e..7ea4973f363 100644 --- a/lib/onebox/engine/json.rb +++ b/lib/onebox/engine/json.rb @@ -5,8 +5,12 @@ module Onebox module JSON private - def raw - @raw ||= ::MultiJson.load(URI.parse(url).open(read_timeout: timeout)) + def raw(http_headers = {}) + @raw ||= ::MultiJson.load(URI.parse(url).open(options.merge(http_headers))) + end + + def options + { read_timeout: timeout } end end end diff --git a/lib/onebox/mixins/git_blob_onebox.rb b/lib/onebox/mixins/git_blob_onebox.rb index 6d8b0fd29df..229d35581f0 100644 --- a/lib/onebox/mixins/git_blob_onebox.rb +++ b/lib/onebox/mixins/git_blob_onebox.rb @@ -170,7 +170,11 @@ module Onebox @model_file = @lang.dup @raw = "https://render.githubusercontent.com/view/solid?url=" + self.raw_template(m) else - contents = URI.parse(self.raw_template(m)).open(read_timeout: timeout).read + contents = + URI + .parse(self.raw_template(m)) + .open({ read_timeout: timeout }.merge(self.auth_headers)) + .read if contents.encoding == Encoding::BINARY || contents.bytes.include?(0) @raw = nil diff --git a/lib/onebox/mixins/github_auth_header.rb b/lib/onebox/mixins/github_auth_header.rb new file mode 100644 index 00000000000..1f6f37964ab --- /dev/null +++ b/lib/onebox/mixins/github_auth_header.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Onebox + module Mixins + module GithubAuthHeader + def github_auth_header + return {} if SiteSetting.github_onebox_access_token.blank? + { "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}" } + end + end + end +end diff --git a/lib/oneboxer.rb b/lib/oneboxer.rb index 40e2b1c76aa..6482437c03b 100644 --- a/lib/oneboxer.rb +++ b/lib/oneboxer.rb @@ -681,6 +681,14 @@ module Oneboxer uri = URI(url) + # For private GitHub repos, we get a 404 when trying to use + # FinalDestination to request the final URL because no auth headers + # are sent. In this case we can ignore redirects and go straight to + # using Onebox.preview + if SiteSetting.github_onebox_access_token.present? && uri.hostname == "github.com" + fd_options[:ignore_redirects] << "https://github.com" + end + strategy = Oneboxer.ordered_strategies(uri.hostname).shift if strategy.blank? if strategy && Oneboxer.strategies[strategy][:force_get_host] diff --git a/spec/lib/onebox/engine/github_actions_onebox_spec.rb b/spec/lib/onebox/engine/github_actions_onebox_spec.rb index 5beb78acc79..37130b3a10b 100644 --- a/spec/lib/onebox/engine/github_actions_onebox_spec.rb +++ b/spec/lib/onebox/engine/github_actions_onebox_spec.rb @@ -4,16 +4,18 @@ RSpec.describe Onebox::Engine::GithubActionsOnebox do describe "PR check run" do before do @link = "https://github.com/discourse/discourse/pull/13128/checks?check_run_id=2660861130" + @pr_run_uri = "https://api.github.com/repos/discourse/discourse/pulls/13128" + @run_uri = "https://api.github.com/repos/discourse/discourse/check-runs/2660861130" - stub_request(:get, "https://api.github.com/repos/discourse/discourse/pulls/13128").to_return( + stub_request(:get, @pr_run_uri).to_return( status: 200, body: onebox_response("githubactions_pr"), ) - stub_request( - :get, - "https://api.github.com/repos/discourse/discourse/check-runs/2660861130", - ).to_return(status: 200, body: onebox_response("githubactions_pr_run")) + stub_request(:get, @run_uri).to_return( + status: 200, + body: onebox_response("githubactions_pr_run"), + ) end include_context "with engines" @@ -28,16 +30,30 @@ RSpec.describe Onebox::Engine::GithubActionsOnebox do expect(html).to include("simplify post and topic deletion language") end end + + context "when github_onebox_access_token is configured" do + before { SiteSetting.github_onebox_access_token = "1234" } + + it "sends it as part of the request" do + html + expect(WebMock).to have_requested(:get, @run_uri).with( + headers: { + "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}", + }, + ) + end + end end describe "GitHub Actions run" do before do @link = "https://github.com/discourse/discourse/actions/runs/873214216" + @pr_run_uri = "https://api.github.com/repos/discourse/discourse/actions/runs/873214216" - stub_request( - :get, - "https://api.github.com/repos/discourse/discourse/actions/runs/873214216", - ).to_return(status: 200, body: onebox_response("githubactions_actions_run")) + stub_request(:get, @pr_run_uri).to_return( + status: 200, + body: onebox_response("githubactions_actions_run"), + ) end include_context "with engines" @@ -56,5 +72,18 @@ RSpec.describe Onebox::Engine::GithubActionsOnebox do expect(html).to include("Linting") end end + + context "when github_onebox_access_token is configured" do + before { SiteSetting.github_onebox_access_token = "1234" } + + it "sends it as part of the request" do + html + expect(WebMock).to have_requested(:get, @pr_run_uri).with( + headers: { + "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}", + }, + ) + end + end end end diff --git a/spec/lib/onebox/engine/github_blob_onebox_spec.rb b/spec/lib/onebox/engine/github_blob_onebox_spec.rb index 295af5fba76..6d77c2e00d3 100644 --- a/spec/lib/onebox/engine/github_blob_onebox_spec.rb +++ b/spec/lib/onebox/engine/github_blob_onebox_spec.rb @@ -5,10 +5,12 @@ RSpec.describe Onebox::Engine::GithubBlobOnebox do @link = "https://github.com/discourse/onebox/blob/master/lib/onebox/engine/github_blob_onebox.rb" @uri = URI.parse(@link) - stub_request( - :get, - "https://raw.githubusercontent.com/discourse/onebox/master/lib/onebox/engine/github_blob_onebox.rb", - ).to_return(status: 200, body: onebox_response(described_class.onebox_name)) + @raw_uri = + "https://raw.githubusercontent.com/discourse/onebox/master/lib/onebox/engine/github_blob_onebox.rb" + stub_request(:get, @raw_uri).to_return( + status: 200, + body: onebox_response(described_class.onebox_name), + ) end include_context "with engines" @@ -38,5 +40,18 @@ RSpec.describe Onebox::Engine::GithubBlobOnebox do expect(html).not_to include("/Pages") expect(html).to include("This file is binary.") end + + context "when github_onebox_access_token is configured" do + before { SiteSetting.github_onebox_access_token = "1234" } + + it "sends it as part of the request" do + html + expect(WebMock).to have_requested(:get, @raw_uri).with( + headers: { + "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}", + }, + ) + end + end end end diff --git a/spec/lib/onebox/engine/github_commit_onebox_spec.rb b/spec/lib/onebox/engine/github_commit_onebox_spec.rb index 46d38eaae65..6551cb4ac7f 100644 --- a/spec/lib/onebox/engine/github_commit_onebox_spec.rb +++ b/spec/lib/onebox/engine/github_commit_onebox_spec.rb @@ -51,6 +51,19 @@ RSpec.describe Onebox::Engine::GithubCommitOnebox do expect(html).to include("2 deletions") end end + + context "when github_onebox_access_token is configured" do + before { SiteSetting.github_onebox_access_token = "1234" } + + it "sends it as part of the request" do + html + expect(WebMock).to have_requested(:get, @uri).with( + headers: { + "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}", + }, + ) + end + end end describe "PR with commit URL" do @@ -58,12 +71,9 @@ RSpec.describe Onebox::Engine::GithubCommitOnebox do @link = "https://github.com/discourse/discourse/pull/4662/commit/803d023e2307309f8b776ab3b8b7e38ba91c0919" @uri = - "https://api.github.com/repos/discourse/discourse/commit/803d023e2307309f8b776ab3b8b7e38ba91c0919" + "https://api.github.com/repos/discourse/discourse/commits/803d023e2307309f8b776ab3b8b7e38ba91c0919" - stub_request( - :get, - "https://api.github.com/repos/discourse/discourse/commits/803d023e2307309f8b776ab3b8b7e38ba91c0919", - ).to_return(status: 200, body: onebox_response("githubcommit")) + stub_request(:get, @uri).to_return(status: 200, body: onebox_response("githubcommit")) end include_context "with engines" @@ -107,5 +117,18 @@ RSpec.describe Onebox::Engine::GithubCommitOnebox do expect(html).to include("2 deletions") end end + + context "when github_onebox_access_token is configured" do + before { SiteSetting.github_onebox_access_token = "1234" } + + it "sends it as part of the request" do + html + expect(WebMock).to have_requested(:get, @uri).with( + headers: { + "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}", + }, + ) + end + end end end diff --git a/spec/lib/onebox/engine/github_issue_onebox_spec.rb b/spec/lib/onebox/engine/github_issue_onebox_spec.rb index c79c1eb4438..6a37be69726 100644 --- a/spec/lib/onebox/engine/github_issue_onebox_spec.rb +++ b/spec/lib/onebox/engine/github_issue_onebox_spec.rb @@ -3,8 +3,9 @@ RSpec.describe Onebox::Engine::GithubIssueOnebox do before do @link = "https://github.com/discourse/discourse/issues/1" + @issue_uri = "https://api.github.com/repos/discourse/discourse/issues/1" - stub_request(:get, "https://api.github.com/repos/discourse/discourse/issues/1").to_return( + stub_request(:get, @issue_uri).to_return( status: 200, body: onebox_response("github_issue_onebox"), ) @@ -20,5 +21,18 @@ RSpec.describe Onebox::Engine::GithubIssueOnebox do expect(html).to include(sanitized_label) end + + context "when github_onebox_access_token is configured" do + before { SiteSetting.github_onebox_access_token = "1234" } + + it "sends it as part of the request" do + html + expect(WebMock).to have_requested(:get, @issue_uri).with( + headers: { + "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}", + }, + ) + end + end end end diff --git a/spec/lib/onebox/engine/github_pull_request_onebox_spec.rb b/spec/lib/onebox/engine/github_pull_request_onebox_spec.rb index b14f42d5635..dbb00b4f133 100644 --- a/spec/lib/onebox/engine/github_pull_request_onebox_spec.rb +++ b/spec/lib/onebox/engine/github_pull_request_onebox_spec.rb @@ -90,4 +90,17 @@ RSpec.describe Onebox::Engine::GithubPullRequestOnebox do expect(html).to include("You've signed the CLA") end end + + context "when github_onebox_access_token is configured" do + before { SiteSetting.github_onebox_access_token = "1234" } + + it "sends it as part of the request" do + html + expect(WebMock).to have_requested(:get, @uri).with( + headers: { + "Authorization" => "Bearer #{SiteSetting.github_onebox_access_token}", + }, + ) + end + end end