FIX: Use Twitter API v2 for oneboxes and restore OpenGraph fallback (#22187)

lib/onebox/engine/twitter_status_onebox.rb

@@ -6,17 +6,13 @@ module Onebox
       include Engine
       include LayoutSupport
       include HTML
+      include ActionView::Helpers::NumberHelper
 
       matches_regexp(
         %r{^https?://(mobile\.|www\.)?twitter\.com/.+?/status(es)?/\d+(/(video|photo)/\d?+)?+(/?\?.*)?/?$},
       )
       always_https
 
-      def self.===(other)
-        client = Onebox.options.twitter_client
-        client && !client.twitter_credentials_missing? && super
-      end
-
       def http_params
         { "User-Agent" => "DiscourseBot/1.0" }
       end
@@ -27,10 +23,46 @@ module Onebox
 
       private
 
+      def get_twitter_data
+        response =
+          begin
+            Onebox::Helpers.fetch_response(url, headers: http_params)
+          rescue StandardError
+            return nil
+          end
+        html = Nokogiri.HTML(response)
+        twitter_data = {}
+        html
+          .css("meta")
+          .each do |m|
+            if m.attribute("property") && m.attribute("property").to_s.match(/^og:/i)
+              m_content = m.attribute("content").to_s.strip
+              m_property = m.attribute("property").to_s.gsub("og:", "").gsub(":", "_")
+              twitter_data[m_property.to_sym] = m_content
+            end
+          end
+        twitter_data
+      end
+
       def match
         @match ||= @url.match(%r{twitter\.com/.+?/status(es)?/(?<id>\d+)})
       end
 
+      def twitter_data
+        @twitter_data ||= get_twitter_data
+      end
+
+      def guess_tweet_index
+        usernames = meta_tags_data("additionalName").compact
+        usernames.each_with_index do |username, index|
+          return index if twitter_data[:url].to_s.include?(username)
+        end
+      end
+
+      def tweet_index
+        @tweet_index ||= guess_tweet_index
+      end
+
       def client
         Onebox.options.twitter_client
       end
@@ -39,66 +71,139 @@ module Onebox
         client && !client.twitter_credentials_missing?
       end
 
-      def raw
+      def symbolize_keys(obj)
-        @raw ||= client.status(match[:id]).to_hash if twitter_api_credentials_present?
+        case obj
+        when Array
+          obj.map { |item| symbolize_keys(item) }
+        when Hash
+          obj.each_with_object({}) do |(key, value), result|
+            result[key.to_sym] = symbolize_keys(value)
+          end
+        else
+          obj
+        end
       end
 
-      def access(*keys)
+      def raw
-        keys.reduce(raw) do |memo, key|
+        if twitter_api_credentials_present?
-          next unless memo
+          @raw ||= symbolize_keys(client.status(match[:id]))
-          memo[key] || memo[key.to_s]
+        else
+          super
         end
       end
 
       def tweet
+        if twitter_api_credentials_present?
           client.prettify_tweet(raw)&.strip
+        else
+          twitter_data[:description].gsub(/“(.+?)”/im) { $1 } if twitter_data[:description]
+        end
       end
 
       def timestamp
-        date = DateTime.strptime(access(:created_at), "%a %b %d %H:%M:%S %z %Y")
+        if twitter_api_credentials_present? && (created_at = raw.dig(:data, :created_at))
-        user_offset = access(:user, :utc_offset).to_i
+          date = DateTime.strptime(created_at, "%Y-%m-%dT%H:%M:%S.%L%z")
-        offset = (user_offset >= 0 ? "+" : "-") + Time.at(user_offset.abs).gmtime.strftime("%H%M")
+          date.strftime("%-l:%M %p - %-d %b %Y")
-        date.new_offset(offset).strftime("%-l:%M %p - %-d %b %Y")
+        end
       end
 
       def title
-        access(:user, :name)
+        if twitter_api_credentials_present?
+          raw.dig(:includes, :users)&.first&.dig(:name)
+        else
+          meta_tags_data("givenName")[tweet_index]
+        end
       end
 
       def screen_name
-        access(:user, :screen_name)
+        if twitter_api_credentials_present?
+          raw.dig(:includes, :users)&.first&.dig(:username)
+        else
+          meta_tags_data("additionalName")[tweet_index]
+        end
       end
 
       def avatar
-        access(:user, :profile_image_url_https).sub("normal", "400x400")
+        if twitter_api_credentials_present?
+          raw.dig(:includes, :users)&.first&.dig(:profile_image_url)
+        end
       end
 
       def likes
-        prettify_number(access(:favorite_count).to_i)
+        if twitter_api_credentials_present?
+          prettify_number(raw.dig(:data, :public_metrics, :like_count).to_i)
+        end
       end
 
       def retweets
-        prettify_number(access(:retweet_count).to_i)
+        if twitter_api_credentials_present?
+          prettify_number(raw.dig(:data, :public_metrics, :retweet_count).to_i)
+        end
       end
 
       def quoted_full_name
-        access(:quoted_status, :user, :name)
+        if twitter_api_credentials_present? && quoted_tweet_author.present?
+          quoted_tweet_author[:name]
+        end
       end
 
       def quoted_screen_name
-        access(:quoted_status, :user, :screen_name)
+        if twitter_api_credentials_present? && quoted_tweet_author.present?
+          quoted_tweet_author[:username]
+        end
       end
 
-      def quoted_tweet
+      def quoted_text
-        access(:quoted_status, :full_text)
+        quoted_tweet[:text] if twitter_api_credentials_present? && quoted_tweet.present?
       end
 
       def quoted_link
-        "https://twitter.com/#{quoted_screen_name}/status/#{access(:quoted_status, :id)}"
+        if twitter_api_credentials_present?
+          "https://twitter.com/#{quoted_screen_name}/status/#{quoted_status_id}"
+        end
+      end
+
+      def quoted_status_id
+        raw.dig(:data, :referenced_tweets)&.find { |ref| ref[:type] == "quoted" }&.dig(:id)
+      end
+
+      def quoted_tweet
+        raw.dig(:includes, :tweets)&.find { |tweet| tweet[:id] == quoted_status_id }
+      end
+
+      def quoted_tweet_author
+        raw.dig(:includes, :users)&.find { |user| user[:id] == quoted_tweet&.dig(:author_id) }
       end
 
       def prettify_number(count)
-        count > 0 ? client.prettify_number(count) : nil
+        if count > 0
+          number_to_human(
+            count,
+            format: "%n%u",
+            precision: 2,
+            units: {
+              thousand: "K",
+              million: "M",
+              billion: "B",
+            },
+          )
+        end
+      end
+
+      def attr_at_css(css_property, attribute_name)
+        raw.at_css(css_property)&.attr(attribute_name)
+      end
+
+      def meta_tags_data(attribute_name)
+        data = []
+        raw
+          .css("meta")
+          .each do |m|
+            if m.attribute("itemprop") && m.attribute("itemprop").to_s.strip == attribute_name
+              data.push(m.attribute("content").to_s.strip)
+            end
+          end
+        data
       end
 
       def data
@@ -111,7 +216,7 @@ module Onebox
           avatar: avatar,
           likes: likes,
           retweets: retweets,
-          quoted_tweet: quoted_tweet,
+          quoted_text: quoted_text,
           quoted_full_name: quoted_full_name,
           quoted_screen_name: quoted_screen_name,
           quoted_link: quoted_link,

lib/onebox/templates/twitterstatus.mustache

@@ -4,15 +4,15 @@
 
 <div class="tweet">
   <span class="tweet-description">{{{tweet}}}</span>
-  {{#quoted_tweet}}
+  {{#quoted_text}}
     <div class="quoted">
       <a class="quoted-link" href="{{quoted_link}}">
         <p class="quoted-title">{{quoted_full_name}} <span>@{{quoted_screen_name}}</span></p>
       </a>
 
-      <div>{{quoted_tweet}}</div>
+      <div>{{quoted_text}}</div>
     </div>
-  {{/quoted_tweet}}
+  {{/quoted_text}}
 </div>
 
 <div class="date">

lib/twitter_api.rb

@@ -3,17 +3,21 @@
 # lightweight Twitter api calls
 class TwitterApi
   class << self
-    include ActionView::Helpers::NumberHelper
-
     BASE_URL = "https://api.twitter.com"
+    URL_PARAMS = %w[
+      tweet.fields=id,author_id,text,created_at,entities,referenced_tweets,public_metrics
+      user.fields=id,name,username,profile_image_url
+      media.fields=type,height,width,variants,preview_image_url,url
+      expansions=attachments.media_keys,referenced_tweets.id.author_id
+    ]
 
     def prettify_tweet(tweet)
-      text = tweet["full_text"].dup
+      text = tweet[:data][:text].dup.to_s
-      if (entities = tweet["entities"]) && (urls = entities["urls"])
+      if (entities = tweet[:data][:entities]) && (urls = entities[:urls])
         urls.each do |url|
           text.gsub!(
-            url["url"],
+            url[:url],
-            "<a target='_blank' href='#{url["expanded_url"]}'>#{url["display_url"]}</a>",
+            "<a target='_blank' href='#{url[:expanded_url]}'>#{url[:display_url]}</a>",
           )
         end
       end
@@ -22,25 +26,23 @@ class TwitterApi
 
       result = Rinku.auto_link(text, :all, 'target="_blank"').to_s
 
-      if tweet["extended_entities"] && media = tweet["extended_entities"]["media"]
+      if tweet[:includes] && media = tweet[:includes][:media]
         media.each do |m|
-          if m["type"] == "photo"
+          if m[:type] == "photo"
-            if large = m["sizes"]["large"]
+            result << "<div class='tweet-images'><img class='tweet-image' src='#{m[:url]}' width='#{m[:width]}' height='#{m[:height]}'></div>"
-              result << "<div class='tweet-images'><img class='tweet-image' src='#{m["media_url_https"]}' width='#{large["w"]}' height='#{large["h"]}'></div>"
+          elsif m[:type] == "video" || m[:type] == "animated_gif"
-            end
-          elsif m["type"] == "video" || m["type"] == "animated_gif"
             video_to_display =
-              m["video_info"]["variants"]
+              m[:variants]
-                .select { |v| v["content_type"] == "video/mp4" }
+                .select { |v| v[:content_type] == "video/mp4" }
-                .sort { |v| v["bitrate"] }
+                .sort { |v| v[:bit_rate] }
                 .last # choose highest bitrate
 
-            if video_to_display && url = video_to_display["url"]
+            if video_to_display && url = video_to_display[:url]
-              width = m["sizes"]["large"]["w"]
+              width = m[:width]
-              height = m["sizes"]["large"]["h"]
+              height = m[:height]
 
               attributes =
-                if m["type"] == "animated_gif"
+                if m[:type] == "animated_gif"
                   %w[playsinline loop muted autoplay disableRemotePlayback disablePictureInPicture]
                 else
                   %w[controls playsinline]
@@ -52,7 +54,7 @@ class TwitterApi
                     <video class='tweet-video' #{attributes}
                       width='#{width}'
                       height='#{height}'
-                      poster='#{m["media_url_https"]}'>
+                      poster='#{m[:preview_image_url]}'>
                       <source src='#{url}' type="video/mp4">
                     </video>
                   </div>
@@ -66,19 +68,6 @@ class TwitterApi
       result
     end
 
-    def prettify_number(count)
-      number_to_human(
-        count,
-        format: "%n%u",
-        precision: 2,
-        units: {
-          thousand: "K",
-          million: "M",
-          billion: "B",
-        },
-      )
-    end
-
     def tweet_for(id)
       JSON.parse(twitter_get(tweet_uri_for(id)))
     end
@@ -111,7 +100,7 @@ class TwitterApi
     end
 
     def tweet_uri_for(id)
-      URI.parse "#{BASE_URL}/1.1/statuses/show.json?id=#{id}&tweet_mode=extended"
+      URI.parse "#{BASE_URL}/2/tweets/#{id}?#{URL_PARAMS.join("&")}"
     end
 
     def twitter_get(uri)

spec/lib/onebox/engine/twitter_status_onebox_spec.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+include ActionView::Helpers::NumberHelper
 
 RSpec.describe Onebox::Engine::TwitterStatusOnebox do
   shared_examples_for "#to_html" do
@@ -42,37 +43,35 @@ RSpec.describe Onebox::Engine::TwitterStatusOnebox do
 
   shared_context "with standard tweet info" do
     before do
-      @link = "https://twitter.com/vyki_e/status/363116819147538433"
+      @link = "https://twitter.com/MKBHD/status/1625192182859632661"
       @onebox_fixture = "twitterstatus"
     end
 
-    let(:full_name) { "Vyki Englert" }
+    let(:full_name) { "Marques Brownlee" }
-    let(:screen_name) { "vyki_e" }
+    let(:screen_name) { "MKBHD" }
-    let(:avatar) { "732349210264133632/RTNgZLrm_400x400.jpg" }
+    let(:avatar) { "https://pbs.twimg.com/profile_images/1468001914302390278/B_Xv_8gu_normal.jpg" }
-    let(:timestamp) { "6:59 PM - 1 Aug 2013" }
+    let(:timestamp) { "5:56 PM - 13 Feb 2023" }
     let(:link) { @link }
-    let(:favorite_count) { "0" }
+    let(:favorite_count) { "47K" }
-    let(:retweets_count) { "0" }
+    let(:retweets_count) { "1.5K" }
   end
 
   shared_context "with quoted tweet info" do
     before do
-      @link = "https://twitter.com/metallica/status/1128068672289890305"
+      @link = "https://twitter.com/Metallica/status/1128068672289890305"
       @onebox_fixture = "twitterstatus_quoted"
 
-      stub_request(:get, @link.downcase).to_return(
+      stub_request(:head, @link)
-        status: 200,
+      stub_request(:get, @link).to_return(status: 200, body: onebox_response(@onebox_fixture))
-        body: onebox_response(@onebox_fixture),
-      )
     end
 
     let(:full_name) { "Metallica" }
     let(:screen_name) { "Metallica" }
-    let(:avatar) { "profile_images/766360293953802240/kt0hiSmv_400x400.jpg" }
+    let(:avatar) { "https://pbs.twimg.com/profile_images/1597280886809952256/gsJvGiqU_normal.jpg" }
     let(:timestamp) { "10:45 PM - 13 May 2019" }
     let(:link) { @link }
-    let(:favorite_count) { "1.7K" }
+    let(:favorite_count) { "1.4K" }
-    let(:retweets_count) { "201" }
+    let(:retweets_count) { "170" }
   end
 
   shared_context "with featured image info" do
@@ -88,11 +87,11 @@ RSpec.describe Onebox::Engine::TwitterStatusOnebox do
 
     let(:full_name) { "Jeff Atwood" }
     let(:screen_name) { "codinghorror" }
-    let(:avatar) { "" }
+    let(:avatar) { "https://pbs.twimg.com/profile_images/1517287320235298816/Qx-O6UCY_normal.jpg" }
     let(:timestamp) { "3:02 PM - 27 Jun 2021" }
     let(:link) { @link }
     let(:favorite_count) { "90" }
-    let(:retweets_count) { "0" }
+    let(:retweets_count) { "5" }
   end
 
   shared_examples "includes quoted tweet data" do
@@ -119,9 +118,9 @@ RSpec.describe Onebox::Engine::TwitterStatusOnebox do
     let(:link) { "https://twitter.com/discourse/status/1428031057186627589" }
     let(:html) { described_class.new(link).to_html }
 
-    it "does not match the url" do
+    it "does match the url" do
       onebox = Onebox::Matcher.new(link, { allowed_iframe_regexes: [/.*/] }).oneboxed
-      expect(onebox).not_to be(described_class)
+      expect(onebox).to be(described_class)
     end
 
     it "logs a warn message if rate limited" do
@@ -137,7 +136,7 @@ RSpec.describe Onebox::Engine::TwitterStatusOnebox do
 
       stub_request(
         :get,
-        "https://api.twitter.com/1.1/statuses/show.json?id=1428031057186627589&tweet_mode=extended",
+        "https://api.twitter.com/2/tweets/1428031057186627589?tweet.fields=id,author_id,text,created_at,entities,referenced_tweets,public_metrics&user.fields=id,name,username,profile_image_url&media.fields=type,height,width,variants,preview_image_url,url&expansions=attachments.media_keys,referenced_tweets.id.author_id",
       ).to_return(status: 429, body: "{}", headers: {})
 
       Rails.logger.expects(:warn).with(regexp_matches(/rate limit/)).at_least_once
@@ -154,7 +153,6 @@ RSpec.describe Onebox::Engine::TwitterStatusOnebox do
           status: api_response,
           prettify_tweet: tweet_content,
           twitter_credentials_missing?: false,
-          prettify_number: favorite_count,
         )
 
       @previous_options = Onebox.options.to_h
@@ -164,118 +162,47 @@ RSpec.describe Onebox::Engine::TwitterStatusOnebox do
     after { Onebox.options = @previous_options }
 
     context "with a standard tweet" do
-      let(:tweet_content) do
+      let(:tweet_content) { "I've never played Minecraft" }
-        "I'm a sucker for pledges.  <a href='https://twitter.com/Peers' target='_blank'>@Peers</a> Pledge <a href='https://twitter.com/search?q=%23sharingeconomy' target='_blank'>#sharingeconomy</a> <a target='_blank' href='http://www.peers.org/action/peers-pledgea/'>peers.org/action/peers-p…</a>"
-      end
 
       let(:api_response) do
         {
-          created_at: "Fri Aug 02 01:59:30 +0000 2013",
+          data: {
-          id: 363_116_819_147_538_400,
+            edit_history_tweet_ids: ["1625192182859632661"],
-          id_str: "363116819147538433",
+            created_at: "2023-02-13T17:56:25.000Z",
-          text: "I'm a sucker for pledges. @Peers Pledge #sharingeconomy http://t.co/T4Sc47KAzh",
+            author_id: "29873662",
-          truncated: false,
+            public_metrics: {
+              retweet_count: 1460,
+              reply_count: 2734,
+              like_count: 46_756,
+              quote_count: 477,
+              bookmark_count: 168,
+              impression_count: 4_017_878,
+            },
+            text: "I've never played Minecraft",
             entities: {
-            hashtags: [{ text: "sharingeconomy", indices: [41, 56] }],
+              annotations: [
-            symbols: [],
-            user_mentions: [
                 {
-                screen_name: "peers",
+                  start: 18,
-                name: "Peers",
+                  end: 26,
-                id: 1_428_357_889,
+                  probability: 0.9807,
-                id_str: "1428357889",
+                  type: "Other",
-                indices: [27, 33],
+                  normalized_text: "Minecraft",
-              },
-            ],
-            urls: [
-              {
-                url: "http://t.co/T4Sc47KAzh",
-                expanded_url: "http://www.peers.org/action/peers-pledgea/",
-                display_url: "peers.org/action/peers-p…",
-                indices: [57, 79],
                 },
               ],
             },
-          source:
+            id: "1625192182859632661",
-            "<a href=\"https://dev.twitter.com/docs/tfw\" rel=\"nofollow\">Twitter for Websites</a>",
+          },
-          in_reply_to_status_id: nil,
+          includes: {
-          in_reply_to_status_id_str: nil,
+            users: [
-          in_reply_to_user_id: nil,
-          in_reply_to_user_id_str: nil,
-          in_reply_to_screen_name: nil,
-          user: {
-            id: 1_087_064_150,
-            id_str: "1087064150",
-            name: "Vyki Englert",
-            screen_name: "vyki_e",
-            location: "Los Angeles, CA",
-            description:
-              "Rides bikes, writes code, likes maps. @CompilerLA / @CityGrows / Brigade Captain @HackforLA",
-            url: "http://t.co/YCAP3asRG1",
-            entities: {
-              url: {
-                urls: [
               {
-                    url: "http://t.co/YCAP3asRG1",
+                name: "Marques Brownlee",
-                    expanded_url: "http://www.compiler.la",
+                id: "29873662",
-                    display_url: "compiler.la",
-                    indices: [0, 22],
-                  },
-                ],
-              },
-              description: {
-                urls: [],
-              },
-            },
-            protected: false,
-            followers_count: 1128,
-            friends_count: 2244,
-            listed_count: 83,
-            created_at: "Sun Jan 13 19:53:00 +0000 2013",
-            favourites_count: 2928,
-            utc_offset: -25_200,
-            time_zone: "Pacific Time (US & Canada)",
-            geo_enabled: true,
-            verified: false,
-            statuses_count: 3295,
-            lang: "en",
-            contributors_enabled: false,
-            is_translator: false,
-            is_translation_enabled: false,
-            profile_background_color: "ACDED6",
-            profile_background_image_url: "http://abs.twimg.com/images/themes/theme18/bg.gif",
-            profile_background_image_url_https:
-              "https://abs.twimg.com/images/themes/theme18/bg.gif",
-            profile_background_tile: false,
                 profile_image_url:
-              "http://pbs.twimg.com/profile_images/732349210264133632/RTNgZLrm_normal.jpg",
+                  "https://pbs.twimg.com/profile_images/1468001914302390278/B_Xv_8gu_normal.jpg",
-            profile_image_url_https:
+                username: "MKBHD",
-              "https://pbs.twimg.com/profile_images/732349210264133632/RTNgZLrm_normal.jpg",
+              },
-            profile_banner_url: "https://pbs.twimg.com/profile_banners/1087064150/1424315468",
+            ],
-            profile_link_color: "4E99D1",
-            profile_sidebar_border_color: "EEEEEE",
-            profile_sidebar_fill_color: "F6F6F6",
-            profile_text_color: "333333",
-            profile_use_background_image: true,
-            has_extended_profile: false,
-            default_profile: false,
-            default_profile_image: false,
-            following: false,
-            follow_request_sent: false,
-            notifications: false,
           },
-          geo: nil,
-          coordinates: nil,
-          place: nil,
-          contributors: nil,
-          is_quote_status: false,
-          retweet_count: 0,
-          favorite_count: 0,
-          favorited: false,
-          retweeted: false,
-          possibly_sensitive: false,
-          possibly_sensitive_appealable: false,
-          lang: "en",
         }
       end
 
@@ -293,372 +220,167 @@ RSpec.describe Onebox::Engine::TwitterStatusOnebox do
 
       let(:api_response) do
         {
-          created_at: "Mon May 13 22:45:04 +0000 2019",
+          data: {
-          id: 1_128_068_672_289_890_305,
+            text:
-          id_str: "1128068672289890305",
-          full_text:
               "Thank you to everyone who came out for #MetInParis last night for helping us support @EMMAUSolidarite &amp; @PompiersParis. #AWMH #MetalicaGivesBack https://t.co/gLtZSdDFmN",
-          truncated: false,
+            edit_history_tweet_ids: ["1128068672289890305"],
-          display_text_range: [0, 148],
             entities: {
-            hashtags: [
+              mentions: [
-              { text: "MetInParis", indices: [39, 50] },
+                { start: 85, end: 101, username: "EMMAUSolidarite", id: "2912493406" },
-              { text: "AWMH", indices: [124, 129] },
+                { start: 108, end: 122, username: "PompiersParis", id: "1342191438" },
-              { text: "MetalicaGivesBack", indices: [130, 148] },
-            ],
-            symbols: [],
-            user_mentions: [
-              {
-                screen_name: "EMMAUSolidarite",
-                name: "EMMAÜS Solidarité",
-                id: 2_912_493_406,
-                id_str: "2912493406",
-                indices: [85, 101],
-              },
-              {
-                screen_name: "PompiersParis",
-                name: "Pompiers de Paris",
-                id: 1_342_191_438,
-                id_str: "1342191438",
-                indices: [108, 122],
-              },
               ],
               urls: [
                 {
+                  start: 149,
+                  end: 172,
                   url: "https://t.co/gLtZSdDFmN",
                   expanded_url: "https://twitter.com/AWMHFoundation/status/1127646016931487744",
                   display_url: "twitter.com/AWMHFoundation…",
-                indices: [149, 172],
                 },
               ],
-          },
-          source: "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
-          in_reply_to_status_id: nil,
-          in_reply_to_status_id_str: nil,
-          in_reply_to_user_id: nil,
-          in_reply_to_user_id_str: nil,
-          in_reply_to_screen_name: nil,
-          user: {
-            id: 238_475_531,
-            id_str: "238475531",
-            name: "Metallica",
-            screen_name: "Metallica",
-            location: "San Francisco, CA",
-            description: "http://t.co/EAkqroM0OA | http://t.co/BEu6OVRhKG",
-            url: "http://t.co/kVxaQpmqSI",
-            entities: {
-              url: {
-                urls: [
-                  {
-                    url: "http://t.co/kVxaQpmqSI",
-                    expanded_url: "http://www.metallica.com",
-                    display_url: "metallica.com",
-                    indices: [0, 22],
-                  },
-                ],
-              },
-              description: {
-                urls: [
-                  {
-                    url: "http://t.co/EAkqroM0OA",
-                    expanded_url: "http://metallica.com",
-                    display_url: "metallica.com",
-                    indices: [0, 22],
-                  },
-                  {
-                    url: "http://t.co/BEu6OVRhKG",
-                    expanded_url: "http://livemetallica.com",
-                    display_url: "livemetallica.com",
-                    indices: [25, 47],
-                  },
-                ],
-              },
-            },
-            protected: false,
-            followers_count: 5_760_661,
-            friends_count: 31,
-            listed_count: 12_062,
-            created_at: "Sat Jan 15 07:34:59 +0000 2011",
-            favourites_count: 567,
-            utc_offset: nil,
-            time_zone: nil,
-            geo_enabled: true,
-            verified: true,
-            statuses_count: 3764,
-            lang: nil,
-            contributors_enabled: false,
-            is_translator: false,
-            is_translation_enabled: false,
-            profile_background_color: "000000",
-            profile_background_image_url: "http://abs.twimg.com/images/themes/theme9/bg.gif",
-            profile_background_image_url_https: "https://abs.twimg.com/images/themes/theme9/bg.gif",
-            profile_background_tile: false,
-            profile_image_url:
-              "http://pbs.twimg.com/profile_images/766360293953802240/kt0hiSmv_normal.jpg",
-            profile_image_url_https:
-              "https://pbs.twimg.com/profile_images/766360293953802240/kt0hiSmv_normal.jpg",
-            profile_banner_url: "https://pbs.twimg.com/profile_banners/238475531/1479538295",
-            profile_link_color: "2FC2EF",
-            profile_sidebar_border_color: "000000",
-            profile_sidebar_fill_color: "252429",
-            profile_text_color: "666666",
-            profile_use_background_image: false,
-            has_extended_profile: false,
-            default_profile: false,
-            default_profile_image: false,
-            following: false,
-            follow_request_sent: false,
-            notifications: false,
-            translator_type: "regular",
-          },
-          geo: nil,
-          coordinates: nil,
-          place: nil,
-          contributors: nil,
-          is_quote_status: true,
-          quoted_status_id: 1_127_646_016_931_487_744,
-          quoted_status_id_str: "1127646016931487744",
-          quoted_status_permalink: {
-            url: "https://t.co/gLtZSdDFmN",
-            expanded: "https://twitter.com/AWMHFoundation/status/1127646016931487744",
-            display: "twitter.com/AWMHFoundation…",
-          },
-          quoted_status: {
-            created_at: "Sun May 12 18:45:35 +0000 2019",
-            id: 1_127_646_016_931_487_744,
-            id_str: "1127646016931487744",
-            full_text:
-              "If you bought a ticket for tonight’s @Metallica show at Stade de France, you have helped contribute to @EMMAUSolidarite &amp; @PompiersParis. #MetallicaGivesBack #AWMH #MetInParis https://t.co/wlUtDQbQEK",
-            truncated: false,
-            display_text_range: [0, 179],
-            entities: {
               hashtags: [
-                { text: "MetallicaGivesBack", indices: [142, 161] },
+                { start: 39, end: 50, tag: "MetInParis" },
-                { text: "AWMH", indices: [162, 167] },
+                { start: 124, end: 129, tag: "AWMH" },
-                { text: "MetInParis", indices: [168, 179] },
+                { start: 130, end: 148, tag: "MetalicaGivesBack" },
               ],
-              symbols: [],
+              annotations: [
-              user_mentions: [
                 {
-                  screen_name: "Metallica",
+                  start: 40,
+                  end: 49,
+                  probability: 0.6012,
+                  type: "Other",
+                  normalized_text: "MetInParis",
+                },
+                {
+                  start: 125,
+                  end: 128,
+                  probability: 0.5884,
+                  type: "Other",
+                  normalized_text: "AWMH",
+                },
+                {
+                  start: 131,
+                  end: 147,
+                  probability: 0.6366,
+                  type: "Other",
+                  normalized_text: "MetalicaGivesBack",
+                },
+              ],
+            },
+            id: "1128068672289890305",
+            referenced_tweets: [{ type: "quoted", id: "1127646016931487744" }],
+            created_at: "2019-05-13T22:45:04.000Z",
+            public_metrics: {
+              retweet_count: 171,
+              reply_count: 21,
+              like_count: 1424,
+              quote_count: 0,
+              bookmark_count: 2,
+              impression_count: 0,
+            },
+            author_id: "238475531",
+          },
+          includes: {
+            users: [
+              {
+                profile_image_url:
+                  "https://pbs.twimg.com/profile_images/1597280886809952256/gsJvGiqU_normal.jpg",
                 name: "Metallica",
-                  id: 238_475_531,
+                id: "238475531",
-                  id_str: "238475531",
+                username: "Metallica",
-                  indices: [37, 47],
               },
               {
-                  screen_name: "EMMAUSolidarite",
+                profile_image_url:
-                  name: "EMMAÜS Solidarité",
+                  "https://pbs.twimg.com/profile_images/935181032185241600/D8FoOIRJ_normal.jpg",
-                  id: 2_912_493_406,
-                  id_str: "2912493406",
-                  indices: [103, 119],
-                },
-                {
-                  screen_name: "PompiersParis",
-                  name: "Pompiers de Paris",
-                  id: 1_342_191_438,
-                  id_str: "1342191438",
-                  indices: [126, 140],
-                },
-              ],
-              urls: [],
-              media: [
-                {
-                  id: 1_127_645_176_183_250_944,
-                  id_str: "1127645176183250944",
-                  indices: [180, 203],
-                  media_url: "http://pbs.twimg.com/media/D6YzUC8V4AApDdF.jpg",
-                  media_url_https: "https://pbs.twimg.com/media/D6YzUC8V4AApDdF.jpg",
-                  url: "https://t.co/wlUtDQbQEK",
-                  display_url: "pic.twitter.com/wlUtDQbQEK",
-                  expanded_url:
-                    "https://twitter.com/AWMHFoundation/status/1127646016931487744/photo/1",
-                  type: "photo",
-                  sizes: {
-                    large: {
-                      w: 2048,
-                      h: 1498,
-                      resize: "fit",
-                    },
-                    thumb: {
-                      w: 150,
-                      h: 150,
-                      resize: "crop",
-                    },
-                    medium: {
-                      w: 1200,
-                      h: 877,
-                      resize: "fit",
-                    },
-                    small: {
-                      w: 680,
-                      h: 497,
-                      resize: "fit",
-                    },
-                  },
-                },
-              ],
-            },
-            extended_entities: {
-              media: [
-                {
-                  id: 1_127_645_176_183_250_944,
-                  id_str: "1127645176183250944",
-                  indices: [180, 203],
-                  media_url: "http://pbs.twimg.com/media/D6YzUC8V4AApDdF.jpg",
-                  media_url_https: "https://pbs.twimg.com/media/D6YzUC8V4AApDdF.jpg",
-                  url: "https://t.co/wlUtDQbQEK",
-                  display_url: "pic.twitter.com/wlUtDQbQEK",
-                  expanded_url:
-                    "https://twitter.com/AWMHFoundation/status/1127646016931487744/photo/1",
-                  type: "photo",
-                  sizes: {
-                    large: {
-                      w: 2048,
-                      h: 1498,
-                      resize: "fit",
-                    },
-                    thumb: {
-                      w: 150,
-                      h: 150,
-                      resize: "crop",
-                    },
-                    medium: {
-                      w: 1200,
-                      h: 877,
-                      resize: "fit",
-                    },
-                    small: {
-                      w: 680,
-                      h: 497,
-                      resize: "fit",
-                    },
-                  },
-                },
-                {
-                  id: 1_127_645_195_384_774_657,
-                  id_str: "1127645195384774657",
-                  indices: [180, 203],
-                  media_url: "http://pbs.twimg.com/media/D6YzVKeV4AEPpSQ.jpg",
-                  media_url_https: "https://pbs.twimg.com/media/D6YzVKeV4AEPpSQ.jpg",
-                  url: "https://t.co/wlUtDQbQEK",
-                  display_url: "pic.twitter.com/wlUtDQbQEK",
-                  expanded_url:
-                    "https://twitter.com/AWMHFoundation/status/1127646016931487744/photo/1",
-                  type: "photo",
-                  sizes: {
-                    thumb: {
-                      w: 150,
-                      h: 150,
-                      resize: "crop",
-                    },
-                    medium: {
-                      w: 1200,
-                      h: 922,
-                      resize: "fit",
-                    },
-                    small: {
-                      w: 680,
-                      h: 522,
-                      resize: "fit",
-                    },
-                    large: {
-                      w: 2048,
-                      h: 1574,
-                      resize: "fit",
-                    },
-                  },
-                },
-              ],
-            },
-            source: "<a href=\"http://twitter.com\" rel=\"nofollow\">Twitter Web Client</a>",
-            in_reply_to_status_id: nil,
-            in_reply_to_status_id_str: nil,
-            in_reply_to_user_id: nil,
-            in_reply_to_user_id_str: nil,
-            in_reply_to_screen_name: nil,
-            user: {
-              id: 886_959_980_254_871_552,
-              id_str: "886959980254871552",
                 name: "All Within My Hands Foundation",
-              screen_name: "AWMHFoundation",
+                id: "886959980254871552",
-              location: "",
+                username: "AWMHFoundation",
-              description: "",
+              },
-              url: "https://t.co/KgwIPrVVhg",
+            ],
+            tweets: [
+              {
+                text:
+                  "If you bought a ticket for tonight’s @Metallica show at Stade de France, you have helped contribute to @EMMAUSolidarite &amp; @PompiersParis. #MetallicaGivesBack #AWMH #MetInParis https://t.co/wlUtDQbQEK",
+                edit_history_tweet_ids: ["1127646016931487744"],
                 entities: {
-                url: {
+                  mentions: [
+                    { start: 37, end: 47, username: "Metallica", id: "238475531" },
+                    { start: 103, end: 119, username: "EMMAUSolidarite", id: "2912493406" },
+                    { start: 126, end: 140, username: "PompiersParis", id: "1342191438" },
+                  ],
                   urls: [
                     {
-                      url: "https://t.co/KgwIPrVVhg",
+                      start: 180,
-                      expanded_url: "http://allwithinmyhands.org",
+                      end: 203,
-                      display_url: "allwithinmyhands.org",
+                      url: "https://t.co/wlUtDQbQEK",
-                      indices: [0, 23],
+                      expanded_url:
+                        "https://twitter.com/AWMHFoundation/status/1127646016931487744/photo/1",
+                      display_url: "pic.twitter.com/wlUtDQbQEK",
+                      media_key: "3_1127645176183250944",
+                    },
+                    {
+                      start: 180,
+                      end: 203,
+                      url: "https://t.co/wlUtDQbQEK",
+                      expanded_url:
+                        "https://twitter.com/AWMHFoundation/status/1127646016931487744/photo/1",
+                      display_url: "pic.twitter.com/wlUtDQbQEK",
+                      media_key: "3_1127645195384774657",
+                    },
+                  ],
+                  hashtags: [
+                    { start: 142, end: 161, tag: "MetallicaGivesBack" },
+                    { start: 162, end: 167, tag: "AWMH" },
+                    { start: 168, end: 179, tag: "MetInParis" },
+                  ],
+                  annotations: [
+                    {
+                      start: 56,
+                      end: 70,
+                      probability: 0.7845,
+                      type: "Place",
+                      normalized_text: "Stade de France",
+                    },
+                    {
+                      start: 143,
+                      end: 160,
+                      probability: 0.5569,
+                      type: "Organization",
+                      normalized_text: "MetallicaGivesBack",
+                    },
+                    {
+                      start: 163,
+                      end: 166,
+                      probability: 0.4496,
+                      type: "Other",
+                      normalized_text: "AWMH",
+                    },
+                    {
+                      start: 169,
+                      end: 178,
+                      probability: 0.3784,
+                      type: "Place",
+                      normalized_text: "MetInParis",
                     },
                   ],
                 },
-                description: {
+                id: "1127646016931487744",
-                  urls: [],
+                created_at: "2019-05-12T18:45:35.000Z",
+                attachments: {
+                  media_keys: %w[3_1127645176183250944 3_1127645195384774657],
                 },
+                public_metrics: {
+                  retweet_count: 34,
+                  reply_count: 5,
+                  like_count: 241,
+                  quote_count: 9,
+                  bookmark_count: 0,
+                  impression_count: 0,
                 },
-              protected: false,
+                author_id: "886959980254871552",
-              followers_count: 5962,
-              friends_count: 6,
-              listed_count: 15,
-              created_at: "Mon Jul 17 14:45:13 +0000 2017",
-              favourites_count: 30,
-              utc_offset: nil,
-              time_zone: nil,
-              geo_enabled: true,
-              verified: false,
-              statuses_count: 241,
-              lang: nil,
-              contributors_enabled: false,
-              is_translator: false,
-              is_translation_enabled: false,
-              profile_background_color: "000000",
-              profile_background_image_url: "http://abs.twimg.com/images/themes/theme1/bg.png",
-              profile_background_image_url_https:
-                "https://abs.twimg.com/images/themes/theme1/bg.png",
-              profile_background_tile: false,
-              profile_image_url:
-                "http://pbs.twimg.com/profile_images/935181032185241600/D8FoOIRJ_normal.jpg",
-              profile_image_url_https:
-                "https://pbs.twimg.com/profile_images/935181032185241600/D8FoOIRJ_normal.jpg",
-              profile_banner_url:
-                "https://pbs.twimg.com/profile_banners/886959980254871552/1511799663",
-              profile_link_color: "000000",
-              profile_sidebar_border_color: "000000",
-              profile_sidebar_fill_color: "000000",
-              profile_text_color: "000000",
-              profile_use_background_image: false,
-              has_extended_profile: false,
-              default_profile: false,
-              default_profile_image: false,
-              following: false,
-              follow_request_sent: false,
-              notifications: false,
-              translator_type: "none",
               },
-            geo: nil,
+            ],
-            coordinates: nil,
-            place: nil,
-            contributors: nil,
-            is_quote_status: false,
-            retweet_count: 46,
-            favorite_count: 275,
-            favorited: false,
-            retweeted: false,
-            possibly_sensitive: false,
-            possibly_sensitive_appealable: false,
-            lang: "en",
           },
-          retweet_count: 201,
-          favorite_count: 1664,
-          favorited: false,
-          retweeted: false,
-          possibly_sensitive: false,
-          possibly_sensitive_appealable: false,
-          lang: "en",
         }
       end
 

spec/lib/oneboxer_spec.rb

@@ -777,7 +777,7 @@ RSpec.describe Oneboxer do
 
       stub_request(
         :get,
-        "https://api.twitter.com/1.1/statuses/show.json?id=1428031057186627589&tweet_mode=extended",
+        "https://api.twitter.com/2/tweets/1428031057186627589?tweet.fields=id,author_id,text,created_at,entities,referenced_tweets,public_metrics&user.fields=id,name,username,profile_image_url&media.fields=type,height,width,variants,preview_image_url,url&expansions=attachments.media_keys,referenced_tweets.id.author_id",
       ).to_return(status: 429, body: "{}", headers: {})
 
       stub_request(:post, "https://api.twitter.com/oauth2/token").to_return(