FEATURE: Show localized posts and topics based on user's locale (#32618)

Related:
- https://github.com/discourse/discourse-translator/pull/205
- https://github.com/discourse/discourse-translator/pull/274
- https://github.com/discourse/discourse-translator/pull/294

With this PR, we will start showing localized posts (if available) based
on the user's locale.

This work had been done in discourse-translator, but is now moving to
core.

app/assets/javascripts/discourse/app/components/post/meta-data.gjs

@@ -4,6 +4,7 @@ import { service } from "@ember/service";
 import PostMetaDataDate from "./meta-data/date";
 import PostMetaDataEditsIndicator from "./meta-data/edits-indicator";
 import PostMetaDataEmailIndicator from "./meta-data/email-indicator";
+import PostMetaDataLanguage from "./meta-data/language";
 import PostMetaDataLockedIndicator from "./meta-data/locked-indicator";
 import PostMetaDataPosterName from "./meta-data/poster-name";
 import PostMetaDataReadIndicator from "./meta-data/read-indicator";
@@ -34,6 +35,10 @@ export default class PostMetaData extends Component {
     );
   }
 
+  get shouldDisplayLanguage() {
+    return this.args.post.is_localized && this.args.post.language;
+  }
+
   <template>
     <div class="topic-meta-data" role="heading" aria-level="2">
       {{#if this.displayPosterName}}
@@ -88,6 +93,10 @@ export default class PostMetaData extends Component {
           />
         {{/if}}
 
+        {{#if this.shouldDisplayLanguage}}
+          <PostMetaDataLanguage @post={{@post}} />
+        {{/if}}
+
         <PostMetaDataDate @post={{@post}} />
 
         <PostMetaDataReadIndicator @post={{@post}} />

app/assets/javascripts/discourse/app/components/post/meta-data/language.gjs

@@ -0,0 +1,23 @@
+import Component from "@glimmer/component";
+import { i18n } from "discourse-i18n";
+import DTooltip from "float-kit/components/d-tooltip";
+
+export default class PostMetaDataLanguage extends Component {
+  get tooltip() {
+    // once we switch to glimmer, we can remove `this.args.data.language`
+    const language = this.args.data?.language || this.args.post?.language;
+    return i18n("post.original_language", {
+      language,
+    });
+  }
+
+  <template>
+    <div class="post-info post-language">
+      <DTooltip
+        @identifier="post-language"
+        @icon="language"
+        @content={{this.tooltip}}
+      />
+    </div>
+  </template>
+}

app/assets/javascripts/discourse/app/components/topic-localized-content-toggle.gjs

@@ -0,0 +1,53 @@
+import Component from "@glimmer/component";
+import { tracked } from "@glimmer/tracking";
+import { action } from "@ember/object";
+import { service } from "@ember/service";
+import DButton from "discourse/components/d-button";
+import concatClass from "discourse/helpers/concat-class";
+import cookie, { removeCookie } from "discourse/lib/cookie";
+
+const SHOW_ORIGINAL_COOKIE = "content-localization-show-original";
+const SHOW_ORIGINAL_COOKIE_EXPIRY = 30;
+
+export default class TopicLocalizedContentToggle extends Component {
+  @service router;
+
+  @tracked showingOriginal = false;
+
+  constructor() {
+    super(...arguments);
+    this.showingOriginal = cookie(SHOW_ORIGINAL_COOKIE);
+  }
+
+  @action
+  async showOriginal() {
+    if (this.showingOriginal) {
+      removeCookie(SHOW_ORIGINAL_COOKIE, { path: "/" });
+    } else {
+      cookie(SHOW_ORIGINAL_COOKIE, true, {
+        path: "/",
+        expires: SHOW_ORIGINAL_COOKIE_EXPIRY,
+      });
+    }
+
+    this.router.refresh();
+  }
+
+  get title() {
+    return this.showingOriginal
+      ? "translator.content_not_translated"
+      : "translator.content_translated";
+  }
+
+  <template>
+    <DButton
+      @icon="language"
+      @title={{this.title}}
+      class={{concatClass
+        "btn btn-default btn-toggle-localized-content no-text"
+        (unless this.showingOriginal "btn-active")
+      }}
+      @action={{this.showOriginal}}
+    />
+  </template>
+}

app/assets/javascripts/discourse/app/components/topic-timeline/container.gjs

@@ -10,6 +10,7 @@ import { and, not, or } from "truth-helpers";
 import DButton from "discourse/components/d-button";
 import PluginOutlet from "discourse/components/plugin-outlet";
 import TopicAdminMenu from "discourse/components/topic-admin-menu";
+import TopicLocalizedContentToggle from "discourse/components/topic-localized-content-toggle";
 import UserTip from "discourse/components/user-tip";
 import ageWithTooltip from "discourse/helpers/age-with-tooltip";
 import categoryLink from "discourse/helpers/category-link";
@@ -159,6 +160,13 @@ export default class TopicTimelineScrollArea extends Component {
     return true;
   }
 
+  get displayLocalizationToggle() {
+    return (
+      this.siteSettings.experimental_content_localization &&
+      this.args.model.has_localized_content
+    );
+  }
+
   get canCreatePost() {
     return this.args.model.details?.can_create_post;
   }
@@ -541,6 +549,11 @@ export default class TopicTimelineScrollArea extends Component {
           @name="timeline-controls-before"
           @outletArgs={{hash model=@model}}
         />
+
+        {{#if this.displayLocalizationToggle}}
+          <TopicLocalizedContentToggle @topic={{@model}} />
+        {{/if}}
+
         <TopicAdminMenu
           @topic={{@model}}
           @toggleMultiSelect={{@toggleMultiSelect}}

app/assets/javascripts/discourse/app/lib/transform-post.js

@@ -91,6 +91,9 @@ export function transformBasicPost(post) {
     canPublishPage: false,
     trustLevel: post.trust_level,
     userSuspended: post.user_suspended,
+    locale: post.locale,
+    is_localized: post.is_localized,
+    language: post.language,
   };
 
   _additionalAttributes.forEach((a) => (postAtts[a] = post[a]));

app/assets/javascripts/discourse/app/widgets/post.js

@@ -3,6 +3,7 @@ import { hbs } from "ember-cli-htmlbars";
 import { Promise } from "rsvp";
 import { h } from "virtual-dom";
 import ShareTopicModal from "discourse/components/modal/share-topic";
+import PostMetaDataLanguage from "discourse/components/post/meta-data/language";
 import { dateNode } from "discourse/helpers/node";
 import autoGroupFlairForUser from "discourse/lib/avatar-flair";
 import { avatarUrl, translateSize } from "discourse/lib/avatar-utils";
@@ -374,6 +375,10 @@ createWidget("post-meta-data", {
       postInfo.push(this.attach("reply-to-tab", attrs));
     }
 
+    if (attrs.language && attrs.is_localized) {
+      postInfo.push(this.attach("post-language", attrs));
+    }
+
     postInfo.push(this.attach("post-date", attrs));
 
     postInfo.push(
@@ -447,6 +452,19 @@ createWidget("post-date", {
   },
 });
 
+// glimmer-post-stream: has glimmer version
+createWidget("post-language", {
+  tagName: "div.post-info.post-language",
+
+  html(attrs) {
+    return [
+      new RenderGlimmer(this, "div", PostMetaDataLanguage, {
+        language: attrs.language,
+      }),
+    ];
+  },
+});
+
 // glimmer-post-stream: has glimmer version
 createWidget("expand-post-button", {
   tagName: "button.btn.expand-post",

app/assets/javascripts/discourse/tests/integration/components/post/post-test.gjs

@@ -275,6 +275,15 @@ module("Integration | Component | Post", function (hooks) {
     assert.strictEqual(count(".post-info.whisper"), 1);
   });
 
+  test("language", async function (assert) {
+    this.post.is_localized = true;
+    this.post.language = "English";
+
+    await renderComponent(this.post);
+
+    assert.dom(".post-language").exists();
+  });
+
   test("read indicator", async function (assert) {
     this.post.read = true;
 

app/assets/stylesheets/common/base/topic.scss

@@ -453,7 +453,8 @@ a[data-clicks]::after {
   @include click-counter-badge;
 }
 
-.post-info a {
+.post-info a,
+.post-info svg {
   color: var(--primary-medium);
 }
 

app/models/locale_site_setting.rb

@@ -53,6 +53,10 @@ class LocaleSiteSetting < EnumSiteSetting
     @lock.synchronize { @values = @language_names = @supported_locales = nil }
   end
 
+  def self.get_language_name(locale)
+    values.find { |v| v[:value] == locale.to_s.sub("-", "_") }&.[](:name)
+  end
+
   FALLBACKS = { en_GB: :en }
 
   def self.fallback_locale(locale)

app/models/post.rb

@@ -1323,7 +1323,7 @@ class Post < ActiveRecord::Base
     PrettyText.extract_mentions(Nokogiri::HTML5.fragment(cooked))
   end
 
-  def has_localization?(locale)
+  def has_localization?(locale = I18n.locale)
     post_localizations.exists?(locale: locale.to_s.sub("-", "_"))
   end
 

app/models/topic_list.rb

@@ -138,6 +138,10 @@ class TopicList
       { category: :parent_category },
     ]
 
+    if SiteSetting.experimental_content_localization
+      topic_preloader_associations << :topic_localizations
+    end
+
     DiscoursePluginRegistry.topic_preloader_associations.each do |a|
       fields = a[:fields]
       condition = a[:condition]

app/serializers/basic_post_serializer.rb

@@ -35,8 +35,11 @@ class BasicPostSerializer < ApplicationSerializer
       end
     else
       cooked = object.filter_quotes(@parent_post)
-      modified = DiscoursePluginRegistry.apply_modifier(:basic_post_serializer_cooked, cooked, self)
-      modified || cooked
+
+      translated_cooked =
+        object.get_localization&.cooked if ContentLocalization.show_translated_post?(object, scope)
+
+      translated_cooked || cooked
     end
   end
 

app/serializers/basic_topic_serializer.rb

@@ -6,7 +6,11 @@ class BasicTopicSerializer < ApplicationSerializer
 
   def fancy_title
     f = object.fancy_title
-    modified = DiscoursePluginRegistry.apply_modifier(:topic_serializer_fancy_title, f, self)
-    modified || f
+
+    if (ContentLocalization.show_translated_topic?(object, scope))
+      object.get_localization&.fancy_title.presence || f
+    else
+      f
+    end
   end
 end

app/serializers/post_serializer.rb

@@ -96,7 +96,10 @@ class PostSerializer < BasicPostSerializer
              :mentioned_users,
              :post_url,
              :has_post_localizations,
-             :post_localizations
+             :post_localizations,
+             :locale,
+             :is_localized,
+             :language
 
   def initialize(object, opts)
     super(object, opts)
@@ -663,6 +666,34 @@ class PostSerializer < BasicPostSerializer
     ).as_json
   end
 
+  def raw
+    if ContentLocalization.show_translated_post?(object, scope)
+      object.get_localization(I18n.locale)&.raw || object.raw
+    else
+      object.raw
+    end
+  end
+
+  def include_locale?
+    SiteSetting.experimental_content_localization
+  end
+
+  def is_localized
+    ContentLocalization.show_translated_post?(object, scope) && object.has_localization?
+  end
+
+  def include_is_localized?
+    SiteSetting.experimental_content_localization
+  end
+
+  def language
+    LocaleSiteSetting.get_language_name(object.locale) || locale
+  end
+
+  def include_language?
+    SiteSetting.experimental_content_localization && object.locale.present?
+  end
+
   private
 
   def can_review_topic?

app/serializers/topic_view_serializer.rb

@@ -78,6 +78,7 @@ class TopicViewSerializer < ApplicationSerializer
     :user_last_posted_at,
     :is_shared_draft,
     :slow_mode_enabled_until,
+    :has_localized_content,
   )
 
   has_one :details, serializer: TopicViewDetailsSerializer, root: false, embed: :objects
@@ -320,7 +321,22 @@ class TopicViewSerializer < ApplicationSerializer
 
   def fancy_title
     f = object.topic.fancy_title
-    modified = DiscoursePluginRegistry.apply_modifier(:topic_view_serializer_fancy_title, f, self)
-    modified || f
+
+    if ContentLocalization.show_translated_topic?(object.topic, scope)
+      object.topic.get_localization&.fancy_title.presence || f
+    else
+      f
+    end
+  end
+
+  def has_localized_content
+    topic_has_localization = !object.topic.in_user_locale? && object.topic.has_localization?
+    return true if topic_has_localization
+
+    object.posts.any? { |post| !post.in_user_locale? && post.has_localization? }
+  end
+
+  def include_has_localized_content?
+    SiteSetting.experimental_content_localization
   end
 end

config/locales/client.en.yml

@@ -4127,6 +4127,8 @@ en:
         title: "Share Post #%{post_number}"
         instructions: "Share a link to this post:"
 
+      original_language: "This post was originally written in %{language}"
+
     category:
       none: "(no category)"
       all: "All categories"

spec/models/locale_site_setting_spec.rb

@@ -116,4 +116,19 @@ RSpec.describe LocaleSiteSetting do
       end
     end
   end
+
+  describe ".get_language_name" do
+    it "returns the language name for a valid locale" do
+      expect(LocaleSiteSetting.get_language_name("en")).to eq("English (US)")
+      expect(LocaleSiteSetting.get_language_name("es")).to eq("Español")
+    end
+
+    it "returns nil for a locale that doesn't exist" do
+      expect(LocaleSiteSetting.get_language_name("xx")).to be_nil
+    end
+
+    it "handles symbol locales" do
+      expect(LocaleSiteSetting.get_language_name(:en_GB)).to eq("English (UK)")
+    end
+  end
 end

spec/serializers/basic_post_serializer_spec.rb

@@ -2,8 +2,8 @@
 
 RSpec.describe BasicPostSerializer do
   describe "#name" do
-    let(:user) { Fabricate.build(:user) }
-    let(:post) { Fabricate.build(:post, user: user, cooked: "Hur dur I am a cooked raw") }
+    fab!(:user)
+    fab!(:post) { Fabricate(:post, user: user, cooked: "Hur dur I am a cooked raw") }
     let(:serializer) { BasicPostSerializer.new(post, scope: Guardian.new, root: false) }
     let(:json) { serializer.as_json }
 
@@ -22,15 +22,13 @@ RSpec.describe BasicPostSerializer do
         expect(json[:cooked]).to eq(post.cooked)
       end
 
-      it "returns the modified cooked when register modified" do
-        plugin = Plugin::Instance.new
-        modifier = :basic_post_serializer_cooked
-        proc = Proc.new { "X" }
-        DiscoursePluginRegistry.register_modifier(plugin, modifier, &proc)
+      it "returns the localized cooked" do
+        SiteSetting.experimental_content_localization = true
+        Fabricate(:post_localization, post: post, cooked: "X", locale: "ja")
+        I18n.locale = "ja"
+        post.update!(locale: "en")
 
         expect(json[:cooked]).to eq("X")
-      ensure
-        DiscoursePluginRegistry.unregister_modifier(plugin, modifier, &proc)
       end
     end
   end

spec/serializers/basic_topic_serializer_spec.rb

@@ -11,15 +11,14 @@ describe BasicTopicSerializer do
     end
 
     it "returns the fancy title with a modifier" do
-      plugin = Plugin::Instance.new
-      modifier = :topic_serializer_fancy_title
-      proc = Proc.new { "X" }
-      DiscoursePluginRegistry.register_modifier(plugin, modifier, &proc)
+      SiteSetting.experimental_content_localization = true
+      Fabricate(:topic_localization, topic:, fancy_title: "X", locale: "ja")
+      I18n.locale = "ja"
+      topic.update!(locale: "en")
+
       json = BasicTopicSerializer.new(topic).as_json
 
       expect(json[:basic_topic][:fancy_title]).to eq("X")
-    ensure
-      DiscoursePluginRegistry.unregister_modifier(plugin, modifier, &proc)
     end
   end
 end

spec/serializers/post_serializer_spec.rb

@@ -703,7 +703,112 @@ RSpec.describe PostSerializer do
     end
   end
 
-  def serialized_post(u)
+  describe "#raw" do
+    fab!(:user)
+    let(:serializer) { serialized_post }
+    let(:json) { serializer.as_json }
+
+    it "returns the post's raw" do
+      expect(json[:raw]).to eq(post.raw)
+    end
+
+    it "returns the localized raw" do
+      SiteSetting.experimental_content_localization = true
+      Fabricate(:post_localization, post: post, raw: "raw", locale: "ja")
+      I18n.locale = "ja"
+      post.update!(locale: "en")
+
+      expect(json[:raw]).to eq("raw")
+    end
+  end
+
+  describe "#locale" do
+    let(:serializer) { serialized_post }
+    let(:json) { serializer.as_json }
+
+    it "is included when experimental_content_localization is enabled" do
+      SiteSetting.experimental_content_localization = true
+      post.update!(locale: "ja")
+
+      expect(json[:locale]).to eq("ja")
+    end
+
+    it "is excluded when experimental_content_localization is disabled" do
+      SiteSetting.experimental_content_localization = false
+      post.update!(locale: "ja")
+
+      expect(json[:locale]).to eq(nil)
+    end
+  end
+
+  describe "#is_localized?" do
+    let(:serializer) { serialized_post }
+    let(:json) { serializer.as_json }
+
+    it "is excluded when experimental_content_localization is disabled" do
+      SiteSetting.experimental_content_localization = false
+
+      expect(json[:is_localized]).to eq(nil)
+    end
+
+    describe "content localization enabled" do
+      before do
+        SiteSetting.experimental_content_localization = true
+        I18n.locale = "en"
+      end
+
+      it "returns true when the post is localized" do
+        post.update!(locale: "ja")
+        Fabricate(:post_localization, post:, locale: "en")
+
+        expect(json[:is_localized]).to eq(true)
+      end
+
+      it "returns false when the post is same language as user" do
+        post.update!(locale: "ja")
+        I18n.locale = "ja"
+
+        expect(json[:is_localized]).to eq(false)
+      end
+
+      it "returns false when no localization" do
+        post.update!(locale: "ja")
+
+        expect(json[:is_localized]).to eq(false)
+      end
+    end
+  end
+
+  describe "#language" do
+    let(:serializer) { serialized_post }
+    let(:json) { serializer.as_json }
+
+    it "is excluded when experimental_content_localization is disabled or no locale" do
+      SiteSetting.experimental_content_localization = false
+      post.update!(locale: "ja")
+      expect(serializer.as_json[:language]).to eq(nil)
+
+      SiteSetting.experimental_content_localization = true
+      post.update!(locale: nil)
+      expect(serializer.as_json[:language]).to eq(nil)
+    end
+
+    it "shows the language of the post based on locale" do
+      SiteSetting.experimental_content_localization = true
+      post.update!(locale: "ja")
+
+      expect(json[:language]).to eq("日本語")
+    end
+
+    it "defaults to locale if language does not exist" do
+      SiteSetting.experimental_content_localization = true
+      post.update!(locale: "aa")
+
+      expect(json[:language]).to eq("aa")
+    end
+  end
+
+  def serialized_post(u = nil)
     s = PostSerializer.new(post, scope: Guardian.new(u), root: false)
     s.add_raw = true
     s

spec/serializers/topic_view_serializer_spec.rb

@@ -654,16 +654,48 @@ RSpec.describe TopicViewSerializer do
       expect(json[:fancy_title]).to eq("Hur dur this is a title")
     end
 
-    it "returns the fancy title with a modifier" do
-      plugin = Plugin::Instance.new
-      modifier = :topic_view_serializer_fancy_title
-      proc = Proc.new { "X" }
-      DiscoursePluginRegistry.register_modifier(plugin, modifier, &proc)
-      json = serialize_topic(topic, user)
+    it "returns the localized fancy_title" do
+      SiteSetting.experimental_content_localization = true
+      Fabricate(:topic_localization, topic:, fancy_title: "X", locale: "ja")
+      I18n.locale = "ja"
+      topic.update!(locale: "en")
 
+      json = serialize_topic(topic, user)
       expect(json[:fancy_title]).to eq("X")
-    ensure
-      DiscoursePluginRegistry.unregister_modifier(plugin, modifier, &proc)
+    end
+  end
+
+  describe "#has_localized_content" do
+    before { SiteSetting.experimental_content_localization = true }
+
+    it "returns true if the topic has localization" do
+      Fabricate(:topic_localization, topic:, locale: "ja")
+      I18n.locale = "ja"
+      topic.update!(locale: "en")
+
+      json = serialize_topic(topic, user)
+      expect(json[:has_localized_content]).to eq(true)
+    end
+
+    it "returns true if any post has localization" do
+      loc = Fabricate(:post_localization, locale: "ja")
+      I18n.locale = "ja"
+      loc.post.update!(locale: "en")
+
+      json = serialize_topic(loc.post.topic, user)
+      expect(json[:has_localized_content]).to eq(true)
+    end
+
+    it "returns false if the topic does not have localization" do
+      json = serialize_topic(topic, user)
+      expect(json[:has_localized_content]).to eq(false)
+    end
+
+    it "does not return attribute if setting is disabled" do
+      SiteSetting.experimental_content_localization = false
+
+      json = serialize_topic(topic, user)
+      expect(json[:has_localized_content]).to eq(nil)
     end
   end
 end

spec/system/topic_view/localized_content_spec.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+RSpec.describe "Localized topic" do
+  fab!(:japanese_user) { Fabricate(:user, locale: "ja") }
+  fab!(:site_local_user) { Fabricate(:user, locale: "en") }
+  fab!(:author) { Fabricate(:user) }
+
+  fab!(:topic) do
+    Fabricate(:topic, title: "Life strategies from The Art of War", locale: "en", user: author)
+  end
+  fab!(:post_1) do
+    Fabricate(
+      :post,
+      topic:,
+      locale: "en",
+      raw: "The masterpiece isn’t just about military strategy",
+    )
+  end
+  fab!(:post_2) do
+    Fabricate(
+      :post,
+      topic:,
+      locale: "en",
+      raw: "The greatest victory is that which requires no battle",
+    )
+  end
+  fab!(:post_3) { Fabricate(:post, topic:, locale: "ja", raw: "将とは、智・信・仁・勇・厳なり。") }
+
+  let(:topic_page) { PageObjects::Pages::Topic.new }
+  let(:topic_list) { PageObjects::Components::TopicList.new }
+
+  before do
+    Fabricate(:topic_localization, topic:, locale: "ja", fancy_title: "孫子兵法からの人生戦略")
+    Fabricate(:topic_localization, topic:, locale: "es", fancy_title: "Estrategias de vida de ...")
+
+    Fabricate(:post_localization, post: post_1, locale: "ja", cooked: "傑作は単なる軍事戦略についてではありません")
+    Fabricate(:post_localization, post: post_2, locale: "ja", cooked: "最大の勝利は戦いを必要としないものです")
+    Fabricate(:post_localization, post: post_3, locale: "en", cooked: "A general is one who ..")
+  end
+
+  context "when the feature is enabled" do
+    before do
+      SiteSetting.allow_user_locale = true
+      SiteSetting.experimental_content_localization = true
+    end
+
+    it "shows the correct language based on the selected language and login status" do
+      sign_in(japanese_user)
+      visit("/")
+      visit("/t/#{topic.id}")
+      expect(topic_page.has_topic_title?("孫子兵法からの人生戦略")).to eq(true)
+    end
+
+    it "shows original content when 'Show Original' is selected" do
+      sign_in(japanese_user)
+
+      visit("/")
+      topic_list.visit_topic_with_title("孫子兵法からの人生戦略")
+
+      expect(topic_page.has_topic_title?("孫子兵法からの人生戦略")).to eq(true)
+      page.find(".timeline-controls button.btn-toggle-localized-content").click
+
+      expect(topic_page.has_topic_title?("Life strategies from The Art of War")).to eq(true)
+
+      visit("/")
+      topic_list.visit_topic_with_title("Life strategies from The Art of War")
+    end
+  end
+end