From ad05924bdf6437c9b82b065c79a5fa3625f473d5 Mon Sep 17 00:00:00 2001
From: Andrei Prigorshnev <a.prigorshnev@gmail.com>
Date: Thu, 27 Jul 2023 21:56:32 +0400
Subject: [PATCH] DEV: Do one query per month when exporting chat messages
 (#22746)

We did some testing and saw that making one query per month is
cheaper than querying all chat messages at ones. Note that even
though the export job will be performing one query per month,
the exported messages will be streamed into a single CSV file, so
nothing changes from the user's point of view.
---
 plugins/chat/lib/chat/messages_exporter.rb    | 60 +++++++++++--------
 .../spec/system/admin/csv_exports_spec.rb     | 18 ++++--
 2 files changed, 50 insertions(+), 28 deletions(-)

diff --git a/plugins/chat/lib/chat/messages_exporter.rb b/plugins/chat/lib/chat/messages_exporter.rb
index f81784233b4..b627049f9ef 100644
--- a/plugins/chat/lib/chat/messages_exporter.rb
+++ b/plugins/chat/lib/chat/messages_exporter.rb
@@ -4,10 +4,44 @@ module Chat
   module MessagesExporter
     LIMIT = 10_000
 
-    def chat_message_export
+    def chat_message_export(&block)
+      # perform 1 db query per month:
+      now = Time.now
+      from = 6.months.ago
+      while from <= now
+        export(from, from + 1.month, &block)
+        from = from + 1.month
+      end
+    end
+
+    def get_header(entity)
+      if entity === "chat_message"
+        %w[
+          id
+          chat_channel_id
+          chat_channel_name
+          user_id
+          username
+          message
+          cooked
+          created_at
+          updated_at
+          deleted_at
+          in_reply_to_id
+          last_editor_id
+          last_editor_username
+        ]
+      else
+        super
+      end
+    end
+
+    private
+
+    def export(from, to)
       Chat::Message
         .unscoped
-        .where(created_at: 6.months.ago..Time.current)
+        .where(created_at: from..to)
         .includes(:chat_channel)
         .includes(:user)
         .includes(:last_editor)
@@ -32,27 +66,5 @@ module Chat
           )
         end
     end
-
-    def get_header(entity)
-      if entity === "chat_message"
-        %w[
-          id
-          chat_channel_id
-          chat_channel_name
-          user_id
-          username
-          message
-          cooked
-          created_at
-          updated_at
-          deleted_at
-          in_reply_to_id
-          last_editor_id
-          last_editor_username
-        ]
-      else
-        super
-      end
-    end
   end
 end
diff --git a/plugins/chat/spec/system/admin/csv_exports_spec.rb b/plugins/chat/spec/system/admin/csv_exports_spec.rb
index 012b948d59a..339f9b61da0 100644
--- a/plugins/chat/spec/system/admin/csv_exports_spec.rb
+++ b/plugins/chat/spec/system/admin/csv_exports_spec.rb
@@ -11,7 +11,10 @@ RSpec.describe "Chat CSV exports", type: :system do
   end
 
   xit "exports chat messages" do
-    message = Fabricate(:chat_message)
+    message_1 = Fabricate(:chat_message, created_at: 12.months.ago)
+    message_2 = Fabricate(:chat_message, created_at: 6.months.ago)
+    message_3 = Fabricate(:chat_message, created_at: 1.months.ago)
+    message_4 = Fabricate(:chat_message, created_at: Time.now)
 
     visit "/admin/plugins/chat"
     click_button "Create export"
@@ -39,8 +42,17 @@ RSpec.describe "Chat CSV exports", type: :system do
       ],
     )
 
+    assert_message(exported_data.second, message_1)
+    assert_message(exported_data.third, message_2)
+    assert_message(exported_data.fourth, message_3)
+    assert_message(exported_data.fifth, message_4)
+  ensure
+    csv_export_pm_page.clear_downloads
+  end
+
+  def assert_message(exported_message, message)
     time_format = "%Y-%m-%d %H:%M:%S UTC"
-    expect(exported_data.second).to eq(
+    expect(exported_message).to eq(
       [
         message.id.to_s,
         message.chat_channel.id.to_s,
@@ -57,7 +69,5 @@ RSpec.describe "Chat CSV exports", type: :system do
         message.last_editor.username,
       ],
     )
-  ensure
-    csv_export_pm_page.clear_downloads
   end
 end