From b9dd9c70a5fd835e6d049555f249d1eced3cee86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Guitaut?= <loic@discourse.org>
Date: Wed, 19 Feb 2025 17:59:55 +0100
Subject: [PATCH] DEV: Migrate Sidekiq to a dedicated Redis DB
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

As we’re currently using a namespace for Sidekiq, in order to upgrade to
the latest version, we need to drop it as it’s not supported anymore.

The recommended way is to use a different Redis DB for Sidekiq.

This patch uses a different config for Sidekiq and also takes care of
migrating existing jobs (in queues and the retry and scheduled sets).
---
 config/initializers/002-freedom_patches.rb    |  5 ++-
 config/initializers/100-sidekiq.rb            |  2 +-
 .../20250227142351_migrate_sidekiq_jobs.rb    | 13 ++++++
 lib/discourse.rb                              |  8 ++--
 lib/freedom_patches/sidekiq.rb                | 14 +++++++
 lib/sidekiq_migration.rb                      | 40 +++++++++++++++++++
 6 files changed, 76 insertions(+), 6 deletions(-)
 create mode 100644 db/post_migrate/20250227142351_migrate_sidekiq_jobs.rb
 create mode 100644 lib/freedom_patches/sidekiq.rb
 create mode 100644 lib/sidekiq_migration.rb

diff --git a/config/initializers/002-freedom_patches.rb b/config/initializers/002-freedom_patches.rb
index b957e037114..5f58652531e 100644
--- a/config/initializers/002-freedom_patches.rb
+++ b/config/initializers/002-freedom_patches.rb
@@ -2,7 +2,10 @@
 
 # Multisite freedom patch defines RailsMultisite::DiscoursePatches.config which is used  by 200-first_middlewares.rb
 # Therefore it can not be postponed with .to_prepare
-RUN_WITHOUT_PREPARE = ["#{Rails.root}/lib/freedom_patches/rails_multisite.rb"]
+RUN_WITHOUT_PREPARE = [
+  "#{Rails.root}/lib/freedom_patches/rails_multisite.rb",
+  "#{Rails.root}/lib/freedom_patches/sidekiq.rb",
+]
 RUN_WITHOUT_PREPARE.each { |path| require(path) }
 
 Rails.application.reloader.to_prepare do
diff --git a/config/initializers/100-sidekiq.rb b/config/initializers/100-sidekiq.rb
index ef94c6b15c2..fdebafb41e3 100644
--- a/config/initializers/100-sidekiq.rb
+++ b/config/initializers/100-sidekiq.rb
@@ -84,7 +84,7 @@ Rails.application.config.to_prepare do
   Dir.glob("#{Rails.root}/app/jobs/scheduled/*.rb") { |f| require(f) } if Rails.env.development?
 
   MiniScheduler.configure do |config|
-    config.redis = Discourse.redis
+    config.redis = DiscourseRedis.new(Discourse.sidekiq_redis_config)
 
     config.job_exception_handler { |ex, context| Discourse.handle_job_exception(ex, context) }
 
diff --git a/db/post_migrate/20250227142351_migrate_sidekiq_jobs.rb b/db/post_migrate/20250227142351_migrate_sidekiq_jobs.rb
new file mode 100644
index 00000000000..64806ee419d
--- /dev/null
+++ b/db/post_migrate/20250227142351_migrate_sidekiq_jobs.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Delete this migration instead of promoting it (along with the
+# `SidekiqMigration` class)
+class MigrateSidekiqJobs < ActiveRecord::Migration[7.2]
+  def up
+    SidekiqMigration.call
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end
diff --git a/lib/discourse.rb b/lib/discourse.rb
index 870523d4f2d..440bbbbf87a 100644
--- a/lib/discourse.rb
+++ b/lib/discourse.rb
@@ -1047,10 +1047,10 @@ module Discourse
 
   SIDEKIQ_NAMESPACE = "sidekiq"
 
-  def self.sidekiq_redis_config
-    conf = GlobalSetting.redis_config.dup
-    conf[:namespace] = SIDEKIQ_NAMESPACE
-    conf
+  def self.sidekiq_redis_config(old: false)
+    redis_config = GlobalSetting.redis_config.dup
+    return redis_config.merge(namespace: SIDEKIQ_NAMESPACE) if old
+    redis_config.merge(db: redis_config[:db].to_i + 1)
   end
 
   def self.static_doc_topic_ids
diff --git a/lib/freedom_patches/sidekiq.rb b/lib/freedom_patches/sidekiq.rb
new file mode 100644
index 00000000000..656759dcf8d
--- /dev/null
+++ b/lib/freedom_patches/sidekiq.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# TODO (2025-03-03): delete this once the migration is propagated everywhere
+# (in about 6 months or so)
+module Sidekiq
+  def self.redis_pool
+    @redis ||= RedisConnection.create
+    Thread.current[:sidekiq_via_pool] || @redis
+  end
+
+  def self.old_pool
+    @old_pool ||= RedisConnection.create(Discourse.sidekiq_redis_config(old: true))
+  end
+end
diff --git a/lib/sidekiq_migration.rb b/lib/sidekiq_migration.rb
new file mode 100644
index 00000000000..e82195f8e13
--- /dev/null
+++ b/lib/sidekiq_migration.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# TODO (2025-03-03): delete this once the migration is propagated everywhere
+# (in about 6 months or so)
+class SidekiqMigration
+  delegate :old_pool, to: :Sidekiq
+
+  def self.call
+    new.call
+  end
+
+  def call
+    migrate_all_queues
+    migrate(klass: Sidekiq::RetrySet)
+    migrate(klass: Sidekiq::ScheduledSet)
+  end
+
+  private
+
+  def migrate_all_queues
+    migrate(
+      old_jobs: -> { Sidekiq::Queue.all.flat_map(&:to_a) },
+      enqueue_jobs: ->(job) { client.push(job.item) },
+    )
+  end
+
+  def migrate(
+    klass: nil,
+    old_jobs: -> { klass.new.to_a },
+    enqueue_jobs: ->(job) { klass.new.schedule(job.score, job.item) }
+  )
+    jobs_to_migrate = Sidekiq::Client.via(old_pool, &old_jobs)
+    jobs_to_migrate.each(&enqueue_jobs)
+    Sidekiq::Client.via(old_pool) { jobs_to_migrate.each(&:delete) }
+  end
+
+  def client
+    @client ||= Sidekiq::Client.new
+  end
+end