DEV: Adds a new converter for migrating from Discourse

It only contains a few user-related steps right now

migrations/.gitignore

@@ -1,7 +1,8 @@
 !/db/**/*.sql
 !/spec/support/fixtures/**/*.sql
 
-tmp/*
-private/
-Gemfile.lock
-/config/upload.yml
+/config/upload.yml
+/lib/converters/**/settings.local.yml
+
+# converters within this directory should be ignored
+/private/

migrations/lib/converters.rb

@@ -39,6 +39,9 @@ module Migrations
     end
 
     def self.default_settings_path(converter_name)
+      local_settings_path = File.join(path_of(converter_name), "settings.local.yml")
+      return local_settings_path if File.exist?(local_settings_path)
+
       File.join(path_of(converter_name), "settings.yml")
     end
   end

migrations/lib/converters/discourse/converter.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Migrations::Converters::Discourse
+  class Converter < ::Migrations::Converters::Base::Converter
+    def initialize(settings)
+      super
+      @source_db = ::Migrations::Database::Adapter::Postgres.new(settings[:source_db])
+    end
+
+    def step_args(step_class)
+      { source_db: @source_db }
+    end
+  end
+end

migrations/lib/converters/discourse/settings.yml

@@ -0,0 +1,9 @@
+intermediate_db:
+  path: "/shared/import/intermediate.db"
+
+source_db:
+  host: "127.0.0.1"
+  port: 5432
+  user: "username"
+  password: "password"
+  dbname: "discourse_source_db"

migrations/lib/converters/discourse/steps/user_emails.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Migrations::Converters::Discourse
+  class UserEmails < ::Migrations::Converters::Base::ProgressStep
+    attr_accessor :source_db
+
+    def max_progress
+      @source_db.count <<~SQL
+        SELECT COUNT(*)
+        FROM user_emails
+        WHERE user_id >= 0
+      SQL
+    end
+
+    def items
+      @source_db.query <<~SQL
+        SELECT user_id, email, "primary", created_at
+        FROM user_emails
+        WHERE user_id >= 0
+        ORDER BY user_id, email
+      SQL
+    end
+
+    def process_item(item)
+      IntermediateDB::UserEmail.create(
+        email: item[:email],
+        primary: item[:primary],
+        user_id: item[:user_id],
+        created_at: item[:created_at],
+      )
+    end
+  end
+end

migrations/lib/converters/discourse/steps/user_options.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Migrations::Converters::Discourse
+  class UserOptions < ::Migrations::Converters::Base::ProgressStep
+    attr_accessor :source_db
+
+    def max_progress
+      @source_db.count <<~SQL
+        SELECT COUNT(*)
+        FROM user_options
+        WHERE user_id >= 0
+      SQL
+    end
+
+    def items
+      @source_db.query <<~SQL
+        SELECT *
+        FROM user_options
+        WHERE user_id >= 0
+        ORDER BY user_id
+      SQL
+    end
+
+    def process_item(item)
+      IntermediateDB::UserOption.create(
+        user_id: item[:user_id],
+        allow_private_messages: item[:allow_private_messages],
+        auto_track_topics_after_msecs: item[:auto_track_topics_after_msecs],
+        automatically_unpin_topics: item[:automatically_unpin_topics],
+        bookmark_auto_delete_preference: item[:bookmark_auto_delete_preference],
+        chat_email_frequency: item[:chat_email_frequency],
+        chat_enabled: item[:chat_enabled],
+        chat_header_indicator_preference: item[:chat_header_indicator_preference],
+        chat_send_shortcut: item[:chat_send_shortcut],
+        chat_separate_sidebar_mode: item[:chat_separate_sidebar_mode],
+        chat_sound: item[:chat_sound],
+        color_scheme_id: item[:color_scheme_id],
+        dark_scheme_id: item[:dark_scheme_id],
+        default_calendar: item[:default_calendar],
+        digest_after_minutes: item[:digest_after_minutes],
+        dismissed_channel_retention_reminder: item[:dismissed_channel_retention_reminder],
+        dismissed_dm_retention_reminder: item[:dismissed_dm_retention_reminder],
+        dynamic_favicon: item[:dynamic_favicon],
+        email_digests: item[:email_digests],
+        email_in_reply_to: item[:email_in_reply_to],
+        email_level: item[:email_level],
+        email_messages_level: item[:email_messages_level],
+        email_previous_replies: item[:email_previous_replies],
+        enable_allowed_pm_users: item[:enable_allowed_pm_users],
+        enable_defer: item[:enable_defer],
+        enable_experimental_sidebar: item[:enable_experimental_sidebar],
+        enable_quoting: item[:enable_quoting],
+        enable_smart_lists: item[:enable_smart_lists],
+        external_links_in_new_tab: item[:external_links_in_new_tab],
+        hide_presence: item[:hide_presence],
+        hide_profile: item[:hide_profile],
+        hide_profile_and_presence: item[:hide_profile_and_presence],
+        homepage_id: item[:homepage_id],
+        ignore_channel_wide_mention: item[:ignore_channel_wide_mention],
+        include_tl0_in_digests: item[:include_tl0_in_digests],
+        last_redirected_to_top_at: item[:last_redirected_to_top_at],
+        like_notification_frequency: item[:like_notification_frequency],
+        mailing_list_mode: item[:mailing_list_mode],
+        mailing_list_mode_frequency: item[:mailing_list_mode_frequency],
+        new_topic_duration_minutes: item[:new_topic_duration_minutes],
+        notification_level_when_replying: item[:notification_level_when_replying],
+        oldest_search_log_date: item[:oldest_search_log_date],
+        only_chat_push_notifications: item[:only_chat_push_notifications],
+        seen_popups: item[:seen_popups],
+        show_thread_title_prompts: item[:show_thread_title_prompts],
+        sidebar_link_to_filtered_list: item[:sidebar_link_to_filtered_list],
+        sidebar_show_count_of_new_items: item[:sidebar_show_count_of_new_items],
+        skip_new_user_tips: item[:skip_new_user_tips],
+        text_size_key: item[:text_size_key],
+        text_size_seq: item[:text_size_seq],
+        theme_ids: item[:theme_ids],
+        theme_key_seq: item[:theme_key_seq],
+        timezone: item[:timezone],
+        title_count_mode_key: item[:title_count_mode_key],
+        topics_unread_when_closed: item[:topics_unread_when_closed],
+        watched_precedence_over_muted: item[:watched_precedence_over_muted],
+      )
+    end
+  end
+end

migrations/lib/converters/discourse/steps/user_suspensions.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Migrations::Converters::Discourse
+  class UserSuspensions < ::Migrations::Converters::Base::ProgressStep
+    attr_accessor :source_db
+
+    def max_progress
+      @source_db.count <<~SQL
+        SELECT COUNT(*)
+        FROM user_histories uh
+             JOIN users u ON uh.target_user_id = u.id
+        WHERE uh.action = 10 -- Suspend (10)
+      SQL
+    end
+
+    def items
+      @source_db.query <<~SQL
+        WITH actions AS (SELECT target_user_id,
+                                acting_user_id,
+                                action,
+                                created_at,
+                                details,
+                                LEAD(created_at) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action_date,
+                                LEAD(action) OVER (PARTITION BY target_user_id ORDER BY id)     AS next_action
+                         FROM user_histories
+                         WHERE action IN (10, 11) -- Suspend (10) / Unsuspend (11)
+        )
+        SELECT a.target_user_id AS user_id,
+               CASE
+                   WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_at - a.created_at))) <= 2
+                       THEN u.suspended_at
+                   ELSE a.created_at
+                   END          AS suspended_at,
+               CASE
+                   WHEN a.next_action = 11 THEN
+                       CASE
+                           WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_till - a.next_action_date))) <= 2
+                               THEN u.suspended_till
+                           ELSE a.next_action_date
+                           END
+                   ELSE u.suspended_till
+                   END          AS suspended_till,
+               a.acting_user_id AS suspended_by,
+               a.details        AS reason
+        FROM actions a
+             JOIN users u ON a.target_user_id = u.id
+        WHERE a.action = 10 -- Only take suspend actions
+        ORDER BY user_id, suspended_at
+      SQL
+    end
+
+    def process_item(item)
+      IntermediateDB::UserSuspension.create(
+        user_id: item[:user_id],
+        suspended_at: item[:suspended_at],
+        suspended_till: item[:suspended_till],
+        suspended_by_id: item[:suspended_by_id],
+        reason: item[:reason],
+      )
+    end
+  end
+end

migrations/lib/converters/discourse/steps/users.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Migrations::Converters::Discourse
+  class Users < ::Migrations::Converters::Base::ProgressStep
+    attr_accessor :source_db
+
+    def max_progress
+      @source_db.count <<~SQL
+        SELECT COUNT(*)
+        FROM users
+        WHERE id >= 0
+      SQL
+    end
+
+    def items
+      @source_db.query <<~SQL
+        SELECT *
+        FROM users
+        WHERE id >= 0
+        ORDER BY id
+      SQL
+    end
+
+    def process_item(item)
+      IntermediateDB::User.create(
+        original_id: item[:id],
+        active: item[:active],
+        admin: item[:admin],
+        approved: item[:approved],
+        approved_at: item[:approved_at],
+        approved_by_id: item[:approved_by_id],
+        created_at: item[:created_at],
+        date_of_birth: item[:date_of_birth],
+        first_seen_at: item[:first_seen_at],
+        flair_group_id: item[:flair_group_id],
+        group_locked_trust_level: item[:group_locked_trust_level],
+        ip_address: item[:ip_address],
+        last_seen_at: item[:last_seen_at],
+        locale: item[:locale],
+        manual_locked_trust_level: item[:manual_locked_trust_level],
+        moderator: item[:moderator],
+        name: item[:name],
+        primary_group_id: item[:primary_group_id],
+        registration_ip_address: item[:registration_ip_address],
+        silenced_till: item[:silenced_till],
+        staged: item[:staged],
+        title: item[:title],
+        trust_level: item[:trust_level],
+        uploaded_avatar_id: item[:uploaded_avatar_id],
+        username: item[:username],
+        views: item[:views],
+      )
+    end
+  end
+end

migrations/lib/database/adapter/postgres.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "pg"
+
+module Migrations::Database::Adapter
+  class Postgres
+    def initialize(settings)
+      @connection = PG::Connection.new(settings)
+      @connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)
+      @connection.field_name_type = :symbol
+      configure_connection
+    end
+
+    def exec(sql)
+      @connection.exec(sql)
+    end
+
+    def query(sql, *params)
+      @connection.send_query_params(sql, params)
+      @connection.set_single_row_mode
+
+      Enumerator.new do |y|
+        while (result = @connection.get_result)
+          result.stream_each { |row| y.yield(row) }
+          result.clear
+        end
+      end
+    end
+
+    def query_first(sql)
+      @connection.exec(sql).first
+    end
+
+    def query_value(sql, column)
+      query_first(sql)[column]
+    end
+
+    def count(sql)
+      query_first(sql).values.first.to_i
+    end
+
+    def close
+      unless @connection&.finished?
+        @connection.finish
+        @connection = nil
+      end
+    end
+
+    def reset
+      @connection.reset
+      configure_connection
+    end
+
+    def escape_string(str)
+      @connection.escape_string(str)
+    end
+
+    private
+
+    def configure_connection
+      @connection.exec("SET client_min_messages TO WARNING")
+    end
+  end
+end