Adds import script for Zendesk

It also adds a generic SQLite database that can be used when the data needs some transformation before the actual import.

script/import_scripts/base/generic_database.rb

@@ -0,0 +1,220 @@
+require 'sqlite3'
+
+module ImportScripts
+  class GenericDatabase
+    def initialize(directory, batch_size:, recreate: false)
+      filename = "#{directory}/index.db"
+      File.delete(filename) if recreate && File.exists?(filename)
+
+      @db = SQLite3::Database.new(filename, results_as_hash: true)
+      @batch_size = batch_size
+
+      configure_database
+      create_category_table
+      create_user_table
+      create_topic_table
+      create_post_table
+    end
+
+    def insert_category(category)
+      @db.execute(<<-SQL, prepare(category))
+        INSERT OR REPLACE INTO category (id, name, description, position, url)
+        VALUES (:id, :name, :description, :position, :url)
+      SQL
+    end
+
+    def insert_user(user)
+      @db.execute(<<-SQL, prepare(user))
+        INSERT OR REPLACE INTO user (id, email, username, name, created_at, last_seen_at, active)
+        VALUES (:id, :email, :username, :name, :created_at, :last_seen_at, :active)
+      SQL
+    end
+
+    def insert_topic(topic)
+      @db.execute(<<-SQL, prepare(topic))
+        INSERT OR REPLACE INTO topic (id, title, raw, category_id, closed, user_id, created_at, url)
+        VALUES (:id, :title, :raw, :category_id, :closed, :user_id, :created_at, :url)
+      SQL
+    end
+
+    def insert_post(post)
+      @db.execute(<<-SQL, prepare(post))
+        INSERT OR REPLACE INTO post (id, raw, topic_id, user_id, created_at, reply_to_post_id, url)
+        VALUES (:id, :raw, :topic_id, :user_id, :created_at, :reply_to_post_id, :url)
+      SQL
+    end
+
+    def sort_posts_by_created_at
+      @db.execute 'DELETE FROM post_order'
+
+      @db.execute <<-SQL
+        INSERT INTO post_order (id)
+        SELECT id
+        FROM post
+        ORDER BY created_at, topic_id, id
+      SQL
+    end
+
+    def fetch_categories
+      @db.execute(<<-SQL)
+        SELECT *
+        FROM category
+        ORDER BY position, name
+      SQL
+    end
+
+    def count_users
+      @db.get_first_value(<<-SQL)
+        SELECT COUNT(*)
+        FROM user
+      SQL
+    end
+
+    def fetch_users(last_id)
+      rows = @db.execute(<<-SQL, last_id)
+        SELECT *
+        FROM user
+        WHERE id > :last_id
+        ORDER BY id
+        LIMIT #{@batch_size}
+      SQL
+
+      add_last_column_value(rows, 'id')
+    end
+
+    def count_topics
+      @db.get_first_value(<<-SQL)
+        SELECT COUNT(*)
+        FROM topic
+      SQL
+    end
+
+    def fetch_topics(last_id)
+      rows = @db.execute(<<-SQL, last_id)
+        SELECT *
+        FROM topic
+        WHERE id > :last_id
+        ORDER BY id
+        LIMIT #{@batch_size}
+      SQL
+
+      add_last_column_value(rows, 'id')
+    end
+
+    def count_posts
+      @db.get_first_value(<<-SQL)
+        SELECT COUNT(*)
+        FROM post
+      SQL
+    end
+
+    def fetch_posts(last_row_id)
+      rows = @db.execute(<<-SQL, last_row_id)
+        SELECT o.ROWID, p.*
+        FROM post p
+          JOIN post_order o USING (id)
+        WHERE o.ROWID > :last_row_id
+        ORDER BY o.ROWID
+        LIMIT #{@batch_size}
+      SQL
+
+      add_last_column_value(rows, 'rowid')
+    end
+
+    def execute_sql(sql)
+      @db.execute(sql)
+    end
+
+    private
+
+    def configure_database
+      @db.execute 'PRAGMA journal_mode = OFF'
+      @db.execute 'PRAGMA locking_mode = EXCLUSIVE'
+    end
+
+    def create_category_table
+      @db.execute <<-SQL
+        CREATE TABLE IF NOT EXISTS category (
+          id TEXT NOT NULL PRIMARY KEY,
+          name TEXT NOT NULL,
+          description TEXT,
+          position INTEGER,
+          url TEXT
+        )
+      SQL
+    end
+
+    def create_user_table
+      @db.execute <<-SQL
+        CREATE TABLE IF NOT EXISTS user (
+          id TEXT NOT NULL PRIMARY KEY,
+          email TEXT,
+          username TEXT,
+          name TEXT,
+          created_at DATETIME,
+          last_seen_at DATETIME,
+          active BOOLEAN NOT NULL DEFAULT true
+        )
+      SQL
+    end
+
+    def create_topic_table
+      @db.execute <<-SQL
+        CREATE TABLE IF NOT EXISTS topic (
+          id TEXT NOT NULL PRIMARY KEY,
+          title TEXT,
+          raw TEXT,
+          category_id TEXT NOT NULL,
+          closed BOOLEAN NOT NULL DEFAULT false,
+          user_id TEXT NOT NULL,
+          created_at DATETIME,
+          url TEXT
+        )
+      SQL
+
+      @db.execute 'CREATE INDEX IF NOT EXISTS topic_by_user_id ON topic (user_id)'
+    end
+
+    def create_post_table
+      @db.execute <<-SQL
+        CREATE TABLE IF NOT EXISTS post (
+          id TEXT NOT NULL PRIMARY KEY,
+          raw TEXT,
+          topic_id TEXT NOT NULL,
+          user_id TEXT NOT NULL,
+          created_at DATETIME,
+          reply_to_post_id TEXT,
+          url TEXT
+        )
+      SQL
+
+      @db.execute 'CREATE INDEX IF NOT EXISTS post_by_user_id ON post (user_id)'
+
+      @db.execute <<-SQL
+        CREATE TABLE IF NOT EXISTS post_order (
+          id TEXT NOT NULL PRIMARY KEY
+        )
+      SQL
+    end
+
+    def prepare(hash)
+      hash.each do |key, value|
+        if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+          hash[key] = value ? 1 : 0
+        elsif value.is_a?(Date)
+          hash[key] = value.to_s
+        end
+      end
+    end
+
+    def add_last_column_value(rows, *last_columns)
+      return rows if last_columns.empty?
+
+      result = [rows]
+      last_row = rows.last
+
+      last_columns.each { |column| result.push(last_row ? last_row[column] : nil) }
+      result
+    end
+  end
+end