github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-21 18:12:32 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

FEATURE: Import script for AnswerBase

Browse Source 
Improves the generic database used by some import scripts:
* Adds additional columns for users
* Adds support for attachments
* Allows setting the data type for keys (numeric or string) to ensure correct sorting
This commit is contained in:
Gerhard Schlager
2019-02-28 21:59:36 +01:00
parent ff26b4ed9b
commit c36c9c2ee5
5 changed files with 471 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
script/import_scripts/base/generic_database.rb
View File

@ -2,12 +2,13 @@ require 'sqlite3'
module ImportScripts
class GenericDatabase
def initialize(directory, batch_size:, recreate: false)
def initialize(directory, batch_size:, recreate: false, numeric_keys: false)
filename = "#{directory}/index.db"
File.delete(filename) if recreate && File.exists?(filename)
@db = SQLite3::Database.new(filename, results_as_hash: true)
@batch_size = batch_size
@numeric_keys = numeric_keys
configure_database
create_category_table
@ -25,36 +26,72 @@ module ImportScripts
def insert_user(user)
@db.execute(<<-SQL, prepare(user))
INSERT OR REPLACE INTO user (id, email, username, name, created_at, last_seen_at, active)
VALUES (:id, :email, :username, :name, :created_at, :last_seen_at, :active)
INSERT OR REPLACE
INTO user (id, email, username, name, bio, avatar_path, created_at, last_seen_at, active)
VALUES (:id, :email, :username, :name, :bio, :avatar_path, :created_at, :last_seen_at, :active)
SQL
end
def insert_topic(topic)
attachments = topic.delete(:attachments)
topic[:upload_count] = attachments&.size || 0
@db.execute(<<-SQL, prepare(topic))
INSERT OR REPLACE INTO topic (id, title, raw, category_id, closed, user_id, created_at, url)
VALUES (:id, :title, :raw, :category_id, :closed, :user_id, :created_at, :url)
INSERT OR REPLACE INTO topic (id, title, raw, category_id, closed, user_id, created_at, url, upload_count)
VALUES (:id, :title, :raw, :category_id, :closed, :user_id, :created_at, :url, :upload_count)
SQL
attachments&.each do |attachment|
@db.execute(<<-SQL, topic_id: topic[:id], path: attachment)
INSERT OR REPLACE INTO topic_upload (topic_id, path)
VALUES (:topic_id, :path)
SQL
end
end
def insert_post(post)
attachments = post.delete(:attachments)
post[:upload_count] = attachments&.size || 0
@db.execute(<<-SQL, prepare(post))
INSERT OR REPLACE INTO post (id, raw, topic_id, user_id, created_at, reply_to_post_id, url)
VALUES (:id, :raw, :topic_id, :user_id, :created_at, :reply_to_post_id, :url)
INSERT OR REPLACE INTO post (id, raw, topic_id, user_id, created_at, reply_to_post_id, url, upload_count)
VALUES (:id, :raw, :topic_id, :user_id, :created_at, :reply_to_post_id, :url, :upload_count)
SQL
attachments&.each do |attachment|
@db.execute(<<-SQL, post_id: post[:id], path: attachment)
INSERT OR REPLACE INTO post_upload (post_id, path)
VALUES (:post_id, :path)
SQL
end
end
def sort_posts_by_created_at
@db.execute 'DELETE FROM post_order'
@db.execute <<-SQL
INSERT INTO post_order (id)
INSERT INTO post_order (post_id)
SELECT id
FROM post
ORDER BY created_at, topic_id, id
SQL
end
def delete_unused_users
@db.execute <<~SQL
DELETE FROM user
WHERE NOT EXISTS(
SELECT 1
FROM topic
WHERE topic.user_id = user.id
) AND NOT EXISTS(
SELECT 1
FROM post
WHERE post.user_id = user.id
)
SQL
end
def fetch_categories
@db.execute(<<-SQL)
SELECT *
@ -82,6 +119,14 @@ module ImportScripts
add_last_column_value(rows, 'id')
end
def get_user_id(username)
@db.get_first_value(<<-SQL, username)
SELECT id
FROM user
WHERE username = :username
SQL
end
def count_topics
@db.get_first_value(<<-SQL)
SELECT COUNT(*)
@ -101,6 +146,14 @@ module ImportScripts
add_last_column_value(rows, 'id')
end
def fetch_topic_attachments(topic_id)
@db.execute(<<-SQL, topic_id)
SELECT path
FROM topic_upload
WHERE topic_id = :topic_id
SQL
end
def count_posts
@db.get_first_value(<<-SQL)
SELECT COUNT(*)
@ -110,9 +163,21 @@ module ImportScripts
def fetch_posts(last_row_id)
rows = @db.execute(<<-SQL, last_row_id)
SELECT o.ROWID, p.*
SELECT ROWID AS rowid, *
FROM post
WHERE ROWID > :last_row_id
ORDER BY ROWID
LIMIT #{@batch_size}
SQL
add_last_column_value(rows, 'rowid')
end
def fetch_sorted_posts(last_row_id)
rows = @db.execute(<<-SQL, last_row_id)
SELECT o.ROWID AS rowid, p.*
FROM post p
JOIN post_order o USING (id)
JOIN post_order o ON (p.id = o.post_id)
WHERE o.ROWID > :last_row_id
ORDER BY o.ROWID
LIMIT #{@batch_size}
@ -121,6 +186,14 @@ module ImportScripts
add_last_column_value(rows, 'rowid')
end
def fetch_post_attachments(post_id)
@db.execute(<<-SQL, post_id)
SELECT path
FROM post_upload
WHERE post_id = :post_id
SQL
end
def execute_sql(sql)
@db.execute(sql)
end
@ -136,10 +209,14 @@ module ImportScripts
@db.execute 'PRAGMA locking_mode = EXCLUSIVE'
end
def key_data_type
@numeric_keys ? 'INTEGER' : 'TEXT'
end
def create_category_table
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS category (
id TEXT NOT NULL PRIMARY KEY,
id #{key_data_type} NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
description TEXT,
position INTEGER,
@ -151,44 +228,59 @@ module ImportScripts
def create_user_table
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user (
id TEXT NOT NULL PRIMARY KEY,
id #{key_data_type} NOT NULL PRIMARY KEY,
email TEXT,
username TEXT,
name TEXT,
bio TEXT,
avatar_path TEXT,
created_at DATETIME,
last_seen_at DATETIME,
active BOOLEAN NOT NULL DEFAULT true
)
SQL
@db.execute 'CREATE INDEX IF NOT EXISTS user_by_username ON user (username)'
end
def create_topic_table
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic (
id TEXT NOT NULL PRIMARY KEY,
id #{key_data_type} NOT NULL PRIMARY KEY,
title TEXT,
raw TEXT,
category_id TEXT NOT NULL,
category_id #{key_data_type},
closed BOOLEAN NOT NULL DEFAULT false,
user_id TEXT NOT NULL,
user_id #{key_data_type} NOT NULL,
created_at DATETIME,
url TEXT
url TEXT,
upload_count INTEGER DEFAULT 0
)
SQL
@db.execute 'CREATE INDEX IF NOT EXISTS topic_by_user_id ON topic (user_id)'
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_upload (
topic_id #{key_data_type} NOT NULL,
path TEXT NOT NULL
)
SQL
@db.execute 'CREATE UNIQUE INDEX IF NOT EXISTS topic_upload_unique ON topic_upload(topic_id, path)'
end
def create_post_table
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS post (
id TEXT NOT NULL PRIMARY KEY,
id #{key_data_type} NOT NULL PRIMARY KEY,
raw TEXT,
topic_id TEXT NOT NULL,
user_id TEXT NOT NULL,
topic_id #{key_data_type} NOT NULL,
user_id #{key_data_type} NOT NULL,
created_at DATETIME,
reply_to_post_id TEXT,
url TEXT
reply_to_post_id #{key_data_type},
url TEXT,
upload_count INTEGER DEFAULT 0
)
SQL
@ -196,9 +288,18 @@ module ImportScripts
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS post_order (
id TEXT NOT NULL PRIMARY KEY
post_id #{key_data_type} NOT NULL PRIMARY KEY
)
SQL
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS post_upload (
post_id #{key_data_type} NOT NULL,
path TEXT NOT NULL
)
SQL
@db.execute 'CREATE UNIQUE INDEX IF NOT EXISTS post_upload_unique ON post_upload(post_id, path)'
end
def prepare(hash)