mirror of
https://github.com/discourse/discourse.git
synced 2025-06-10 03:43:39 +08:00
improve simple press importer
This commit is contained in:
@ -22,7 +22,8 @@ class ImportScripts::SimplePress < ImportScripts::Base
|
|||||||
def execute
|
def execute
|
||||||
import_users
|
import_users
|
||||||
import_categories
|
import_categories
|
||||||
import_topics_and_posts
|
import_topics
|
||||||
|
import_posts
|
||||||
end
|
end
|
||||||
|
|
||||||
def import_users
|
def import_users
|
||||||
@ -93,12 +94,12 @@ class ImportScripts::SimplePress < ImportScripts::Base
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def import_topics_and_posts
|
def import_topics
|
||||||
puts "", "creating topics and posts"
|
puts "", "creating topics"
|
||||||
|
|
||||||
total_count = mysql_query("SELECT count(*) count from #{TABLE_PREFIX}posts").first["count"]
|
total_count = mysql_query("SELECT COUNT(*) count FROM #{TABLE_PREFIX}posts WHERE post_index = 1").first["count"]
|
||||||
|
|
||||||
topic_first_post_id = {}
|
@topic_first_post_id = {}
|
||||||
|
|
||||||
batches(BATCH_SIZE) do |offset|
|
batches(BATCH_SIZE) do |offset|
|
||||||
results = mysql_query("
|
results = mysql_query("
|
||||||
@ -106,14 +107,16 @@ class ImportScripts::SimplePress < ImportScripts::Base
|
|||||||
p.topic_id topic_id,
|
p.topic_id topic_id,
|
||||||
t.forum_id category_id,
|
t.forum_id category_id,
|
||||||
t.topic_name title,
|
t.topic_name title,
|
||||||
p.post_index post_index,
|
t.topic_opened views,
|
||||||
|
t.topic_pinned pinned,
|
||||||
p.user_id user_id,
|
p.user_id user_id,
|
||||||
p.post_content raw,
|
p.post_content raw,
|
||||||
p.post_date post_time
|
p.post_date post_time
|
||||||
FROM #{TABLE_PREFIX}posts p,
|
FROM #{TABLE_PREFIX}posts p,
|
||||||
#{TABLE_PREFIX}topics t
|
#{TABLE_PREFIX}topics t
|
||||||
WHERE p.topic_id = t.topic_id
|
WHERE p.topic_id = t.topic_id
|
||||||
ORDER BY p.post_date
|
AND p.post_index = 1
|
||||||
|
ORDER BY p.post_id
|
||||||
LIMIT #{BATCH_SIZE}
|
LIMIT #{BATCH_SIZE}
|
||||||
OFFSET #{offset};
|
OFFSET #{offset};
|
||||||
")
|
")
|
||||||
@ -123,29 +126,60 @@ class ImportScripts::SimplePress < ImportScripts::Base
|
|||||||
next if all_records_exist? :posts, results.map { |m| m['id'].to_i }
|
next if all_records_exist? :posts, results.map { |m| m['id'].to_i }
|
||||||
|
|
||||||
create_posts(results, total: total_count, offset: offset) do |m|
|
create_posts(results, total: total_count, offset: offset) do |m|
|
||||||
skip = false
|
@topic_first_post_id[m['topic_id']] = m['id']
|
||||||
mapped = {}
|
created_at = Time.zone.at(m['post_time'])
|
||||||
|
{
|
||||||
mapped[:id] = m['id']
|
id: m['id'],
|
||||||
mapped[:user_id] = user_id_from_imported_user_id(m['user_id']) || -1
|
user_id: user_id_from_imported_user_id(m['user_id']) || -1,
|
||||||
mapped[:raw] = process_simplepress_post(m['raw'], m['id'])
|
raw: process_simplepress_post(m['raw'], m['id']),
|
||||||
mapped[:created_at] = Time.zone.at(m['post_time'])
|
created_at: created_at,
|
||||||
|
category: category_id_from_imported_category_id(m['category_id']),
|
||||||
if m['post_index'] == 1
|
title: CGI.unescapeHTML(m['title']),
|
||||||
mapped[:category] = category_id_from_imported_category_id(m['category_id'])
|
views: m['views'],
|
||||||
mapped[:title] = CGI.unescapeHTML(m['title'])
|
pinned_at: m['pinned'] == 1 ? created_at : nil,
|
||||||
topic_first_post_id[m['topic_id']] = m['id']
|
}
|
||||||
else
|
end
|
||||||
parent = topic_lookup_from_imported_post_id(topic_first_post_id[m['topic_id']])
|
|
||||||
if parent
|
|
||||||
mapped[:topic_id] = parent[:topic_id]
|
|
||||||
else
|
|
||||||
puts "Parent post #{first_post_id} doesn't exist. Skipping #{m["id"]}: #{m["title"][0..40]}"
|
|
||||||
skip = true
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
skip ? nil : mapped
|
def import_posts
|
||||||
|
puts "", "creating posts"
|
||||||
|
|
||||||
|
total_count = mysql_query("SELECT count(*) count FROM #{TABLE_PREFIX}posts WHERE post_index <> 1").first["count"]
|
||||||
|
|
||||||
|
batches(BATCH_SIZE) do |offset|
|
||||||
|
results = mysql_query("
|
||||||
|
SELECT p.post_id id,
|
||||||
|
p.topic_id topic_id,
|
||||||
|
p.user_id user_id,
|
||||||
|
p.post_content raw,
|
||||||
|
p.post_date post_time
|
||||||
|
FROM #{TABLE_PREFIX}posts p,
|
||||||
|
#{TABLE_PREFIX}topics t
|
||||||
|
WHERE p.topic_id = t.topic_id
|
||||||
|
AND p.post_index <> 1
|
||||||
|
ORDER BY p.post_id
|
||||||
|
LIMIT #{BATCH_SIZE}
|
||||||
|
OFFSET #{offset};
|
||||||
|
")
|
||||||
|
|
||||||
|
break if results.size < 1
|
||||||
|
|
||||||
|
next if all_records_exist? :posts, results.map { |m| m['id'].to_i }
|
||||||
|
|
||||||
|
create_posts(results, total: total_count, offset: offset) do |m|
|
||||||
|
if parent = topic_lookup_from_imported_post_id(@topic_first_post_id[m['topic_id']])
|
||||||
|
{
|
||||||
|
id: m['id'],
|
||||||
|
user_id: user_id_from_imported_user_id(m['user_id']) || -1,
|
||||||
|
topic_id: parent[:topic_id],
|
||||||
|
raw: process_simplepress_post(m['raw'], m['id']),
|
||||||
|
created_at: Time.zone.at(m['post_time']),
|
||||||
|
}
|
||||||
|
else
|
||||||
|
puts "Parent post #{m['topic_id']} doesn't exist. Skipping #{m["id"]}"
|
||||||
|
nil
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
Reference in New Issue
Block a user