improvements to importer

- improve perf of test for existing posts
- always use a system guardian when importing posts
- for lithuim importer requery raw (transform is not repeatable)
This commit is contained in:
Sam
2015-10-15 13:25:10 +11:00
parent 06f616792d
commit 606aeb9d55
3 changed files with 37 additions and 15 deletions

View File

@ -197,13 +197,20 @@ class ImportScripts::Base
def all_records_exist?(type, import_ids)
return false if import_ids.empty?
Post.exec_sql('create temp table import_ids(val varchar(200) primary key)')
import_id_clause = import_ids.map{|id| "('#{PG::Connection.escape_string(id)}')"}.join(",")
Post.exec_sql("insert into import_ids values #{import_id_clause}")
existing = "#{type.to_s.classify}CustomField".constantize.where(name: 'import_id')
existing = existing.where('value in (?)', import_ids.map(&:to_s))
existing = existing.joins('JOIN import_ids ON val=value')
if existing.count == import_ids.length
# puts "Skipping #{import_ids.length} already imported #{type}"
true
puts "Skipping #{import_ids.length} already imported #{type}"
return true
end
ensure
Post.exec_sql('drop table import_ids')
end
# Iterate through a list of user records to be imported.
@ -444,6 +451,8 @@ class ImportScripts::Base
[created, skipped]
end
STAFF_GUARDIAN = Guardian.new(User.find(-1))
def create_post(opts, import_id)
user = User.find(opts[:user_id])
post_create_action = opts.delete(:post_create_action)
@ -452,6 +461,7 @@ class ImportScripts::Base
opts[:custom_fields] ||= {}
opts[:custom_fields]['import_id'] = import_id
opts[:guardian] = STAFF_GUARDIAN
if @bbcode_to_md
opts[:raw] = opts[:raw].bbcode_to_md(false) rescue opts[:raw]
end