Enhance bulk import scripts (#5010)

* Enhance bulk import scripts

* Fix: restore running statement of BulkImport::VBulletin
This commit is contained in:
Quangbuu Le
2017-07-31 15:56:57 +07:00
committed by Régis Hanol
parent 83bb042311
commit 0daa177805
2 changed files with 27 additions and 8 deletions

View File

@ -176,7 +176,7 @@ class BulkImport::Base
POST_COLUMNS ||= %i{
id user_id last_editor_id topic_id post_number sort_order reply_to_post_number
raw cooked hidden word_count created_at last_version_at updated_at
like_count raw cooked hidden word_count created_at last_version_at updated_at
}
TOPIC_ALLOWED_USER_COLUMNS ||= %i{
@ -306,7 +306,8 @@ class BulkImport::Base
end
def process_category(category)
@categories[category[:imported_id].to_s] = category[:id] = @last_category_id += 1
category[:id] ||= @last_category_id += 1
@categories[category[:imported_id].to_s] ||= category[:id]
category[:name] = category[:name][0...50].scrub.strip
# TODO: unique name
category[:name_lower] = category[:name].downcase
@ -347,6 +348,7 @@ class BulkImport::Base
@topic_id_by_post_id[post[:id]] = post[:topic_id]
post[:raw] = (post[:raw] || "").scrub.strip.presence || "<Empty imported post>"
post[:raw] = process_raw post[:raw]
post[:like_count] ||= 0
post[:cooked] = pre_cook post[:raw]
post[:hidden] ||= false
post[:word_count] = post[:raw].scan(/[[:word:]]+/).size
@ -484,7 +486,8 @@ class BulkImport::Base
mapped = yield(row)
next unless mapped
processed = send(process_method_name, mapped)
imported_ids << mapped[:imported_id]
imported_ids << mapped[:imported_id] unless mapped[:imported_id].nil?
imported_ids |= mapped[:imported_ids] unless mapped[:imported_ids].nil?
@raw_connection.put_copy_data columns.map { |c| processed[c] }
print "\r%7d - %6d/sec".freeze % [imported_ids.size, imported_ids.size.to_f / (Time.now - start)] if imported_ids.size % 5000 == 0
end
@ -538,7 +541,18 @@ class BulkImport::Base
end
def pre_cook(raw)
cooked = @markdown.render(raw).scrub.strip
cooked = raw
# Convert YouTube URLs to lazyYT DOMs before being transformed into links
cooked.gsub!(/\nhttps\:\/\/www.youtube.com\/watch\?v=(\w+)\n/) do
video_id = $1
result = <<-HTML
<div class="lazyYT" data-youtube-id="#{video_id}" data-width="480" data-height="270" data-parameters="feature=oembed&amp;wmode=opaque"></div>
HTML
result.strip
end
cooked = @markdown.render(cooked).scrub.strip
cooked.gsub!(/\[QUOTE="?([^,"]+)(?:, post:(\d+), topic:(\d+))?"?\](.+?)\[\/QUOTE\]/im) do
username, post_id, topic_id = $1, $2, $3