mirror of
https://github.com/discourse/discourse.git
synced 2025-05-22 04:41:25 +08:00
Enhance bulk import scripts (#5010)
* Enhance bulk import scripts * Fix: restore running statement of BulkImport::VBulletin
This commit is contained in:
@ -176,7 +176,7 @@ class BulkImport::Base
|
||||
|
||||
POST_COLUMNS ||= %i{
|
||||
id user_id last_editor_id topic_id post_number sort_order reply_to_post_number
|
||||
raw cooked hidden word_count created_at last_version_at updated_at
|
||||
like_count raw cooked hidden word_count created_at last_version_at updated_at
|
||||
}
|
||||
|
||||
TOPIC_ALLOWED_USER_COLUMNS ||= %i{
|
||||
@ -306,7 +306,8 @@ class BulkImport::Base
|
||||
end
|
||||
|
||||
def process_category(category)
|
||||
@categories[category[:imported_id].to_s] = category[:id] = @last_category_id += 1
|
||||
category[:id] ||= @last_category_id += 1
|
||||
@categories[category[:imported_id].to_s] ||= category[:id]
|
||||
category[:name] = category[:name][0...50].scrub.strip
|
||||
# TODO: unique name
|
||||
category[:name_lower] = category[:name].downcase
|
||||
@ -347,6 +348,7 @@ class BulkImport::Base
|
||||
@topic_id_by_post_id[post[:id]] = post[:topic_id]
|
||||
post[:raw] = (post[:raw] || "").scrub.strip.presence || "<Empty imported post>"
|
||||
post[:raw] = process_raw post[:raw]
|
||||
post[:like_count] ||= 0
|
||||
post[:cooked] = pre_cook post[:raw]
|
||||
post[:hidden] ||= false
|
||||
post[:word_count] = post[:raw].scan(/[[:word:]]+/).size
|
||||
@ -484,7 +486,8 @@ class BulkImport::Base
|
||||
mapped = yield(row)
|
||||
next unless mapped
|
||||
processed = send(process_method_name, mapped)
|
||||
imported_ids << mapped[:imported_id]
|
||||
imported_ids << mapped[:imported_id] unless mapped[:imported_id].nil?
|
||||
imported_ids |= mapped[:imported_ids] unless mapped[:imported_ids].nil?
|
||||
@raw_connection.put_copy_data columns.map { |c| processed[c] }
|
||||
print "\r%7d - %6d/sec".freeze % [imported_ids.size, imported_ids.size.to_f / (Time.now - start)] if imported_ids.size % 5000 == 0
|
||||
end
|
||||
@ -538,7 +541,18 @@ class BulkImport::Base
|
||||
end
|
||||
|
||||
def pre_cook(raw)
|
||||
cooked = @markdown.render(raw).scrub.strip
|
||||
cooked = raw
|
||||
|
||||
# Convert YouTube URLs to lazyYT DOMs before being transformed into links
|
||||
cooked.gsub!(/\nhttps\:\/\/www.youtube.com\/watch\?v=(\w+)\n/) do
|
||||
video_id = $1
|
||||
result = <<-HTML
|
||||
<div class="lazyYT" data-youtube-id="#{video_id}" data-width="480" data-height="270" data-parameters="feature=oembed&wmode=opaque"></div>
|
||||
HTML
|
||||
result.strip
|
||||
end
|
||||
|
||||
cooked = @markdown.render(cooked).scrub.strip
|
||||
|
||||
cooked.gsub!(/\[QUOTE="?([^,"]+)(?:, post:(\d+), topic:(\d+))?"?\](.+?)\[\/QUOTE\]/im) do
|
||||
username, post_id, topic_id = $1, $2, $3
|
||||
|
Reference in New Issue
Block a user