github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-06-04 20:34:50 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Improvements to phpBB3 import script (#10999)

Browse Source 
* FEATURE: Import attachments

* FEATURE: Add support for importing multiple forums in one

* FEATURE: Add support for category and tag mapping

* FEATURE: Import groups

* FIX: Add spaces around images

* FEATURE: Custom mapping of user rank to trust levels

* FIX: Do not fail import if it cannot import polls

* FIX: Optimize existing records lookup

Co-authored-by: Gerhard Schlager <mail@gerhard-schlager.at>
Co-authored-by: Jarek Radosz <jradosz@gmail.com>
This commit is contained in:
Bianca Nenciu
2021-01-14 21:44:43 +02:00
committed by GitHub
parent 82af278ae5
commit a71b219c9a
20 changed files with 1606 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
script/import_scripts/phpbb3/importer.rb
View File

@ -27,8 +27,13 @@ module ImportScripts::PhpBB3
def execute
puts '', "importing from phpBB #{@php_config[:phpbb_version]}"
SiteSetting.tagging_enabled = true if @settings.tag_mappings.present?
import_users
import_anonymous_users if @settings.import_anonymous_users
import_groups
import_user_groups
import_new_categories
import_categories
import_posts
import_private_messages if @settings.import_private_messages
@ -67,12 +72,12 @@ module ImportScripts::PhpBB3
batches do |offset|
rows, last_user_id = @database.fetch_users(last_user_id)
rows = rows.to_a.uniq { |row| row[:user_id] }
break if rows.size < 1
next if all_records_exist?(:users, importer.map_users_to_import_ids(rows))
create_users(rows, total: total_count, offset: offset) do |row|
begin
next if user_id_from_imported_user_id(@settings.prefix(row[:user_id]))
importer.map_user(row)
rescue => e
log_error("Failed to map user with ID #{row[:user_id]}", e)
@ -91,10 +96,9 @@ module ImportScripts::PhpBB3
rows, last_username = @database.fetch_anonymous_users(last_username)
break if rows.size < 1
next if all_records_exist?(:users, importer.map_anonymous_users_to_import_ids(rows))
create_users(rows, total: total_count, offset: offset) do |row|
begin
next if user_id_from_imported_user_id(@settings.prefix(row[:post_username]))
importer.map_anonymous_user(row)
rescue => e
log_error("Failed to map anonymous user with ID #{row[:user_id]}", e)
@ -103,12 +107,74 @@ module ImportScripts::PhpBB3
end
end
def import_groups
puts '', 'creating groups'
rows = @database.fetch_groups
create_groups(rows) do |row|
begin
next if row[:group_type] == 3
group_name = if @settings.site_name.present?
"#{@settings.site_name}_#{row[:group_name]}"
else
row[:group_name]
end[0..19].gsub(/[^a-zA-Z0-9\-_. ]/, '_')
bio_raw = @importers.text_processor.process_raw_text(row[:group_desc]) rescue row[:group_desc]
{
id: @settings.prefix(row[:group_id]),
name: group_name,
full_name: row[:group_name],
bio_raw: bio_raw
}
rescue => e
log_error("Failed to map group with ID #{row[:group_id]}", e)
end
end
end
def import_user_groups
puts '', 'creating user groups'
rows = @database.fetch_group_users
rows.each do |row|
group_id = @lookup.group_id_from_imported_group_id(@settings.prefix(row[:group_id]))
next if !group_id
user_id = @lookup.user_id_from_imported_user_id(@settings.prefix(row[:user_id]))
begin
GroupUser.find_or_create_by(user_id: user_id, group_id: group_id, owner: row[:group_leader])
rescue => e
log_error("Failed to add user #{row[:user_id]} to group #{row[:group_id]}", e)
end
end
end
def import_new_categories
puts '', 'creating new categories'
create_categories(@settings.new_categories) do |row|
next if row == "SKIP"
{
id: @settings.prefix(row[:forum_id]),
name: row[:name],
parent_category_id: @lookup.category_id_from_imported_category_id(@settings.prefix(row[:parent_id]))
}
end
end
def import_categories
puts '', 'creating categories'
rows = @database.fetch_categories
importer = @importers.category_importer
create_categories(rows) do |row|
next if @settings.category_mappings[row[:forum_id].to_s] == 'SKIP'
importer.map_category(row)
end
end
@ -123,10 +189,9 @@ module ImportScripts::PhpBB3
rows, last_post_id = @database.fetch_posts(last_post_id)
break if rows.size < 1
next if all_records_exist?(:posts, importer.map_to_import_ids(rows))
create_posts(rows, total: total_count, offset: offset) do |row|
begin
next if post_id_from_imported_post_id(@settings.prefix(row[:post_id]))
importer.map_post(row)
rescue => e
log_error("Failed to map post with ID #{row[:post_id]}", e)
@ -145,10 +210,9 @@ module ImportScripts::PhpBB3
rows, last_msg_id = @database.fetch_messages(last_msg_id)
break if rows.size < 1
next if all_records_exist?(:posts, importer.map_to_import_ids(rows))
create_posts(rows, total: total_count, offset: offset) do |row|
begin
next if post_id_from_imported_post_id(@settings.prefix("pm:#{row[:msg_id]}"))
importer.map_message(row)
rescue => e
log_error("Failed to map message with ID #{row[:msg_id]}", e)
@ -168,7 +232,11 @@ module ImportScripts::PhpBB3
break if rows.size < 1
create_bookmarks(rows, total: total_count, offset: offset) do |row|
importer.map_bookmark(row)
begin
importer.map_bookmark(row)
rescue => e
log_error("Failed to map bookmark (#{row[:user_id]}, #{row[:topic_first_post_id]})", e)
end
end
end
end