github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-23 18:51:07 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: Apply syntax_tree formatting to script/*

Browse Source
This commit is contained in:
David Taylor
2023-01-07 11:53:14 +00:00
parent ff508d1ae5
commit 436b3b392b
143 changed files with 8905 additions and 7353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
script/import_scripts/phpbb3/database/database.rb
View File

@ -1,6 +1,6 @@
# frozen_string_literal: true
require 'mysql2'
require "mysql2"
module ImportScripts::PhpBB3
class Database
@ -19,11 +19,11 @@ module ImportScripts::PhpBB3
def create_database
version = get_phpbb_version
if version.start_with?('3.0')
require_relative 'database_3_0'
if version.start_with?("3.0")
require_relative "database_3_0"
Database_3_0.new(@database_client, @database_settings)
elsif version.start_with?('3.1') || version.start_with?('3.2') || version.start_with?('3.3')
require_relative 'database_3_1'
elsif version.start_with?("3.1") || version.start_with?("3.2") || version.start_with?("3.3")
require_relative "database_3_1"
Database_3_1.new(@database_client, @database_settings)
else
raise UnsupportedVersionError, <<~TEXT
@ -42,7 +42,7 @@ module ImportScripts::PhpBB3
username: @database_settings.username,
password: @database_settings.password,
database: @database_settings.schema,
reconnect: true
reconnect: true,
)
end

4
script/import_scripts/phpbb3/database/database_3_0.rb
View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
require_relative 'database_base'
require_relative '../support/constants'
require_relative "database_base"
require_relative "../support/constants"
module ImportScripts::PhpBB3
class Database_3_0 < DatabaseBase

19
script/import_scripts/phpbb3/database/database_3_1.rb
View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
require_relative 'database_3_0'
require_relative '../support/constants'
require_relative "database_3_0"
require_relative "../support/constants"
module ImportScripts::PhpBB3
class Database_3_1 < Database_3_0
@ -32,14 +32,15 @@ module ImportScripts::PhpBB3
private
def profile_fields_query(profile_fields)
@profile_fields_query ||= begin
if profile_fields.present?
columns = profile_fields.map { |field| "pf_#{field[:phpbb_field_name]}" }
", #{columns.join(', ')}"
else
""
@profile_fields_query ||=
begin
if profile_fields.present?
columns = profile_fields.map { |field| "pf_#{field[:phpbb_field_name]}" }
", #{columns.join(", ")}"
else
""
end
end
end
end
end
end

4
script/import_scripts/phpbb3/database/database_base.rb
View File

@ -39,9 +39,7 @@ module ImportScripts::PhpBB3
def find_last_row(rows)
last_index = rows.size - 1
rows.each_with_index do |row, index|
return row if index == last_index
end
rows.each_with_index { |row, index| return row if index == last_index }
nil
end

75
script/import_scripts/phpbb3/importer.rb
View File

@ -1,9 +1,9 @@
# frozen_string_literal: true
require_relative '../base'
require_relative 'support/settings'
require_relative 'database/database'
require_relative 'importers/importer_factory'
require_relative "../base"
require_relative "support/settings"
require_relative "database/database"
require_relative "importers/importer_factory"
module ImportScripts::PhpBB3
class Importer < ImportScripts::Base
@ -25,7 +25,7 @@ module ImportScripts::PhpBB3
protected
def execute
puts '', "importing from phpBB #{@php_config[:phpbb_version]}"
puts "", "importing from phpBB #{@php_config[:phpbb_version]}"
SiteSetting.tagging_enabled = true if @settings.tag_mappings.present?
@ -55,8 +55,14 @@ module ImportScripts::PhpBB3
settings[:max_attachment_size_kb] = [max_file_size_kb, SiteSetting.max_attachment_size_kb].max
# temporarily disable validation since we want to import all existing images and attachments
SiteSetting.type_supervisor.load_setting(:max_image_size_kb, max: settings[:max_image_size_kb])
SiteSetting.type_supervisor.load_setting(:max_attachment_size_kb, max: settings[:max_attachment_size_kb])
SiteSetting.type_supervisor.load_setting(
:max_image_size_kb,
max: settings[:max_image_size_kb],
)
SiteSetting.type_supervisor.load_setting(
:max_attachment_size_kb,
max: settings[:max_attachment_size_kb],
)
settings
end
@ -66,7 +72,7 @@ module ImportScripts::PhpBB3
end
def import_users
puts '', 'creating users'
puts "", "creating users"
total_count = @database.count_users
importer = @importers.user_importer
last_user_id = 0
@ -88,10 +94,10 @@ module ImportScripts::PhpBB3
end
def import_anonymous_users
puts '', 'creating anonymous users'
puts "", "creating anonymous users"
total_count = @database.count_anonymous_users
importer = @importers.user_importer
last_username = ''
last_username = ""
batches do |offset|
rows, last_username = @database.fetch_anonymous_users(last_username)
@ -109,26 +115,34 @@ module ImportScripts::PhpBB3
end
def import_groups
puts '', 'creating groups'
puts "", "creating groups"
rows = @database.fetch_groups
create_groups(rows) do |row|
begin
next if row[:group_type] == 3
group_name = if @settings.site_name.present?
"#{@settings.site_name}_#{row[:group_name]}"
else
row[:group_name]
end[0..19].gsub(/[^a-zA-Z0-9\-_. ]/, '_')
group_name =
if @settings.site_name.present?
"#{@settings.site_name}_#{row[:group_name]}"
else
row[:group_name]
end[
0..19
].gsub(/[^a-zA-Z0-9\-_. ]/, "_")
bio_raw = @importers.text_processor.process_raw_text(row[:group_desc]) rescue row[:group_desc]
bio_raw =
begin
@importers.text_processor.process_raw_text(row[:group_desc])
rescue StandardError
row[:group_desc]
end
{
id: @settings.prefix(row[:group_id]),
name: group_name,
full_name: row[:group_name],
bio_raw: bio_raw
bio_raw: bio_raw,
}
rescue => e
log_error("Failed to map group with ID #{row[:group_id]}", e)
@ -137,7 +151,7 @@ module ImportScripts::PhpBB3
end
def import_user_groups
puts '', 'creating user groups'
puts "", "creating user groups"
rows = @database.fetch_group_users
rows.each do |row|
@ -147,7 +161,11 @@ module ImportScripts::PhpBB3
user_id = @lookup.user_id_from_imported_user_id(@settings.prefix(row[:user_id]))
begin
GroupUser.find_or_create_by(user_id: user_id, group_id: group_id, owner: row[:group_leader])
GroupUser.find_or_create_by(
user_id: user_id,
group_id: group_id,
owner: row[:group_leader],
)
rescue => e
log_error("Failed to add user #{row[:user_id]} to group #{row[:group_id]}", e)
end
@ -155,7 +173,7 @@ module ImportScripts::PhpBB3
end
def import_new_categories
puts '', 'creating new categories'
puts "", "creating new categories"
create_categories(@settings.new_categories) do |row|
next if row == "SKIP"
@ -163,13 +181,14 @@ module ImportScripts::PhpBB3
{
id: @settings.prefix(row[:forum_id]),
name: row[:name],
parent_category_id: @lookup.category_id_from_imported_category_id(@settings.prefix(row[:parent_id]))
parent_category_id:
@lookup.category_id_from_imported_category_id(@settings.prefix(row[:parent_id])),
}
end
end
def import_categories
puts '', 'creating categories'
puts "", "creating categories"
rows = @database.fetch_categories
importer = @importers.category_importer
@ -181,7 +200,7 @@ module ImportScripts::PhpBB3
end
def import_posts
puts '', 'creating topics and posts'
puts "", "creating topics and posts"
total_count = @database.count_posts
importer = @importers.post_importer
last_post_id = 0
@ -202,7 +221,7 @@ module ImportScripts::PhpBB3
end
def import_private_messages
puts '', 'creating private messages'
puts "", "creating private messages"
total_count = @database.count_messages
importer = @importers.message_importer
last_msg_id = 0
@ -223,7 +242,7 @@ module ImportScripts::PhpBB3
end
def import_bookmarks
puts '', 'creating bookmarks'
puts "", "creating bookmarks"
total_count = @database.count_bookmarks
importer = @importers.bookmark_importer
last_user_id = last_topic_id = 0
@ -243,7 +262,7 @@ module ImportScripts::PhpBB3
end
def import_likes
puts '', 'importing likes'
puts "", "importing likes"
total_count = @database.count_likes
last_post_id = last_user_id = 0
@ -255,7 +274,7 @@ module ImportScripts::PhpBB3
{
post_id: @settings.prefix(row[:post_id]),
user_id: @settings.prefix(row[:user_id]),
created_at: Time.zone.at(row[:thanks_time])
created_at: Time.zone.at(row[:thanks_time]),
}
end
end

29
script/import_scripts/phpbb3/importers/avatar_importer.rb
View File

@ -49,12 +49,12 @@ module ImportScripts::PhpBB3
def get_avatar_path(avatar_type, filename)
case avatar_type
when Constants::AVATAR_TYPE_UPLOADED, Constants::AVATAR_TYPE_STRING_UPLOADED then
filename.gsub!(/_[0-9]+\./, '.') # we need 1337.jpg, not 1337_2983745.jpg
get_uploaded_path(filename)
when Constants::AVATAR_TYPE_GALLERY, Constants::AVATAR_TYPE_STRING_GALLERY then
when Constants::AVATAR_TYPE_UPLOADED, Constants::AVATAR_TYPE_STRING_UPLOADED
filename.gsub!(/_[0-9]+\./, ".") # we need 1337.jpg, not 1337_2983745.jpg
get_uploaded_path(filename)
when Constants::AVATAR_TYPE_GALLERY, Constants::AVATAR_TYPE_STRING_GALLERY
get_gallery_path(filename)
when Constants::AVATAR_TYPE_REMOTE, Constants::AVATAR_TYPE_STRING_REMOTE then
when Constants::AVATAR_TYPE_REMOTE, Constants::AVATAR_TYPE_STRING_REMOTE
download_avatar(filename)
else
puts "Invalid avatar type #{avatar_type}. Skipping..."
@ -67,12 +67,13 @@ module ImportScripts::PhpBB3
max_image_size_kb = SiteSetting.max_image_size_kb.kilobytes
begin
avatar_file = FileHelper.download(
url,
max_file_size: max_image_size_kb,
tmp_file_name: 'discourse-avatar',
follow_redirect: true
)
avatar_file =
FileHelper.download(
url,
max_file_size: max_image_size_kb,
tmp_file_name: "discourse-avatar",
follow_redirect: true,
)
rescue StandardError => err
warn "Error downloading avatar: #{err.message}. Skipping..."
return nil
@ -100,11 +101,11 @@ module ImportScripts::PhpBB3
def is_allowed_avatar_type?(avatar_type)
case avatar_type
when Constants::AVATAR_TYPE_UPLOADED, Constants::AVATAR_TYPE_STRING_UPLOADED then
when Constants::AVATAR_TYPE_UPLOADED, Constants::AVATAR_TYPE_STRING_UPLOADED
@settings.import_uploaded_avatars
when Constants::AVATAR_TYPE_REMOTE, Constants::AVATAR_TYPE_STRING_REMOTE then
when Constants::AVATAR_TYPE_REMOTE, Constants::AVATAR_TYPE_STRING_REMOTE
@settings.import_remote_avatars
when Constants::AVATAR_TYPE_GALLERY, Constants::AVATAR_TYPE_STRING_GALLERY then
when Constants::AVATAR_TYPE_GALLERY, Constants::AVATAR_TYPE_STRING_GALLERY
@settings.import_gallery_avatars
else
false

2
script/import_scripts/phpbb3/importers/bookmark_importer.rb
View File

@ -9,7 +9,7 @@ module ImportScripts::PhpBB3
def map_bookmark(row)
{
user_id: @settings.prefix(row[:user_id]),
post_id: @settings.prefix(row[:topic_first_post_id])
post_id: @settings.prefix(row[:topic_first_post_id]),
}
end
end

23
script/import_scripts/phpbb3/importers/category_importer.rb
View File

@ -23,11 +23,13 @@ module ImportScripts::PhpBB3
{
id: @settings.prefix(row[:forum_id]),
name: CGI.unescapeHTML(row[:forum_name]),
parent_category_id: @lookup.category_id_from_imported_category_id(@settings.prefix(row[:parent_id])),
post_create_action: proc do |category|
update_category_description(category, row)
@permalink_importer.create_for_category(category, row[:forum_id]) # skip @settings.prefix because ID is used in permalink generation
end
parent_category_id:
@lookup.category_id_from_imported_category_id(@settings.prefix(row[:parent_id])),
post_create_action:
proc do |category|
update_category_description(category, row)
@permalink_importer.create_for_category(category, row[:forum_id]) # skip @settings.prefix because ID is used in permalink generation
end,
}
end
@ -51,7 +53,16 @@ module ImportScripts::PhpBB3
end
if row[:forum_desc].present?
changes = { raw: (@text_processor.process_raw_text(row[:forum_desc]) rescue row[:forum_desc]) }
changes = {
raw:
(
begin
@text_processor.process_raw_text(row[:forum_desc])
rescue StandardError
row[:forum_desc]
end
),
}
opts = { revised_at: post.created_at, bypass_bump: true }
post.revise(Discourse.system_user, changes, opts)
end

34
script/import_scripts/phpbb3/importers/importer_factory.rb
View File

@ -1,16 +1,16 @@
# frozen_string_literal: true
require_relative 'attachment_importer'
require_relative 'avatar_importer'
require_relative 'bookmark_importer'
require_relative 'category_importer'
require_relative 'message_importer'
require_relative 'poll_importer'
require_relative 'post_importer'
require_relative 'permalink_importer'
require_relative 'user_importer'
require_relative '../support/smiley_processor'
require_relative '../support/text_processor'
require_relative "attachment_importer"
require_relative "avatar_importer"
require_relative "bookmark_importer"
require_relative "category_importer"
require_relative "message_importer"
require_relative "poll_importer"
require_relative "post_importer"
require_relative "permalink_importer"
require_relative "user_importer"
require_relative "../support/smiley_processor"
require_relative "../support/text_processor"
module ImportScripts::PhpBB3
class ImporterFactory
@ -36,7 +36,14 @@ module ImportScripts::PhpBB3
end
def post_importer
PostImporter.new(@lookup, text_processor, attachment_importer, poll_importer, permalink_importer, @settings)
PostImporter.new(
@lookup,
text_processor,
attachment_importer,
poll_importer,
permalink_importer,
@settings,
)
end
def message_importer
@ -64,7 +71,8 @@ module ImportScripts::PhpBB3
end
def text_processor
@text_processor ||= TextProcessor.new(@lookup, @database, smiley_processor, @settings, @phpbb_config)
@text_processor ||=
TextProcessor.new(@lookup, @database, smiley_processor, @settings, @phpbb_config)
end
def smiley_processor

43
script/import_scripts/phpbb3/importers/message_importer.rb
View File

@ -20,14 +20,16 @@ module ImportScripts::PhpBB3
end
def map_message(row)
user_id = @lookup.user_id_from_imported_user_id(@settings.prefix(row[:author_id])) || Discourse.system_user.id
user_id =
@lookup.user_id_from_imported_user_id(@settings.prefix(row[:author_id])) ||
Discourse.system_user.id
attachments = import_attachments(row, user_id)
mapped = {
id: get_import_id(row[:msg_id]),
user_id: user_id,
created_at: Time.zone.at(row[:message_time]),
raw: @text_processor.process_private_msg(row[:message_text], attachments)
raw: @text_processor.process_private_msg(row[:message_text], attachments),
}
root_user_ids = sorted_user_ids(row[:root_author_id], row[:root_to_address])
@ -43,7 +45,7 @@ module ImportScripts::PhpBB3
protected
RE_PREFIX = 're: '
RE_PREFIX = "re: "
def import_attachments(row, user_id)
if @settings.import_attachments && row[:attachment_count] > 0
@ -55,7 +57,7 @@ module ImportScripts::PhpBB3
mapped[:title] = get_topic_title(row)
mapped[:archetype] = Archetype.private_message
mapped[:target_usernames] = get_recipient_usernames(row)
mapped[:custom_fields] = { import_user_ids: current_user_ids.join(',') }
mapped[:custom_fields] = { import_user_ids: current_user_ids.join(",") }
if mapped[:target_usernames].empty?
puts "Private message without recipients. Skipping #{row[:msg_id]}: #{row[:message_subject][0..40]}"
@ -75,9 +77,9 @@ module ImportScripts::PhpBB3
# to_address looks like this: "u_91:u_1234:g_200"
# If there is a "u_" prefix, the prefix is discarded and the rest is a user_id
user_ids = to_address.split(':')
user_ids = to_address.split(":")
user_ids.uniq!
user_ids.map! { |u| u[2..-1].to_i if u[0..1] == 'u_' }.compact
user_ids.map! { |u| u[2..-1].to_i if u[0..1] == "u_" }.compact
end
def get_recipient_group_ids(to_address)
@ -85,16 +87,19 @@ module ImportScripts::PhpBB3
# to_address looks like this: "u_91:u_1234:g_200"
# If there is a "g_" prefix, the prefix is discarded and the rest is a group_id
group_ids = to_address.split(':')
group_ids = to_address.split(":")
group_ids.uniq!
group_ids.map! { |g| g[2..-1].to_i if g[0..1] == 'g_' }.compact
group_ids.map! { |g| g[2..-1].to_i if g[0..1] == "g_" }.compact
end
def get_recipient_usernames(row)
import_user_ids = get_recipient_user_ids(row[:to_address])
usernames = import_user_ids.map do |import_user_id|
@lookup.find_user_by_import_id(@settings.prefix(import_user_id)).try(:username)
end.compact
usernames =
import_user_ids
.map do |import_user_id|
@lookup.find_user_by_import_id(@settings.prefix(import_user_id)).try(:username)
end
.compact
import_group_ids = get_recipient_group_ids(row[:to_address])
import_group_ids.each do |import_group_id|
@ -142,13 +147,19 @@ module ImportScripts::PhpBB3
topic_titles = [topic_title]
topic_titles << topic_title[RE_PREFIX.length..-1] if topic_title.start_with?(RE_PREFIX)
Post.select(:topic_id)
Post
.select(:topic_id)
.joins(:topic)
.joins(:_custom_fields)
.where(["LOWER(topics.title) IN (:titles) AND post_custom_fields.name = 'import_user_ids' AND post_custom_fields.value = :user_ids",
{ titles: topic_titles, user_ids: current_user_ids.join(',') }])
.order('topics.created_at DESC')
.first.try(:topic_id)
.where(
[
"LOWER(topics.title) IN (:titles) AND post_custom_fields.name = 'import_user_ids' AND post_custom_fields.value = :user_ids",
{ titles: topic_titles, user_ids: current_user_ids.join(",") },
],
)
.order("topics.created_at DESC")
.first
.try(:topic_id)
end
end
end

12
script/import_scripts/phpbb3/importers/permalink_importer.rb
View File

@ -13,13 +13,15 @@ module ImportScripts::PhpBB3
def change_site_settings
normalizations = SiteSetting.permalink_normalizations
normalizations = normalizations.blank? ? [] : normalizations.split('|')
normalizations = normalizations.blank? ? [] : normalizations.split("|")
add_normalization(normalizations, CATEGORY_LINK_NORMALIZATION) if @settings.create_category_links
if @settings.create_category_links
add_normalization(normalizations, CATEGORY_LINK_NORMALIZATION)
end
add_normalization(normalizations, POST_LINK_NORMALIZATION) if @settings.create_post_links
add_normalization(normalizations, TOPIC_LINK_NORMALIZATION) if @settings.create_topic_links
SiteSetting.permalink_normalizations = normalizations.join('|')
SiteSetting.permalink_normalizations = normalizations.join("|")
end
def create_for_category(category, import_id)
@ -50,8 +52,8 @@ module ImportScripts::PhpBB3
def add_normalization(normalizations, normalization)
if @settings.normalization_prefix.present?
prefix = @settings.normalization_prefix[%r|^/?(.*?)/?$|, 1]
normalization = "/#{prefix.gsub('/', '\/')}\\#{normalization}"
prefix = @settings.normalization_prefix[%r{^/?(.*?)/?$}, 1]
normalization = "/#{prefix.gsub("/", '\/')}\\#{normalization}"
end
normalizations << normalization unless normalizations.include?(normalization)

24
script/import_scripts/phpbb3/importers/poll_importer.rb
View File

@ -49,7 +49,12 @@ module ImportScripts::PhpBB3
end
def get_option_text(row)
text = @text_processor.process_raw_text(row[:poll_option_text]) rescue row[:poll_option_text]
text =
begin
@text_processor.process_raw_text(row[:poll_option_text])
rescue StandardError
row[:poll_option_text]
end
text.squish!
text.gsub!(/^(\d+)\./, '\1\.')
text
@ -57,7 +62,12 @@ module ImportScripts::PhpBB3
# @param poll_data [ImportScripts::PhpBB3::PollData]
def get_poll_text(poll_data)
title = @text_processor.process_raw_text(poll_data.title) rescue poll_data.title
title =
begin
@text_processor.process_raw_text(poll_data.title)
rescue StandardError
poll_data.title
end
text = +"#{title}\n\n"
arguments = ["results=always"]
@ -69,11 +79,9 @@ module ImportScripts::PhpBB3
arguments << "type=regular"
end
text << "[poll #{arguments.join(' ')}]"
text << "[poll #{arguments.join(" ")}]"
poll_data.options.each do |option|
text << "\n* #{option[:text]}"
end
poll_data.options.each { |option| text << "\n* #{option[:text]}" }
text << "\n[/poll]"
end
@ -104,9 +112,7 @@ module ImportScripts::PhpBB3
poll.poll_options.each_with_index do |option, index|
imported_option = poll_data.options[index]
imported_option[:ids].each do |imported_id|
option_ids[imported_id] = option.id
end
imported_option[:ids].each { |imported_id| option_ids[imported_id] = option.id }
end
option_ids

21
script/import_scripts/phpbb3/importers/post_importer.rb
View File

@ -8,7 +8,14 @@ module ImportScripts::PhpBB3
# @param poll_importer [ImportScripts::PhpBB3::PollImporter]
# @param permalink_importer [ImportScripts::PhpBB3::PermalinkImporter]
# @param settings [ImportScripts::PhpBB3::Settings]
def initialize(lookup, text_processor, attachment_importer, poll_importer, permalink_importer, settings)
def initialize(
lookup,
text_processor,
attachment_importer,
poll_importer,
permalink_importer,
settings
)
@lookup = lookup
@text_processor = text_processor
@attachment_importer = attachment_importer
@ -24,7 +31,8 @@ module ImportScripts::PhpBB3
def map_post(row)
return if @settings.category_mappings.dig(row[:forum_id].to_s, :skip)
imported_user_id = @settings.prefix(row[:post_username].blank? ? row[:poster_id] : row[:post_username])
imported_user_id =
@settings.prefix(row[:post_username].blank? ? row[:poster_id] : row[:post_username])
user_id = @lookup.user_id_from_imported_user_id(imported_user_id) || -1
is_first_post = row[:post_id] == row[:topic_first_post_id]
@ -35,7 +43,7 @@ module ImportScripts::PhpBB3
user_id: user_id,
created_at: Time.zone.at(row[:post_time]),
raw: @text_processor.process_post(row[:post_text], attachments),
import_topic_id: @settings.prefix(row[:topic_id])
import_topic_id: @settings.prefix(row[:topic_id]),
}
if is_first_post
@ -58,7 +66,9 @@ module ImportScripts::PhpBB3
mapped[:category] = if category_mapping = @settings.category_mappings[row[:forum_id].to_s]
category_mapping[:discourse_category_id] ||
@lookup.category_id_from_imported_category_id(@settings.prefix(category_mapping[:target_category_id]))
@lookup.category_id_from_imported_category_id(
@settings.prefix(category_mapping[:target_category_id]),
)
else
@lookup.category_id_from_imported_category_id(@settings.prefix(row[:forum_id]))
end
@ -81,7 +91,8 @@ module ImportScripts::PhpBB3
end
def map_other_post(row, mapped)
parent = @lookup.topic_lookup_from_imported_post_id(@settings.prefix(row[:topic_first_post_id]))
parent =
@lookup.topic_lookup_from_imported_post_id(@settings.prefix(row[:topic_first_post_id]))
if parent.blank?
puts "Parent post #{@settings.prefix(row[:topic_first_post_id])} doesn't exist. Skipping #{@settings.prefix(row[:post_id])}: #{row[:topic_title][0..40]}"

81
script/import_scripts/phpbb3/importers/user_importer.rb
View File

@ -1,6 +1,6 @@
# frozen_string_literal: true
require_relative '../support/constants'
require_relative "../support/constants"
module ImportScripts::PhpBB3
class UserImporter
@ -29,8 +29,22 @@ module ImportScripts::PhpBB3
password: @settings.import_passwords ? row[:user_password] : nil,
name: @settings.username_as_name ? row[:username] : row[:name].presence,
created_at: Time.zone.at(row[:user_regdate]),
last_seen_at: row[:user_lastvisit] == 0 ? Time.zone.at(row[:user_regdate]) : Time.zone.at(row[:user_lastvisit]),
registration_ip_address: (IPAddr.new(row[:user_ip]) rescue nil),
last_seen_at:
(
if row[:user_lastvisit] == 0
Time.zone.at(row[:user_regdate])
else
Time.zone.at(row[:user_lastvisit])
end
),
registration_ip_address:
(
begin
IPAddr.new(row[:user_ip])
rescue StandardError
nil
end
),
active: is_active_user,
trust_level: trust_level,
manual_locked_trust_level: manual_locked_trust_level,
@ -43,10 +57,11 @@ module ImportScripts::PhpBB3
location: row[:user_from],
date_of_birth: parse_birthdate(row),
custom_fields: custom_fields(row),
post_create_action: proc do |user|
suspend_user(user, row)
@avatar_importer.import_avatar(user, row) if row[:user_avatar_type].present?
end
post_create_action:
proc do |user|
suspend_user(user, row)
@avatar_importer.import_avatar(user, row) if row[:user_avatar_type].present?
end,
}
end
@ -61,18 +76,19 @@ module ImportScripts::PhpBB3
id: @settings.prefix(username),
email: "anonymous_#{SecureRandom.hex}@no-email.invalid",
username: username,
name: @settings.username_as_name ? username : '',
name: @settings.username_as_name ? username : "",
created_at: Time.zone.at(row[:first_post_time]),
active: true,
trust_level: TrustLevel[0],
approved: true,
approved_by_id: Discourse.system_user.id,
approved_at: Time.now,
post_create_action: proc do |user|
row[:user_inactive_reason] = Constants::INACTIVE_MANUAL
row[:ban_reason] = 'Anonymous user from phpBB3' # TODO i18n
suspend_user(user, row, true)
end
post_create_action:
proc do |user|
row[:user_inactive_reason] = Constants::INACTIVE_MANUAL
row[:ban_reason] = "Anonymous user from phpBB3" # TODO i18n
suspend_user(user, row, true)
end,
}
end
@ -80,25 +96,32 @@ module ImportScripts::PhpBB3
def parse_birthdate(row)
return nil if row[:user_birthday].blank?
birthdate = Date.strptime(row[:user_birthday].delete(' '), '%d-%m-%Y') rescue nil
birthdate =
begin
Date.strptime(row[:user_birthday].delete(" "), "%d-%m-%Y")
rescue StandardError
nil
end
birthdate && birthdate.year > 0 ? birthdate : nil
end
def user_fields
@user_fields ||= begin
Hash[UserField.all.map { |field| [field.name, field] }]
end
@user_fields ||=
begin
Hash[UserField.all.map { |field| [field.name, field] }]
end
end
def field_mappings
@field_mappings ||= begin
@settings.custom_fields.map do |field|
{
phpbb_field_name: "pf_#{field[:phpbb_field_name]}".to_sym,
discourse_user_field: user_fields[field[:discourse_field_name]]
}
@field_mappings ||=
begin
@settings.custom_fields.map do |field|
{
phpbb_field_name: "pf_#{field[:phpbb_field_name]}".to_sym,
discourse_user_field: user_fields[field[:discourse_field_name]],
}
end
end
end
end
def custom_fields(row)
@ -114,7 +137,8 @@ module ImportScripts::PhpBB3
when "confirm"
value = value == 1 ? true : nil
when "dropdown"
value = user_field.user_field_options.find { |option| option.value == value } ? value : nil
value =
user_field.user_field_options.find { |option| option.value == value } ? value : nil
end
custom_fields["user_field_#{user_field.id}"] = value if value.present?
@ -128,7 +152,8 @@ module ImportScripts::PhpBB3
if row[:user_inactive_reason] == Constants::INACTIVE_MANUAL
user.suspended_at = Time.now
user.suspended_till = 200.years.from_now
ban_reason = row[:ban_reason].blank? ? 'Account deactivated by administrator' : row[:ban_reason] # TODO i18n
ban_reason =
row[:ban_reason].blank? ? "Account deactivated by administrator" : row[:ban_reason] # TODO i18n
elsif row[:ban_start].present?
user.suspended_at = Time.zone.at(row[:ban_start])
user.suspended_till = row[:ban_end] > 0 ? Time.zone.at(row[:ban_end]) : 200.years.from_now
@ -148,7 +173,9 @@ module ImportScripts::PhpBB3
if user.save
StaffActionLogger.new(Discourse.system_user).log_user_suspend(user, ban_reason)
else
Rails.logger.error("Failed to suspend user #{user.username}. #{user.errors.try(:full_messages).try(:inspect)}")
Rails.logger.error(
"Failed to suspend user #{user.username}. #{user.errors.try(:full_messages).try(:inspect)}",
)
end
end
end

6
script/import_scripts/phpbb3/support/bbcode/markdown_node.rb
View File

@ -1,7 +1,9 @@
# frozen_string_literal: true
module ImportScripts; end
module ImportScripts::PhpBB3; end
module ImportScripts
end
module ImportScripts::PhpBB3
end
module ImportScripts::PhpBB3::BBCode
LINEBREAK_AUTO = :auto

107
script/import_scripts/phpbb3/support/bbcode/xml_to_markdown.rb
View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
require 'nokogiri'
require_relative 'markdown_node'
require "nokogiri"
require_relative "markdown_node"
module ImportScripts::PhpBB3::BBCode
class XmlToMarkdown
@ -14,7 +14,7 @@ module ImportScripts::PhpBB3::BBCode
@allow_inline_code = opts.fetch(:allow_inline_code, false)
@traditional_linebreaks = opts.fetch(:traditional_linebreaks, false)
@doc = Nokogiri::XML(xml)
@doc = Nokogiri.XML(xml)
@list_stack = []
end
@ -28,9 +28,9 @@ module ImportScripts::PhpBB3::BBCode
private
IGNORED_ELEMENTS = ["s", "e", "i"]
ELEMENTS_WITHOUT_LEADING_WHITESPACES = ["LIST", "LI"]
ELEMENTS_WITH_HARD_LINEBREAKS = ["B", "I", "U"]
IGNORED_ELEMENTS = %w[s e i]
ELEMENTS_WITHOUT_LEADING_WHITESPACES = %w[LIST LI]
ELEMENTS_WITH_HARD_LINEBREAKS = %w[B I U]
EXPLICIT_LINEBREAK_THRESHOLD = 2
def preprocess_xml
@ -65,9 +65,7 @@ module ImportScripts::PhpBB3::BBCode
xml_node.children.each { |xml_child| visit(xml_child, md_node || md_parent) }
after_hook = "after_#{xml_node.name}"
if respond_to?(after_hook, include_all: true)
send(after_hook, xml_node, md_node)
end
send(after_hook, xml_node, md_node) if respond_to?(after_hook, include_all: true)
end
def create_node(xml_node, md_parent)
@ -84,19 +82,15 @@ module ImportScripts::PhpBB3::BBCode
end
def visit_B(xml_node, md_node)
if xml_node.parent&.name != 'B'
md_node.enclosed_with = "**"
end
md_node.enclosed_with = "**" if xml_node.parent&.name != "B"
end
def visit_I(xml_node, md_node)
if xml_node.parent&.name != 'I'
md_node.enclosed_with = "_"
end
md_node.enclosed_with = "_" if xml_node.parent&.name != "I"
end
def visit_U(xml_node, md_node)
if xml_node.parent&.name != 'U'
if xml_node.parent&.name != "U"
md_node.prefix = "[u]"
md_node.postfix = "[/u]"
end
@ -122,10 +116,7 @@ module ImportScripts::PhpBB3::BBCode
md_node.prefix_linebreaks = md_node.postfix_linebreaks = @list_stack.size == 0 ? 2 : 1
md_node.prefix_linebreak_type = LINEBREAK_HTML if @list_stack.size == 0
@list_stack << {
unordered: xml_node.attribute('type').nil?,
item_count: 0
}
@list_stack << { unordered: xml_node.attribute("type").nil?, item_count: 0 }
end
def after_LIST(xml_node, md_node)
@ -138,21 +129,21 @@ module ImportScripts::PhpBB3::BBCode
list[:item_count] += 1
indentation = ' ' * 2 * depth
symbol = list[:unordered] ? '*' : "#{list[:item_count]}."
indentation = " " * 2 * depth
symbol = list[:unordered] ? "*" : "#{list[:item_count]}."
md_node.prefix = "#{indentation}#{symbol} "
md_node.postfix_linebreaks = 1
end
def visit_IMG(xml_node, md_node)
md_node.text = +"![](#{xml_node.attribute('src')})"
md_node.text = +"![](#{xml_node.attribute("src")})"
md_node.prefix_linebreaks = md_node.postfix_linebreaks = 2
md_node.skip_children
end
def visit_URL(xml_node, md_node)
original_url = xml_node.attribute('url').to_s
original_url = xml_node.attribute("url").to_s
url = CGI.unescapeHTML(original_url)
url = @url_replacement.call(url) if @url_replacement
@ -173,7 +164,8 @@ module ImportScripts::PhpBB3::BBCode
def visit_br(xml_node, md_node)
md_node.postfix_linebreaks += 1
if md_node.postfix_linebreaks > 1 && ELEMENTS_WITH_HARD_LINEBREAKS.include?(xml_node.parent&.name)
if md_node.postfix_linebreaks > 1 &&
ELEMENTS_WITH_HARD_LINEBREAKS.include?(xml_node.parent&.name)
md_node.postfix_linebreak_type = LINEBREAK_HARD
end
end
@ -194,7 +186,8 @@ module ImportScripts::PhpBB3::BBCode
def visit_QUOTE(xml_node, md_node)
if post = quoted_post(xml_node)
md_node.prefix = %Q{[quote="#{post[:username]}, post:#{post[:post_number]}, topic:#{post[:topic_id]}"]\n}
md_node.prefix =
%Q{[quote="#{post[:username]}, post:#{post[:post_number]}, topic:#{post[:topic_id]}"]\n}
md_node.postfix = "\n[/quote]"
elsif username = quoted_username(xml_node)
md_node.prefix = %Q{[quote="#{username}"]\n}
@ -242,11 +235,11 @@ module ImportScripts::PhpBB3::BBCode
return if size.nil?
if size.between?(1, 99)
md_node.prefix = '<small>'
md_node.postfix = '</small>'
md_node.prefix = "<small>"
md_node.postfix = "</small>"
elsif size.between?(101, 200)
md_node.prefix = '<big>'
md_node.postfix = '</big>'
md_node.prefix = "<big>"
md_node.postfix = "</big>"
end
end
@ -267,7 +260,8 @@ module ImportScripts::PhpBB3::BBCode
parent_prefix = prefix_from_parent(md_parent)
if parent_prefix && md_node.xml_node_name != "br" && (md_parent.prefix_children || !markdown.empty?)
if parent_prefix && md_node.xml_node_name != "br" &&
(md_parent.prefix_children || !markdown.empty?)
prefix = "#{parent_prefix}#{prefix}"
end
@ -275,11 +269,21 @@ module ImportScripts::PhpBB3::BBCode
text, prefix, postfix = hoist_whitespaces!(markdown, text, prefix, postfix)
end
add_linebreaks!(markdown, md_node.prefix_linebreaks, md_node.prefix_linebreak_type, parent_prefix)
add_linebreaks!(
markdown,
md_node.prefix_linebreaks,
md_node.prefix_linebreak_type,
parent_prefix,
)
markdown << prefix
markdown << text
markdown << postfix
add_linebreaks!(markdown, md_node.postfix_linebreaks, md_node.postfix_linebreak_type, parent_prefix)
add_linebreaks!(
markdown,
md_node.postfix_linebreaks,
md_node.postfix_linebreak_type,
parent_prefix,
)
end
markdown
@ -296,9 +300,7 @@ module ImportScripts::PhpBB3::BBCode
end
unless postfix.empty?
if ends_with_whitespace?(text)
postfix = "#{postfix}#{text[-1]}"
end
postfix = "#{postfix}#{text[-1]}" if ends_with_whitespace?(text)
text = text.rstrip
end
@ -319,16 +321,24 @@ module ImportScripts::PhpBB3::BBCode
if linebreak_type == LINEBREAK_HTML
max_linebreak_count = [existing_linebreak_count, required_linebreak_count - 1].max + 1
required_linebreak_count = max_linebreak_count if max_linebreak_count > EXPLICIT_LINEBREAK_THRESHOLD
required_linebreak_count = max_linebreak_count if max_linebreak_count >
EXPLICIT_LINEBREAK_THRESHOLD
end
return if existing_linebreak_count >= required_linebreak_count
rstrip!(markdown)
alternative_linebreak_start_index = required_linebreak_count > EXPLICIT_LINEBREAK_THRESHOLD ? 1 : 2
alternative_linebreak_start_index =
required_linebreak_count > EXPLICIT_LINEBREAK_THRESHOLD ? 1 : 2
required_linebreak_count.times do |index|
linebreak = linebreak(linebreak_type, index, alternative_linebreak_start_index, required_linebreak_count)
linebreak =
linebreak(
linebreak_type,
index,
alternative_linebreak_start_index,
required_linebreak_count,
)
markdown << (linebreak == "\n" ? prefix.rstrip : prefix) if prefix && index > 0
markdown << linebreak
@ -336,18 +346,25 @@ module ImportScripts::PhpBB3::BBCode
end
def rstrip!(markdown)
markdown.gsub!(/\s*(?:\\?\n|<br>\n)*\z/, '')
markdown.gsub!(/\s*(?:\\?\n|<br>\n)*\z/, "")
end
def linebreak(linebreak_type, linebreak_index, alternative_linebreak_start_index, required_linebreak_count)
def linebreak(
linebreak_type,
linebreak_index,
alternative_linebreak_start_index,
required_linebreak_count
)
use_alternative_linebreak = linebreak_index >= alternative_linebreak_start_index
is_last_linebreak = linebreak_index + 1 == required_linebreak_count
return "<br>\n" if linebreak_type == LINEBREAK_HTML &&
use_alternative_linebreak && is_last_linebreak
if linebreak_type == LINEBREAK_HTML && use_alternative_linebreak && is_last_linebreak
return "<br>\n"
end
return "\\\n" if linebreak_type == LINEBREAK_HARD ||
@traditional_linebreaks || use_alternative_linebreak
if linebreak_type == LINEBREAK_HARD || @traditional_linebreaks || use_alternative_linebreak
return "\\\n"
end
"\n"
end

10
script/import_scripts/phpbb3/support/constants.rb
View File

@ -8,8 +8,8 @@ module ImportScripts::PhpBB3
INACTIVE_MANUAL = 3 # Account deactivated by administrator
INACTIVE_REMIND = 4 # Forced user account reactivation
GROUP_ADMINISTRATORS = 'ADMINISTRATORS'
GROUP_MODERATORS = 'GLOBAL_MODERATORS'
GROUP_ADMINISTRATORS = "ADMINISTRATORS"
GROUP_MODERATORS = "GLOBAL_MODERATORS"
# https://wiki.phpbb.com/Table.phpbb_users
USER_TYPE_NORMAL = 0
@ -21,9 +21,9 @@ module ImportScripts::PhpBB3
AVATAR_TYPE_REMOTE = 2
AVATAR_TYPE_GALLERY = 3
AVATAR_TYPE_STRING_UPLOADED = 'avatar.driver.upload'
AVATAR_TYPE_STRING_REMOTE = 'avatar.driver.remote'
AVATAR_TYPE_STRING_GALLERY = 'avatar.driver.local'
AVATAR_TYPE_STRING_UPLOADED = "avatar.driver.upload"
AVATAR_TYPE_STRING_REMOTE = "avatar.driver.remote"
AVATAR_TYPE_STRING_GALLERY = "avatar.driver.local"
FORUM_TYPE_CATEGORY = 0
FORUM_TYPE_POST = 1

87
script/import_scripts/phpbb3/support/settings.rb
View File

@ -1,13 +1,13 @@
# frozen_string_literal: true
require 'csv'
require 'yaml'
require_relative '../../base'
require "csv"
require "yaml"
require_relative "../../base"
module ImportScripts::PhpBB3
class Settings
def self.load(filename)
yaml = YAML::load_file(filename)
yaml = YAML.load_file(filename)
Settings.new(yaml.deep_stringify_keys.with_indifferent_access)
end
@ -44,40 +44,41 @@ module ImportScripts::PhpBB3
attr_reader :database
def initialize(yaml)
import_settings = yaml['import']
import_settings = yaml["import"]
@site_name = import_settings['site_name']
@site_name = import_settings["site_name"]
@new_categories = import_settings['new_categories']
@category_mappings = import_settings.fetch('category_mappings', []).to_h { |m| [m[:source_category_id].to_s, m] }
@tag_mappings = import_settings['tag_mappings']
@rank_mapping = import_settings['rank_mapping']
@new_categories = import_settings["new_categories"]
@category_mappings =
import_settings.fetch("category_mappings", []).to_h { |m| [m[:source_category_id].to_s, m] }
@tag_mappings = import_settings["tag_mappings"]
@rank_mapping = import_settings["rank_mapping"]
@import_anonymous_users = import_settings['anonymous_users']
@import_attachments = import_settings['attachments']
@import_private_messages = import_settings['private_messages']
@import_polls = import_settings['polls']
@import_bookmarks = import_settings['bookmarks']
@import_passwords = import_settings['passwords']
@import_likes = import_settings['likes']
@import_anonymous_users = import_settings["anonymous_users"]
@import_attachments = import_settings["attachments"]
@import_private_messages = import_settings["private_messages"]
@import_polls = import_settings["polls"]
@import_bookmarks = import_settings["bookmarks"]
@import_passwords = import_settings["passwords"]
@import_likes = import_settings["likes"]
avatar_settings = import_settings['avatars']
@import_uploaded_avatars = avatar_settings['uploaded']
@import_remote_avatars = avatar_settings['remote']
@import_gallery_avatars = avatar_settings['gallery']
avatar_settings = import_settings["avatars"]
@import_uploaded_avatars = avatar_settings["uploaded"]
@import_remote_avatars = avatar_settings["remote"]
@import_gallery_avatars = avatar_settings["gallery"]
@use_bbcode_to_md = import_settings['use_bbcode_to_md']
@use_bbcode_to_md = import_settings["use_bbcode_to_md"]
@original_site_prefix = import_settings['site_prefix']['original']
@new_site_prefix = import_settings['site_prefix']['new']
@base_dir = import_settings['phpbb_base_dir']
@permalinks = PermalinkSettings.new(import_settings['permalinks'])
@original_site_prefix = import_settings["site_prefix"]["original"]
@new_site_prefix = import_settings["site_prefix"]["new"]
@base_dir = import_settings["phpbb_base_dir"]
@permalinks = PermalinkSettings.new(import_settings["permalinks"])
@username_as_name = import_settings['username_as_name']
@emojis = import_settings.fetch('emojis', [])
@custom_fields = import_settings.fetch('custom_fields', [])
@username_as_name = import_settings["username_as_name"]
@emojis = import_settings.fetch("emojis", [])
@custom_fields = import_settings.fetch("custom_fields", [])
@database = DatabaseSettings.new(yaml['database'])
@database = DatabaseSettings.new(yaml["database"])
end
def prefix(val)
@ -87,7 +88,7 @@ module ImportScripts::PhpBB3
def trust_level_for_posts(rank, trust_level: 0)
if @rank_mapping.present?
@rank_mapping.each do |key, value|
trust_level = [trust_level, key.gsub('trust_level_', '').to_i].max if rank >= value
trust_level = [trust_level, key.gsub("trust_level_", "").to_i].max if rank >= value
end
end
@ -106,14 +107,14 @@ module ImportScripts::PhpBB3
attr_reader :batch_size
def initialize(yaml)
@type = yaml['type']
@host = yaml['host']
@port = yaml['port']
@username = yaml['username']
@password = yaml['password']
@schema = yaml['schema']
@table_prefix = yaml['table_prefix']
@batch_size = yaml['batch_size']
@type = yaml["type"]
@host = yaml["host"]
@port = yaml["port"]
@username = yaml["username"]
@password = yaml["password"]
@schema = yaml["schema"]
@table_prefix = yaml["table_prefix"]
@batch_size = yaml["batch_size"]
end
end
@ -124,10 +125,10 @@ module ImportScripts::PhpBB3
attr_reader :normalization_prefix
def initialize(yaml)
@create_category_links = yaml['categories']
@create_topic_links = yaml['topics']
@create_post_links = yaml['posts']
@normalization_prefix = yaml['prefix']
@create_category_links = yaml["categories"]
@create_topic_links = yaml["topics"]
@create_post_links = yaml["posts"]
@normalization_prefix = yaml["prefix"]
end
end
end

60
script/import_scripts/phpbb3/support/smiley_processor.rb
View File

@ -18,15 +18,16 @@ module ImportScripts::PhpBB3
def replace_smilies(text)
# :) is encoded as <!-- s:) --><img src="{SMILIES_PATH}/icon_e_smile.gif" alt=":)" title="Smile" /><!-- s:) -->
text.gsub!(/<!-- s(\S+) --><img src="\{SMILIES_PATH\}\/.+?" alt=".*?" title=".*?" \/><!-- s?\S+ -->/) do
emoji($1)
end
text.gsub!(
/<!-- s(\S+) --><img src="\{SMILIES_PATH\}\/.+?" alt=".*?" title=".*?" \/><!-- s?\S+ -->/,
) { emoji($1) }
end
def emoji(smiley_code)
@smiley_map.fetch(smiley_code) do
smiley = @database.get_smiley(smiley_code)
emoji = upload_smiley(smiley_code, smiley[:smiley_url], smiley_code, smiley[:emotion]) if smiley
emoji =
upload_smiley(smiley_code, smiley[:smiley_url], smiley_code, smiley[:emotion]) if smiley
emoji || smiley_as_text(smiley_code)
end
end
@ -35,37 +36,34 @@ module ImportScripts::PhpBB3
def add_default_smilies
{
[':D', ':-D', ':grin:'] => ':smiley:',
[':)', ':-)', ':smile:'] => ':slight_smile:',
[';)', ';-)', ':wink:'] => ':wink:',
[':(', ':-(', ':sad:'] => ':frowning:',
[':o', ':-o', ':eek:'] => ':astonished:',
[':shock:'] => ':open_mouth:',
[':?', ':-?', ':???:'] => ':confused:',
['8)', '8-)', ':cool:'] => ':sunglasses:',
[':lol:'] => ':laughing:',
[':x', ':-x', ':mad:'] => ':angry:',
[':P', ':-P', ':razz:'] => ':stuck_out_tongue:',
[':oops:'] => ':blush:',
[':cry:'] => ':cry:',
[':evil:'] => ':imp:',
[':twisted:'] => ':smiling_imp:',
[':roll:'] => ':unamused:',
[':!:'] => ':exclamation:',
[':?:'] => ':question:',
[':idea:'] => ':bulb:',
[':arrow:'] => ':arrow_right:',
[':|', ':-|'] => ':neutral_face:',
[':geek:'] => ':nerd:'
}.each do |smilies, emoji|
smilies.each { |smiley| @smiley_map[smiley] = emoji }
end
%w[:D :-D :grin:] => ":smiley:",
%w[:) :-) :smile:] => ":slight_smile:",
%w[;) ;-) :wink:] => ":wink:",
%w[:( :-( :sad:] => ":frowning:",
%w[:o :-o :eek:] => ":astonished:",
[":shock:"] => ":open_mouth:",
%w[:? :-? :???:] => ":confused:",
%w[8) 8-) :cool:] => ":sunglasses:",
[":lol:"] => ":laughing:",
%w[:x :-x :mad:] => ":angry:",
%w[:P :-P :razz:] => ":stuck_out_tongue:",
[":oops:"] => ":blush:",
[":cry:"] => ":cry:",
[":evil:"] => ":imp:",
[":twisted:"] => ":smiling_imp:",
[":roll:"] => ":unamused:",
[":!:"] => ":exclamation:",
[":?:"] => ":question:",
[":idea:"] => ":bulb:",
[":arrow:"] => ":arrow_right:",
%w[:| :-|] => ":neutral_face:",
[":geek:"] => ":nerd:",
}.each { |smilies, emoji| smilies.each { |smiley| @smiley_map[smiley] = emoji } }
end
def add_configured_smilies(emojis)
emojis.each do |emoji, smilies|
Array.wrap(smilies)
.each { |smiley| @smiley_map[smiley] = ":#{emoji}:" }
Array.wrap(smilies).each { |smiley| @smiley_map[smiley] = ":#{emoji}:" }
end
end

103
script/import_scripts/phpbb3/support/text_processor.rb
View File

@ -1,6 +1,6 @@
# frozen_string_literal: true
require_relative 'bbcode/xml_to_markdown'
require_relative "bbcode/xml_to_markdown"
module ImportScripts::PhpBB3
class TextProcessor
@ -14,7 +14,9 @@ module ImportScripts::PhpBB3
@database = database
@smiley_processor = smiley_processor
@he = HTMLEntities.new
@use_xml_to_markdown = phpbb_config[:phpbb_version].start_with?('3.2') || phpbb_config[:phpbb_version].start_with?('3.3')
@use_xml_to_markdown =
phpbb_config[:phpbb_version].start_with?("3.2") ||
phpbb_config[:phpbb_version].start_with?("3.3")
@settings = settings
@new_site_prefix = settings.new_site_prefix
@ -25,24 +27,27 @@ module ImportScripts::PhpBB3
if @use_xml_to_markdown
unreferenced_attachments = attachments&.dup
converter = BBCode::XmlToMarkdown.new(
raw,
username_from_user_id: lambda { |user_id| @lookup.find_username_by_import_id(user_id) },
smilie_to_emoji: lambda { |smilie| @smiley_processor.emoji(smilie).dup },
quoted_post_from_post_id: lambda { |post_id| @lookup.topic_lookup_from_imported_post_id(post_id) },
upload_md_from_file: (lambda do |filename, index|
unreferenced_attachments[index] = nil
attachments.fetch(index, filename).dup
end if attachments),
url_replacement: nil,
allow_inline_code: false
)
converter =
BBCode::XmlToMarkdown.new(
raw,
username_from_user_id: lambda { |user_id| @lookup.find_username_by_import_id(user_id) },
smilie_to_emoji: lambda { |smilie| @smiley_processor.emoji(smilie).dup },
quoted_post_from_post_id:
lambda { |post_id| @lookup.topic_lookup_from_imported_post_id(post_id) },
upload_md_from_file:
(
lambda do |filename, index|
unreferenced_attachments[index] = nil
attachments.fetch(index, filename).dup
end if attachments
),
url_replacement: nil,
allow_inline_code: false,
)
text = converter.convert
text.gsub!(@short_internal_link_regexp) do |link|
replace_internal_link(link, $1, $2)
end
text.gsub!(@short_internal_link_regexp) { |link| replace_internal_link(link, $1, $2) }
add_unreferenced_attachments(text, unreferenced_attachments)
else
@ -50,9 +55,7 @@ module ImportScripts::PhpBB3
text = CGI.unescapeHTML(text)
clean_bbcodes(text)
if @settings.use_bbcode_to_md
text = bbcode_to_md(text)
end
text = bbcode_to_md(text) if @settings.use_bbcode_to_md
process_smilies(text)
process_links(text)
process_lists(text)
@ -65,11 +68,19 @@ module ImportScripts::PhpBB3
end
def process_post(raw, attachments)
process_raw_text(raw, attachments) rescue raw
begin
process_raw_text(raw, attachments)
rescue StandardError
raw
end
end
def process_private_msg(raw, attachments)
process_raw_text(raw, attachments) rescue raw
begin
process_raw_text(raw, attachments)
rescue StandardError
raw
end
end
protected
@ -78,10 +89,10 @@ module ImportScripts::PhpBB3
# Many phpbb bbcode tags have a hash attached to them. Examples:
# [url=https&#58;//google&#46;com:1qh1i7ky]click here[/url:1qh1i7ky]
# [quote=&quot;cybereality&quot;:b0wtlzex]Some text.[/quote:b0wtlzex]
text.gsub!(/:(?:\w{5,8})\]/, ']')
text.gsub!(/:(?:\w{5,8})\]/, "]")
# remove color tags
text.gsub!(/\[\/?color(=#?[a-z0-9]*)?\]/i, "")
text.gsub!(%r{\[/?color(=#?[a-z0-9]*)?\]}i, "")
end
def bbcode_to_md(text)
@ -101,23 +112,19 @@ module ImportScripts::PhpBB3
# Internal forum links can have this forms:
# for topics: <!-- l --><a class="postlink-local" href="https://example.com/forums/viewtopic.php?f=26&amp;t=3412">viewtopic.php?f=26&amp;t=3412</a><!-- l -->
# for posts: <!-- l --><a class="postlink-local" href="https://example.com/forums/viewtopic.php?p=1732#p1732">viewtopic.php?p=1732#p1732</a><!-- l -->
text.gsub!(@long_internal_link_regexp) do |link|
replace_internal_link(link, $1, $2)
end
text.gsub!(@long_internal_link_regexp) { |link| replace_internal_link(link, $1, $2) }
# Some links look like this: <!-- m --><a class="postlink" href="http://www.onegameamonth.com">http://www.onegameamonth.com</a><!-- m -->
text.gsub!(/<!-- \w --><a(?:.+)href="(\S+)"(?:.*)>(.+)<\/a><!-- \w -->/i, '[\2](\1)')
text.gsub!(%r{<!-- \w --><a(?:.+)href="(\S+)"(?:.*)>(.+)</a><!-- \w -->}i, '[\2](\1)')
# Replace internal forum links that aren't in the <!-- l --> format
text.gsub!(@short_internal_link_regexp) do |link|
replace_internal_link(link, $1, $2)
end
text.gsub!(@short_internal_link_regexp) { |link| replace_internal_link(link, $1, $2) }
# phpBB shortens link text like this, which breaks our markdown processing:
# [http://answers.yahoo.com/question/index ... 223AAkkPli](http://answers.yahoo.com/question/index?qid=20070920134223AAkkPli)
#
# Work around it for now:
text.gsub!(/\[http(s)?:\/\/(www\.)?/i, '[')
text.gsub!(%r{\[http(s)?://(www\.)?}i, "[")
end
def replace_internal_link(link, import_topic_id, import_post_id)
@ -144,19 +151,20 @@ module ImportScripts::PhpBB3
# convert list tags to ul and list=1 tags to ol
# list=a is not supported, so handle it like list=1
# list=9 and list=x have the same result as list=1 and list=a
text.gsub!(/\[list\](.*?)\[\/list:u\]/mi) do
$1.gsub(/\[\*\](.*?)\[\/\*:m\]\n*/mi) { "* #{$1}\n" }
text.gsub!(%r{\[list\](.*?)\[/list:u\]}mi) do
$1.gsub(%r{\[\*\](.*?)\[/\*:m\]\n*}mi) { "* #{$1}\n" }
end
text.gsub!(/\[list=.*?\](.*?)\[\/list:o\]/mi) do
$1.gsub(/\[\*\](.*?)\[\/\*:m\]\n*/mi) { "1. #{$1}\n" }
text.gsub!(%r{\[list=.*?\](.*?)\[/list:o\]}mi) do
$1.gsub(%r{\[\*\](.*?)\[/\*:m\]\n*}mi) { "1. #{$1}\n" }
end
end
# This replaces existing [attachment] BBCodes with the corresponding HTML tags for Discourse.
# All attachments that haven't been referenced in the text are appended to the end of the text.
def process_attachments(text, attachments)
attachment_regexp = /\[attachment=([\d])+\]<!-- [\w]+ -->([^<]+)<!-- [\w]+ -->\[\/attachment\]?/i
attachment_regexp =
%r{\[attachment=([\d])+\]<!-- [\w]+ -->([^<]+)<!-- [\w]+ -->\[/attachment\]?}i
unreferenced_attachments = attachments.dup
text.gsub!(attachment_regexp) do
@ -178,29 +186,34 @@ module ImportScripts::PhpBB3
end
def create_internal_link_regexps(original_site_prefix)
host = original_site_prefix.gsub('.', '\.')
link_regex = "http(?:s)?://#{host}/viewtopic\\.php\\?(?:\\S*)(?:t=(\\d+)|p=(\\d+)(?:#p\\d+)?)(?:[^\\s\\)\\]]*)"
host = original_site_prefix.gsub(".", '\.')
link_regex =
"http(?:s)?://#{host}/viewtopic\\.php\\?(?:\\S*)(?:t=(\\d+)|p=(\\d+)(?:#p\\d+)?)(?:[^\\s\\)\\]]*)"
@long_internal_link_regexp = Regexp.new(%Q|<!-- l --><a(?:.+)href="#{link_regex}"(?:.*)</a><!-- l -->|, Regexp::IGNORECASE)
@long_internal_link_regexp =
Regexp.new(
%Q|<!-- l --><a(?:.+)href="#{link_regex}"(?:.*)</a><!-- l -->|,
Regexp::IGNORECASE,
)
@short_internal_link_regexp = Regexp.new(link_regex, Regexp::IGNORECASE)
end
def process_code(text)
text.gsub!(/<span class="syntax.*?>(.*?)<\/span>/) { "#{$1}" }
text.gsub!(/\[code(=[a-z]*)?\](.*?)\[\/code\]/i) { "[code]\n#{@he.decode($2)}\n[/code]" }
text.gsub!(/<br \/>/, "\n")
text.gsub!(%r{<span class="syntax.*?>(.*?)</span>}) { "#{$1}" }
text.gsub!(%r{\[code(=[a-z]*)?\](.*?)\[/code\]}i) { "[code]\n#{@he.decode($2)}\n[/code]" }
text.gsub!(%r{<br />}, "\n")
text
end
def fix_markdown(text)
text.gsub!(/(\n*\[\/?quote.*?\]\n*)/mi) { |q| "\n#{q.strip}\n" }
text.gsub!(%r{(\n*\[/?quote.*?\]\n*)}mi) { |q| "\n#{q.strip}\n" }
text.gsub!(/^!\[[^\]]*\]\([^\]]*\)$/i) { |img| "\n#{img.strip}\n" } # space out images single on line
text
end
def process_videos(text)
# [YOUTUBE]<id>[/YOUTUBE]
text.gsub(/\[youtube\](.+?)\[\/youtube\]/i) { "\nhttps://www.youtube.com/watch?v=#{$1}\n" }
text.gsub(%r{\[youtube\](.+?)\[/youtube\]}i) { "\nhttps://www.youtube.com/watch?v=#{$1}\n" }
text
end
end