github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-22 07:53:49 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: Apply syntax_tree formatting to script/*

Browse Source
This commit is contained in:
David Taylor
2023-01-07 11:53:14 +00:00
parent ff508d1ae5
commit 436b3b392b
143 changed files with 8905 additions and 7353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
script/import_scripts/bbpress.rb
View File

@ -1,29 +1,29 @@
# frozen_string_literal: true
require 'mysql2'
require "mysql2"
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
class ImportScripts::Bbpress < ImportScripts::Base
BB_PRESS_HOST ||= ENV['BBPRESS_HOST'] || "localhost"
BB_PRESS_DB ||= ENV['BBPRESS_DB'] || "bbpress"
BATCH_SIZE ||= 1000
BB_PRESS_PW ||= ENV['BBPRESS_PW'] || ""
BB_PRESS_USER ||= ENV['BBPRESS_USER'] || "root"
BB_PRESS_PREFIX ||= ENV['BBPRESS_PREFIX'] || "wp_"
BB_PRESS_ATTACHMENTS_DIR ||= ENV['BBPRESS_ATTACHMENTS_DIR'] || "/path/to/attachments"
BB_PRESS_HOST ||= ENV["BBPRESS_HOST"] || "localhost"
BB_PRESS_DB ||= ENV["BBPRESS_DB"] || "bbpress"
BATCH_SIZE ||= 1000
BB_PRESS_PW ||= ENV["BBPRESS_PW"] || ""
BB_PRESS_USER ||= ENV["BBPRESS_USER"] || "root"
BB_PRESS_PREFIX ||= ENV["BBPRESS_PREFIX"] || "wp_"
BB_PRESS_ATTACHMENTS_DIR ||= ENV["BBPRESS_ATTACHMENTS_DIR"] || "/path/to/attachments"
def initialize
super
@he = HTMLEntities.new
@client = Mysql2::Client.new(
host: BB_PRESS_HOST,
username: BB_PRESS_USER,
database: BB_PRESS_DB,
password: BB_PRESS_PW,
)
@client =
Mysql2::Client.new(
host: BB_PRESS_HOST,
username: BB_PRESS_USER,
database: BB_PRESS_DB,
password: BB_PRESS_PW,
)
end
def execute
@ -40,17 +40,16 @@ class ImportScripts::Bbpress < ImportScripts::Base
puts "", "importing users..."
last_user_id = -1
total_users = bbpress_query(<<-SQL
total_users = bbpress_query(<<-SQL).first["cnt"]
SELECT COUNT(DISTINCT(u.id)) AS cnt
FROM #{BB_PRESS_PREFIX}users u
LEFT JOIN #{BB_PRESS_PREFIX}posts p ON p.post_author = u.id
WHERE p.post_type IN ('forum', 'reply', 'topic')
AND user_email LIKE '%@%'
SQL
).first["cnt"]
batches(BATCH_SIZE) do |offset|
users = bbpress_query(<<-SQL
users = bbpress_query(<<-SQL).to_a
SELECT u.id, user_nicename, display_name, user_email, user_registered, user_url, user_pass
FROM #{BB_PRESS_PREFIX}users u
LEFT JOIN #{BB_PRESS_PREFIX}posts p ON p.post_author = u.id
@ -61,7 +60,6 @@ class ImportScripts::Bbpress < ImportScripts::Base
ORDER BY u.id
LIMIT #{BATCH_SIZE}
SQL
).to_a
break if users.empty?
@ -73,22 +71,20 @@ class ImportScripts::Bbpress < ImportScripts::Base
user_ids_sql = user_ids.join(",")
users_description = {}
bbpress_query(<<-SQL
bbpress_query(<<-SQL).each { |um| users_description[um["user_id"]] = um["description"] }
SELECT user_id, meta_value description
FROM #{BB_PRESS_PREFIX}usermeta
WHERE user_id IN (#{user_ids_sql})
AND meta_key = 'description'
SQL
).each { |um| users_description[um["user_id"]] = um["description"] }
users_last_activity = {}
bbpress_query(<<-SQL
bbpress_query(<<-SQL).each { |um| users_last_activity[um["user_id"]] = um["last_activity"] }
SELECT user_id, meta_value last_activity
FROM #{BB_PRESS_PREFIX}usermeta
WHERE user_id IN (#{user_ids_sql})
AND meta_key = 'last_activity'
SQL
).each { |um| users_last_activity[um["user_id"]] = um["last_activity"] }
create_users(users, total: total_users, offset: offset) do |u|
{
@ -96,7 +92,7 @@ class ImportScripts::Bbpress < ImportScripts::Base
username: u["user_nicename"],
password: u["user_pass"],
email: u["user_email"].downcase,
name: u["display_name"].presence || u['user_nicename'],
name: u["display_name"].presence || u["user_nicename"],
created_at: u["user_registered"],
website: u["user_url"],
bio_raw: users_description[u["id"]],
@ -114,67 +110,60 @@ class ImportScripts::Bbpress < ImportScripts::Base
emails = Array.new
# gather anonymous users via postmeta table
bbpress_query(<<-SQL
bbpress_query(<<-SQL).each do |pm|
SELECT post_id, meta_key, meta_value
FROM #{BB_PRESS_PREFIX}postmeta
WHERE meta_key LIKE '_bbp_anonymous%'
SQL
).each do |pm|
anon_posts[pm['post_id']] = Hash.new if not anon_posts[pm['post_id']]
anon_posts[pm["post_id"]] = Hash.new if not anon_posts[pm["post_id"]]
if pm['meta_key'] == '_bbp_anonymous_email'
anon_posts[pm['post_id']]['email'] = pm['meta_value']
if pm["meta_key"] == "_bbp_anonymous_email"
anon_posts[pm["post_id"]]["email"] = pm["meta_value"]
end
if pm['meta_key'] == '_bbp_anonymous_name'
anon_posts[pm['post_id']]['name'] = pm['meta_value']
if pm["meta_key"] == "_bbp_anonymous_name"
anon_posts[pm["post_id"]]["name"] = pm["meta_value"]
end
if pm['meta_key'] == '_bbp_anonymous_website'
anon_posts[pm['post_id']]['website'] = pm['meta_value']
if pm["meta_key"] == "_bbp_anonymous_website"
anon_posts[pm["post_id"]]["website"] = pm["meta_value"]
end
end
# gather every existent username
anon_posts.each do |id, post|
anon_names[post['name']] = Hash.new if not anon_names[post['name']]
anon_names[post["name"]] = Hash.new if not anon_names[post["name"]]
# overwriting email address, one user can only use one email address
anon_names[post['name']]['email'] = post['email']
anon_names[post['name']]['website'] = post['website'] if post['website'] != ''
anon_names[post["name"]]["email"] = post["email"]
anon_names[post["name"]]["website"] = post["website"] if post["website"] != ""
end
# make sure every user name has a unique email address
anon_names.each do |k, name|
if not emails.include? name['email']
emails.push ( name['email'])
if not emails.include? name["email"]
emails.push (name["email"])
else
name['email'] = "anonymous_#{SecureRandom.hex}@no-email.invalid"
name["email"] = "anonymous_#{SecureRandom.hex}@no-email.invalid"
end
end
create_users(anon_names) do |k, n|
{
id: k,
email: n["email"].downcase,
name: k,
website: n["website"]
}
{ id: k, email: n["email"].downcase, name: k, website: n["website"] }
end
end
def import_categories
puts "", "importing categories..."
categories = bbpress_query(<<-SQL
categories = bbpress_query(<<-SQL)
SELECT id, post_name, post_parent
FROM #{BB_PRESS_PREFIX}posts
WHERE post_type = 'forum'
AND LENGTH(COALESCE(post_name, '')) > 0
ORDER BY post_parent, id
SQL
)
create_categories(categories) do |c|
category = { id: c['id'], name: c['post_name'] }
if (parent_id = c['post_parent'].to_i) > 0
category = { id: c["id"], name: c["post_name"] }
if (parent_id = c["post_parent"].to_i) > 0
category[:parent_category_id] = category_id_from_imported_category_id(parent_id)
end
category
@ -185,16 +174,15 @@ class ImportScripts::Bbpress < ImportScripts::Base
puts "", "importing topics and posts..."
last_post_id = -1
total_posts = bbpress_query(<<-SQL
total_posts = bbpress_query(<<-SQL).first["count"]
SELECT COUNT(*) count
FROM #{BB_PRESS_PREFIX}posts
WHERE post_status <> 'spam'
AND post_type IN ('topic', 'reply')
SQL
).first["count"]
batches(BATCH_SIZE) do |offset|
posts = bbpress_query(<<-SQL
posts = bbpress_query(<<-SQL).to_a
SELECT id,
post_author,
post_date,
@ -209,7 +197,6 @@ class ImportScripts::Bbpress < ImportScripts::Base
ORDER BY id
LIMIT #{BATCH_SIZE}
SQL
).to_a
break if posts.empty?
@ -221,31 +208,29 @@ class ImportScripts::Bbpress < ImportScripts::Base
post_ids_sql = post_ids.join(",")
posts_likes = {}
bbpress_query(<<-SQL
bbpress_query(<<-SQL).each { |pm| posts_likes[pm["post_id"]] = pm["likes"].to_i }
SELECT post_id, meta_value likes
FROM #{BB_PRESS_PREFIX}postmeta
WHERE post_id IN (#{post_ids_sql})
AND meta_key = 'Likes'
SQL
).each { |pm| posts_likes[pm["post_id"]] = pm["likes"].to_i }
anon_names = {}
bbpress_query(<<-SQL
bbpress_query(<<-SQL).each { |pm| anon_names[pm["post_id"]] = pm["meta_value"] }
SELECT post_id, meta_value
FROM #{BB_PRESS_PREFIX}postmeta
WHERE post_id IN (#{post_ids_sql})
AND meta_key = '_bbp_anonymous_name'
SQL
).each { |pm| anon_names[pm["post_id"]] = pm["meta_value"] }
create_posts(posts, total: total_posts, offset: offset) do |p|
skip = false
user_id = user_id_from_imported_user_id(p["post_author"]) ||
find_user_by_import_id(p["post_author"]).try(:id) ||
user_id_from_imported_user_id(anon_names[p['id']]) ||
find_user_by_import_id(anon_names[p['id']]).try(:id) ||
-1
user_id =
user_id_from_imported_user_id(p["post_author"]) ||
find_user_by_import_id(p["post_author"]).try(:id) ||
user_id_from_imported_user_id(anon_names[p["id"]]) ||
find_user_by_import_id(anon_names[p["id"]]).try(:id) || -1
post = {
id: p["id"],
@ -256,7 +241,9 @@ class ImportScripts::Bbpress < ImportScripts::Base
}
if post[:raw].present?
post[:raw].gsub!(/\<pre\>\<code(=[a-z]*)?\>(.*?)\<\/code\>\<\/pre\>/im) { "```\n#{@he.decode($2)}\n```" }
post[:raw].gsub!(%r{\<pre\>\<code(=[a-z]*)?\>(.*?)\</code\>\</pre\>}im) do
"```\n#{@he.decode($2)}\n```"
end
end
if p["post_type"] == "topic"
@ -288,17 +275,16 @@ class ImportScripts::Bbpress < ImportScripts::Base
count = 0
last_attachment_id = -1
total_attachments = bbpress_query(<<-SQL
total_attachments = bbpress_query(<<-SQL).first["count"]
SELECT COUNT(*) count
FROM #{BB_PRESS_PREFIX}postmeta pm
JOIN #{BB_PRESS_PREFIX}posts p ON p.id = pm.post_id
WHERE pm.meta_key = '_wp_attached_file'
AND p.post_parent > 0
SQL
).first["count"]
batches(BATCH_SIZE) do |offset|
attachments = bbpress_query(<<-SQL
attachments = bbpress_query(<<-SQL).to_a
SELECT pm.meta_id id, pm.meta_value, p.post_parent post_id
FROM #{BB_PRESS_PREFIX}postmeta pm
JOIN #{BB_PRESS_PREFIX}posts p ON p.id = pm.post_id
@ -308,7 +294,6 @@ class ImportScripts::Bbpress < ImportScripts::Base
ORDER BY pm.meta_id
LIMIT #{BATCH_SIZE}
SQL
).to_a
break if attachments.empty?
last_attachment_id = attachments[-1]["id"].to_i
@ -325,7 +310,9 @@ class ImportScripts::Bbpress < ImportScripts::Base
if !post.raw[html]
post.raw << "\n\n" << html
post.save!
PostUpload.create!(post: post, upload: upload) unless PostUpload.where(post: post, upload: upload).exists?
unless PostUpload.where(post: post, upload: upload).exists?
PostUpload.create!(post: post, upload: upload)
end
end
end
end
@ -340,15 +327,14 @@ class ImportScripts::Bbpress < ImportScripts::Base
count = 0
last_attachment_id = -1
total_attachments = bbpress_query(<<-SQL
total_attachments = bbpress_query(<<-SQL).first["count"]
SELECT COUNT(*) count
FROM #{BB_PRESS_PREFIX}bb_attachments
WHERE post_id IN (SELECT id FROM #{BB_PRESS_PREFIX}posts WHERE post_status <> 'spam' AND post_type IN ('topic', 'reply'))
SQL
).first["count"]
batches(BATCH_SIZE) do |offset|
attachments = bbpress_query(<<-SQL
attachments = bbpress_query(<<-SQL).to_a
SELECT id, filename, post_id
FROM #{BB_PRESS_PREFIX}bb_attachments
WHERE post_id IN (SELECT id FROM #{BB_PRESS_PREFIX}posts WHERE post_status <> 'spam' AND post_type IN ('topic', 'reply'))
@ -356,13 +342,16 @@ class ImportScripts::Bbpress < ImportScripts::Base
ORDER BY id
LIMIT #{BATCH_SIZE}
SQL
).to_a
break if attachments.empty?
last_attachment_id = attachments[-1]["id"].to_i
attachments.each do |a|
print_status(count += 1, total_attachments, get_start_time("attachments_from_bb_attachments"))
print_status(
count += 1,
total_attachments,
get_start_time("attachments_from_bb_attachments"),
)
if path = find_attachment(a["filename"], a["id"])
if post = Post.find_by(id: post_id_from_imported_post_id(a["post_id"]))
upload = create_upload(post.user.id, path, a["filename"])
@ -371,7 +360,9 @@ class ImportScripts::Bbpress < ImportScripts::Base
if !post.raw[html]
post.raw << "\n\n" << html
post.save!
PostUpload.create!(post: post, upload: upload) unless PostUpload.where(post: post, upload: upload).exists?
unless PostUpload.where(post: post, upload: upload).exists?
PostUpload.create!(post: post, upload: upload)
end
end
end
end
@ -391,7 +382,7 @@ class ImportScripts::Bbpress < ImportScripts::Base
last_topic_id = -1
batches(BATCH_SIZE) do |offset|
topics = bbpress_query(<<-SQL
topics = bbpress_query(<<-SQL).to_a
SELECT id,
guid
FROM #{BB_PRESS_PREFIX}posts
@ -401,14 +392,17 @@ class ImportScripts::Bbpress < ImportScripts::Base
ORDER BY id
LIMIT #{BATCH_SIZE}
SQL
).to_a
break if topics.empty?
last_topic_id = topics[-1]["id"].to_i
topics.each do |t|
topic = topic_lookup_from_imported_post_id(t['id'])
Permalink.create(url: URI.parse(t['guid']).path.chomp('/'), topic_id: topic[:topic_id]) rescue nil
topic = topic_lookup_from_imported_post_id(t["id"])
begin
Permalink.create(url: URI.parse(t["guid"]).path.chomp("/"), topic_id: topic[:topic_id])
rescue StandardError
nil
end
end
end
end
@ -417,42 +411,44 @@ class ImportScripts::Bbpress < ImportScripts::Base
puts "", "importing private messages..."
last_post_id = -1
total_posts = bbpress_query("SELECT COUNT(*) count FROM #{BB_PRESS_PREFIX}bp_messages_messages").first["count"]
total_posts =
bbpress_query("SELECT COUNT(*) count FROM #{BB_PRESS_PREFIX}bp_messages_messages").first[
"count"
]
threads = {}
total_count = bbpress_query("SELECT COUNT(*) count FROM #{BB_PRESS_PREFIX}bp_messages_recipients").first["count"]
total_count =
bbpress_query("SELECT COUNT(*) count FROM #{BB_PRESS_PREFIX}bp_messages_recipients").first[
"count"
]
current_count = 0
batches(BATCH_SIZE) do |offset|
rows = bbpress_query(<<-SQL
rows = bbpress_query(<<-SQL).to_a
SELECT thread_id, user_id
FROM #{BB_PRESS_PREFIX}bp_messages_recipients
ORDER BY id
LIMIT #{BATCH_SIZE}
OFFSET #{offset}
SQL
).to_a
break if rows.empty?
rows.each do |row|
current_count += 1
print_status(current_count, total_count, get_start_time('private_messages'))
print_status(current_count, total_count, get_start_time("private_messages"))
threads[row['thread_id']] ||= {
target_user_ids: [],
imported_topic_id: nil
}
user_id = user_id_from_imported_user_id(row['user_id'])
if user_id && !threads[row['thread_id']][:target_user_ids].include?(user_id)
threads[row['thread_id']][:target_user_ids] << user_id
threads[row["thread_id"]] ||= { target_user_ids: [], imported_topic_id: nil }
user_id = user_id_from_imported_user_id(row["user_id"])
if user_id && !threads[row["thread_id"]][:target_user_ids].include?(user_id)
threads[row["thread_id"]][:target_user_ids] << user_id
end
end
end
batches(BATCH_SIZE) do |offset|
posts = bbpress_query(<<-SQL
posts = bbpress_query(<<-SQL).to_a
SELECT id,
thread_id,
date_sent,
@ -464,39 +460,48 @@ class ImportScripts::Bbpress < ImportScripts::Base
ORDER BY thread_id, date_sent
LIMIT #{BATCH_SIZE}
SQL
).to_a
break if posts.empty?
last_post_id = posts[-1]["id"].to_i
create_posts(posts, total: total_posts, offset: offset) do |post|
if tcf = TopicCustomField.where(name: 'bb_thread_id', value: post['thread_id']).first
if tcf = TopicCustomField.where(name: "bb_thread_id", value: post["thread_id"]).first
{
id: "pm#{post['id']}",
topic_id: threads[post['thread_id']][:imported_topic_id],
user_id: user_id_from_imported_user_id(post['sender_id']) || find_user_by_import_id(post['sender_id'])&.id || -1,
raw: post['message'],
created_at: post['date_sent'],
id: "pm#{post["id"]}",
topic_id: threads[post["thread_id"]][:imported_topic_id],
user_id:
user_id_from_imported_user_id(post["sender_id"]) ||
find_user_by_import_id(post["sender_id"])&.id || -1,
raw: post["message"],
created_at: post["date_sent"],
}
else
# First post of the thread
{
id: "pm#{post['id']}",
id: "pm#{post["id"]}",
archetype: Archetype.private_message,
user_id: user_id_from_imported_user_id(post['sender_id']) || find_user_by_import_id(post['sender_id'])&.id || -1,
title: post['subject'],
raw: post['message'],
created_at: post['date_sent'],
target_usernames: User.where(id: threads[post['thread_id']][:target_user_ids]).pluck(:username),
post_create_action: proc do |new_post|
if topic = new_post.topic
threads[post['thread_id']][:imported_topic_id] = topic.id
TopicCustomField.create(topic_id: topic.id, name: 'bb_thread_id', value: post['thread_id'])
else
puts "Error in post_create_action! Can't find topic!"
end
end
user_id:
user_id_from_imported_user_id(post["sender_id"]) ||
find_user_by_import_id(post["sender_id"])&.id || -1,
title: post["subject"],
raw: post["message"],
created_at: post["date_sent"],
target_usernames:
User.where(id: threads[post["thread_id"]][:target_user_ids]).pluck(:username),
post_create_action:
proc do |new_post|
if topic = new_post.topic
threads[post["thread_id"]][:imported_topic_id] = topic.id
TopicCustomField.create(
topic_id: topic.id,
name: "bb_thread_id",
value: post["thread_id"],
)
else
puts "Error in post_create_action! Can't find topic!"
end
end,
}
end
end
@ -506,7 +511,6 @@ class ImportScripts::Bbpress < ImportScripts::Base
def bbpress_query(sql)
@client.query(sql, cache_rows: false)
end
end
ImportScripts::Bbpress.new.perform