mirror of
https://github.com/discourse/discourse.git
synced 2025-05-22 22:21:04 +08:00
DEV: Apply syntax_tree formatting to script/*
This commit is contained in:
David Taylor
@ -1,12 +1,11 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
# Jive importer
|
||||
require 'nokogiri'
|
||||
require 'csv'
|
||||
require "nokogiri"
|
||||
require "csv"
|
||||
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
|
||||
|
||||
class ImportScripts::Jive < ImportScripts::Base
|
||||
|
||||
BATCH_SIZE = 1000
|
||||
CATEGORY_IDS = [2023, 2003, 2004, 2042, 2036, 2029] # categories that should be imported
|
||||
|
||||
@ -17,9 +16,9 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
|
||||
puts "loading post mappings..."
|
||||
@post_number_map = {}
|
||||
Post.pluck(:id, :post_number).each do |post_id, post_number|
|
||||
@post_number_map[post_id] = post_number
|
||||
end
|
||||
Post
|
||||
.pluck(:id, :post_number)
|
||||
.each { |post_id, post_number| @post_number_map[post_id] = post_number }
|
||||
end
|
||||
|
||||
def created_post(post)
|
||||
@ -47,19 +46,13 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
end
|
||||
|
||||
def initialize(cols)
|
||||
cols.each_with_index do |col, idx|
|
||||
self.class.public_send(:define_method, col) do
|
||||
@row[idx]
|
||||
end
|
||||
end
|
||||
cols.each_with_index { |col, idx| self.class.public_send(:define_method, col) { @row[idx] } }
|
||||
end
|
||||
end
|
||||
|
||||
def load_user_batch!(users, offset, total)
|
||||
if users.length > 0
|
||||
create_users(users, offset: offset, total: total) do |user|
|
||||
user
|
||||
end
|
||||
create_users(users, offset: offset, total: total) { |user| user }
|
||||
users.clear
|
||||
end
|
||||
end
|
||||
@ -72,53 +65,55 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
current_row = +""
|
||||
double_quote_count = 0
|
||||
|
||||
File.open(filename).each_line do |line|
|
||||
File
|
||||
.open(filename)
|
||||
.each_line do |line|
|
||||
line.gsub!(/\\(.{1})/) { |m| m[-1] == '"' ? '""' : m[-1] }
|
||||
line.strip!
|
||||
|
||||
line.gsub!(/\\(.{1})/) { |m| m[-1] == '"' ? '""' : m[-1] }
|
||||
line.strip!
|
||||
current_row << "\n" unless current_row.empty?
|
||||
current_row << line
|
||||
|
||||
current_row << "\n" unless current_row.empty?
|
||||
current_row << line
|
||||
double_quote_count += line.scan('"').count
|
||||
|
||||
double_quote_count += line.scan('"').count
|
||||
next if double_quote_count % 2 == 1
|
||||
|
||||
if double_quote_count % 2 == 1
|
||||
next
|
||||
end
|
||||
raw =
|
||||
begin
|
||||
CSV.parse(current_row)
|
||||
rescue CSV::MalformedCSVError => e
|
||||
puts e.message
|
||||
puts "*" * 100
|
||||
puts "Bad row skipped, line is: #{line}"
|
||||
puts
|
||||
puts current_row
|
||||
puts
|
||||
puts "double quote count is : #{double_quote_count}"
|
||||
puts "*" * 100
|
||||
|
||||
raw = begin
|
||||
CSV.parse(current_row)
|
||||
rescue CSV::MalformedCSVError => e
|
||||
puts e.message
|
||||
puts "*" * 100
|
||||
puts "Bad row skipped, line is: #{line}"
|
||||
puts
|
||||
puts current_row
|
||||
puts
|
||||
puts "double quote count is : #{double_quote_count}"
|
||||
puts "*" * 100
|
||||
current_row = ""
|
||||
double_quote_count = 0
|
||||
next
|
||||
end[
|
||||
0
|
||||
]
|
||||
|
||||
current_row = ""
|
||||
double_quote_count = 0
|
||||
next
|
||||
end[0]
|
||||
if first
|
||||
row = RowResolver.create(raw)
|
||||
|
||||
if first
|
||||
row = RowResolver.create(raw)
|
||||
current_row = ""
|
||||
double_quote_count = 0
|
||||
first = false
|
||||
next
|
||||
end
|
||||
|
||||
row.load(raw)
|
||||
|
||||
yield row
|
||||
|
||||
current_row = ""
|
||||
double_quote_count = 0
|
||||
first = false
|
||||
next
|
||||
end
|
||||
|
||||
row.load(raw)
|
||||
|
||||
yield row
|
||||
|
||||
current_row = ""
|
||||
double_quote_count = 0
|
||||
end
|
||||
end
|
||||
|
||||
def total_rows(table)
|
||||
@ -129,13 +124,9 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
puts "", "importing groups..."
|
||||
|
||||
rows = []
|
||||
csv_parse("groups") do |row|
|
||||
rows << { id: row.groupid, name: row.name }
|
||||
end
|
||||
csv_parse("groups") { |row| rows << { id: row.groupid, name: row.name } }
|
||||
|
||||
create_groups(rows) do |row|
|
||||
row
|
||||
end
|
||||
create_groups(rows) { |row| row }
|
||||
end
|
||||
|
||||
def import_users
|
||||
@ -147,15 +138,12 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
total = total_rows("users")
|
||||
|
||||
csv_parse("users") do |row|
|
||||
|
||||
id = row.userid
|
||||
|
||||
email = "#{row.email}"
|
||||
|
||||
# fake it
|
||||
if row.email.blank? || row.email !~ /@/
|
||||
email = fake_email
|
||||
end
|
||||
email = fake_email if row.email.blank? || row.email !~ /@/
|
||||
|
||||
name = "#{row.firstname} #{row.lastname}"
|
||||
username = row.username
|
||||
@ -175,14 +163,11 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
created_at: created_at,
|
||||
last_seen_at: last_seen_at,
|
||||
active: is_activated.to_i == 1,
|
||||
approved: true
|
||||
approved: true,
|
||||
}
|
||||
|
||||
count += 1
|
||||
if count % BATCH_SIZE == 0
|
||||
load_user_batch! users, count - users.length, total
|
||||
end
|
||||
|
||||
load_user_batch! users, count - users.length, total if count % BATCH_SIZE == 0
|
||||
end
|
||||
|
||||
load_user_batch! users, count, total
|
||||
@ -195,9 +180,7 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
user_id = user_id_from_imported_user_id(row.userid)
|
||||
group_id = group_id_from_imported_group_id(row.groupid)
|
||||
|
||||
if user_id && group_id
|
||||
GroupUser.find_or_create_by(user_id: user_id, group_id: group_id)
|
||||
end
|
||||
GroupUser.find_or_create_by(user_id: user_id, group_id: group_id) if user_id && group_id
|
||||
end
|
||||
end
|
||||
|
||||
@ -209,9 +192,7 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
rows << { id: row.communityid, name: "#{row.name} (#{row.communityid})" }
|
||||
end
|
||||
|
||||
create_categories(rows) do |row|
|
||||
row
|
||||
end
|
||||
create_categories(rows) { |row| row }
|
||||
end
|
||||
|
||||
def normalize_raw!(raw)
|
||||
@ -219,9 +200,7 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
raw = raw[5..-6]
|
||||
|
||||
doc = Nokogiri::HTML5.fragment(raw)
|
||||
doc.css('img').each do |img|
|
||||
img.remove if img['class'] == "jive-image"
|
||||
end
|
||||
doc.css("img").each { |img| img.remove if img["class"] == "jive-image" }
|
||||
|
||||
raw = doc.to_html
|
||||
raw = raw[4..-1]
|
||||
@ -231,7 +210,6 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
|
||||
def import_post_batch!(posts, topics, offset, total)
|
||||
create_posts(posts, total: total, offset: offset) do |post|
|
||||
|
||||
mapped = {}
|
||||
|
||||
mapped[:id] = post[:id]
|
||||
@ -271,7 +249,7 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
mapped
|
||||
end
|
||||
|
||||
posts.clear
|
||||
posts.clear
|
||||
end
|
||||
|
||||
def import_posts
|
||||
@ -281,7 +259,6 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
thread_map = {}
|
||||
|
||||
csv_parse("messages") do |thread|
|
||||
|
||||
next unless CATEGORY_IDS.include?(thread.containerid.to_i)
|
||||
|
||||
if !thread.parentmessageid
|
||||
@ -291,32 +268,38 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
|
||||
#IMAGE UPLOADER
|
||||
if thread.imagecount
|
||||
Dir.foreach("/var/www/discourse/script/import_scripts/jive/img/#{thread.messageid}") do |item|
|
||||
next if item == ('.') || item == ('..') || item == ('.DS_Store')
|
||||
photo_path = "/var/www/discourse/script/import_scripts/jive/img/#{thread.messageid}/#{item}"
|
||||
Dir.foreach(
|
||||
"/var/www/discourse/script/import_scripts/jive/img/#{thread.messageid}",
|
||||
) do |item|
|
||||
next if item == (".") || item == ("..") || item == (".DS_Store")
|
||||
photo_path =
|
||||
"/var/www/discourse/script/import_scripts/jive/img/#{thread.messageid}/#{item}"
|
||||
upload = create_upload(thread.userid, photo_path, File.basename(photo_path))
|
||||
if upload.persisted?
|
||||
puts "Image upload is successful for #{photo_path}, new path is #{upload.url}!"
|
||||
thread.body.gsub!(item, upload.url)
|
||||
else
|
||||
puts "Error: Image upload is not successful for #{photo_path}!"
|
||||
end
|
||||
if upload.persisted?
|
||||
puts "Image upload is successful for #{photo_path}, new path is #{upload.url}!"
|
||||
thread.body.gsub!(item, upload.url)
|
||||
else
|
||||
puts "Error: Image upload is not successful for #{photo_path}!"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
#ATTACHMENT UPLOADER
|
||||
if thread.attachmentcount
|
||||
Dir.foreach("/var/www/discourse/script/import_scripts/jive/attach/#{thread.messageid}") do |item|
|
||||
next if item == ('.') || item == ('..') || item == ('.DS_Store')
|
||||
attach_path = "/var/www/discourse/script/import_scripts/jive/attach/#{thread.messageid}/#{item}"
|
||||
Dir.foreach(
|
||||
"/var/www/discourse/script/import_scripts/jive/attach/#{thread.messageid}",
|
||||
) do |item|
|
||||
next if item == (".") || item == ("..") || item == (".DS_Store")
|
||||
attach_path =
|
||||
"/var/www/discourse/script/import_scripts/jive/attach/#{thread.messageid}/#{item}"
|
||||
upload = create_upload(thread.userid, attach_path, File.basename(attach_path))
|
||||
if upload.persisted?
|
||||
puts "Attachment upload is successful for #{attach_path}, new path is #{upload.url}!"
|
||||
thread.body.gsub!(item, upload.url)
|
||||
thread.body << "<br/><br/> #{attachment_html(upload, item)}"
|
||||
else
|
||||
puts "Error: Attachment upload is not successful for #{attach_path}!"
|
||||
end
|
||||
if upload.persisted?
|
||||
puts "Attachment upload is successful for #{attach_path}, new path is #{upload.url}!"
|
||||
thread.body.gsub!(item, upload.url)
|
||||
thread.body << "<br/><br/> #{attachment_html(upload, item)}"
|
||||
else
|
||||
puts "Error: Attachment upload is not successful for #{attach_path}!"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@ -329,7 +312,6 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
body: normalize_raw!(thread.body || thread.subject || "<missing>"),
|
||||
created_at: DateTime.parse(thread.creationdate),
|
||||
}
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
@ -348,35 +330,40 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
next unless CATEGORY_IDS.include?(thread.containerid.to_i)
|
||||
|
||||
if thread.parentmessageid
|
||||
|
||||
#IMAGE UPLOADER
|
||||
if thread.imagecount
|
||||
Dir.foreach("/var/www/discourse/script/import_scripts/jive/img/#{thread.messageid}") do |item|
|
||||
next if item == ('.') || item == ('..') || item == ('.DS_Store')
|
||||
photo_path = "/var/www/discourse/script/import_scripts/jive/img/#{thread.messageid}/#{item}"
|
||||
Dir.foreach(
|
||||
"/var/www/discourse/script/import_scripts/jive/img/#{thread.messageid}",
|
||||
) do |item|
|
||||
next if item == (".") || item == ("..") || item == (".DS_Store")
|
||||
photo_path =
|
||||
"/var/www/discourse/script/import_scripts/jive/img/#{thread.messageid}/#{item}"
|
||||
upload = create_upload(thread.userid, photo_path, File.basename(photo_path))
|
||||
if upload.persisted?
|
||||
puts "Image upload is successful for #{photo_path}, new path is #{upload.url}!"
|
||||
thread.body.gsub!(item, upload.url)
|
||||
else
|
||||
puts "Error: Image upload is not successful for #{photo_path}!"
|
||||
end
|
||||
if upload.persisted?
|
||||
puts "Image upload is successful for #{photo_path}, new path is #{upload.url}!"
|
||||
thread.body.gsub!(item, upload.url)
|
||||
else
|
||||
puts "Error: Image upload is not successful for #{photo_path}!"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
#ATTACHMENT UPLOADER
|
||||
if thread.attachmentcount
|
||||
Dir.foreach("/var/www/discourse/script/import_scripts/jive/attach/#{thread.messageid}") do |item|
|
||||
next if item == ('.') || item == ('..') || item == ('.DS_Store')
|
||||
attach_path = "/var/www/discourse/script/import_scripts/jive/attach/#{thread.messageid}/#{item}"
|
||||
Dir.foreach(
|
||||
"/var/www/discourse/script/import_scripts/jive/attach/#{thread.messageid}",
|
||||
) do |item|
|
||||
next if item == (".") || item == ("..") || item == (".DS_Store")
|
||||
attach_path =
|
||||
"/var/www/discourse/script/import_scripts/jive/attach/#{thread.messageid}/#{item}"
|
||||
upload = create_upload(thread.userid, attach_path, File.basename(attach_path))
|
||||
if upload.persisted?
|
||||
puts "Attachment upload is successful for #{attach_path}, new path is #{upload.url}!"
|
||||
thread.body.gsub!(item, upload.url)
|
||||
thread.body << "<br/><br/> #{attachment_html(upload, item)}"
|
||||
else
|
||||
puts "Error: Attachment upload is not successful for #{attach_path}!"
|
||||
end
|
||||
if upload.persisted?
|
||||
puts "Attachment upload is successful for #{attach_path}, new path is #{upload.url}!"
|
||||
thread.body.gsub!(item, upload.url)
|
||||
thread.body << "<br/><br/> #{attachment_html(upload, item)}"
|
||||
else
|
||||
puts "Error: Attachment upload is not successful for #{attach_path}!"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@ -386,7 +373,7 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
user_id: thread.userid,
|
||||
title: thread.subject,
|
||||
body: normalize_raw!(thread.body),
|
||||
created_at: DateTime.parse(thread.creationdate)
|
||||
created_at: DateTime.parse(thread.creationdate),
|
||||
}
|
||||
posts << row
|
||||
count += 1
|
||||
@ -399,7 +386,6 @@ class ImportScripts::Jive < ImportScripts::Base
|
||||
|
||||
import_post_batch!(posts, topic_map, count - posts.length, total) if posts.length > 0
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
unless ARGV[0] && Dir.exist?(ARGV[0])
|
||||
|
||||
Reference in New Issue
Block a user