github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-22 16:21:18 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: Apply syntax_tree formatting to script/*

Browse Source
This commit is contained in:
David Taylor
2023-01-07 11:53:14 +00:00
parent ff508d1ae5
commit 436b3b392b
143 changed files with 8905 additions and 7353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
script/import_scripts/bespoke_1.rb
View File

@ -2,13 +2,12 @@
# bespoke importer for a customer, feel free to borrow ideas
require 'csv'
require "csv"
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
# Call it like this:
# RAILS_ENV=production bundle exec ruby script/import_scripts/bespoke_1.rb
class ImportScripts::Bespoke < ImportScripts::Base
BATCH_SIZE = 1000
def initialize(path)
@ -18,9 +17,9 @@ class ImportScripts::Bespoke < ImportScripts::Base
puts "loading post mappings..."
@post_number_map = {}
Post.pluck(:id, :post_number).each do |post_id, post_number|
@post_number_map[post_id] = post_number
end
Post
.pluck(:id, :post_number)
.each { |post_id, post_number| @post_number_map[post_id] = post_number }
end
def created_post(post)
@ -32,7 +31,6 @@ class ImportScripts::Bespoke < ImportScripts::Base
import_users
import_categories
import_posts
end
class RowResolver
@ -45,19 +43,13 @@ class ImportScripts::Bespoke < ImportScripts::Base
end
def initialize(cols)
cols.each_with_index do |col, idx|
self.class.public_send(:define_method, col) do
@row[idx]
end
end
cols.each_with_index { |col, idx| self.class.public_send(:define_method, col) { @row[idx] } }
end
end
def load_user_batch!(users, offset, total)
if users.length > 0
create_users(users, offset: offset, total: total) do |user|
user
end
create_users(users, offset: offset, total: total) { |user| user }
users.clear
end
end
@ -70,54 +62,56 @@ class ImportScripts::Bespoke < ImportScripts::Base
current_row = +""
double_quote_count = 0
File.open(filename).each_line do |line|
File
.open(filename)
.each_line do |line|
# escaping is mental here
line.gsub!(/\\(.{1})/) { |m| m[-1] == '"' ? '""' : m[-1] }
line.strip!
# escaping is mental here
line.gsub!(/\\(.{1})/) { |m| m[-1] == '"' ? '""' : m[-1] }
line.strip!
current_row << "\n" unless current_row.empty?
current_row << line
current_row << "\n" unless current_row.empty?
current_row << line
double_quote_count += line.scan('"').count
double_quote_count += line.scan('"').count
next if double_quote_count % 2 == 1
if double_quote_count % 2 == 1
next
end
raw =
begin
CSV.parse(current_row)
rescue CSV::MalformedCSVError => e
puts e.message
puts "*" * 100
puts "Bad row skipped, line is: #{line}"
puts
puts current_row
puts
puts "double quote count is : #{double_quote_count}"
puts "*" * 100
raw = begin
CSV.parse(current_row)
rescue CSV::MalformedCSVError => e
puts e.message
puts "*" * 100
puts "Bad row skipped, line is: #{line}"
puts
puts current_row
puts
puts "double quote count is : #{double_quote_count}"
puts "*" * 100
current_row = ""
double_quote_count = 0
next
end[
0
]
current_row = ""
double_quote_count = 0
next
end[0]
if first
row = RowResolver.create(raw)
if first
row = RowResolver.create(raw)
current_row = ""
double_quote_count = 0
first = false
next
end
row.load(raw)
yield row
current_row = ""
double_quote_count = 0
first = false
next
end
row.load(raw)
yield row
current_row = ""
double_quote_count = 0
end
end
def total_rows(table)
@ -133,14 +127,11 @@ class ImportScripts::Bespoke < ImportScripts::Base
total = total_rows("users")
csv_parse("users") do |row|
id = row.id
email = row.email
# fake it
if row.email.blank? || row.email !~ /@/
email = fake_email
end
email = fake_email if row.email.blank? || row.email !~ /@/
name = row.display_name
username = row.key_custom
@ -150,19 +141,10 @@ class ImportScripts::Bespoke < ImportScripts::Base
username = email.split("@")[0] if username.blank?
name = email.split("@")[0] if name.blank?
users << {
id: id,
email: email,
name: name,
username: username,
created_at: created_at
}
users << { id: id, email: email, name: name, username: username, created_at: created_at }
count += 1
if count % BATCH_SIZE == 0
load_user_batch! users, count - users.length, total
end
load_user_batch! users, count - users.length, total if count % BATCH_SIZE == 0
end
load_user_batch! users, count, total
@ -174,22 +156,19 @@ class ImportScripts::Bespoke < ImportScripts::Base
rows << { id: row.id, name: row.name, description: row.description }
end
create_categories(rows) do |row|
row
end
create_categories(rows) { |row| row }
end
def normalize_raw!(raw)
# purple and #1223f3
raw.gsub!(/\[color=[#a-z0-9]+\]/i, "")
raw.gsub!(/\[\/color\]/i, "")
raw.gsub!(/\[signature\].+\[\/signature\]/im, "")
raw.gsub!(%r{\[/color\]}i, "")
raw.gsub!(%r{\[signature\].+\[/signature\]}im, "")
raw
end
def import_post_batch!(posts, topics, offset, total)
create_posts(posts, total: total, offset: offset) do |post|
mapped = {}
mapped[:id] = post[:id]
@ -223,7 +202,7 @@ class ImportScripts::Bespoke < ImportScripts::Base
mapped
end
posts.clear
posts.clear
end
def import_posts
@ -237,7 +216,7 @@ class ImportScripts::Bespoke < ImportScripts::Base
category_id: topic.forum_category_id,
deleted: topic.is_deleted.to_i == 1,
locked: topic.is_locked.to_i == 1,
pinned: topic.is_pinned.to_i == 1
pinned: topic.is_pinned.to_i == 1,
}
end
@ -246,7 +225,6 @@ class ImportScripts::Bespoke < ImportScripts::Base
posts = []
count = 0
csv_parse("posts") do |row|
unless row.dcreate
puts "NO CREATION DATE FOR POST"
p row
@ -261,7 +239,7 @@ class ImportScripts::Bespoke < ImportScripts::Base
title: row.title,
body: normalize_raw!(row.body),
deleted: row.is_deleted.to_i == 1,
created_at: DateTime.parse(row.dcreate)
created_at: DateTime.parse(row.dcreate),
}
posts << row
count += 1
@ -275,7 +253,6 @@ class ImportScripts::Bespoke < ImportScripts::Base
exit
end
end
unless ARGV[0] && Dir.exist?(ARGV[0])