github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-22 07:53:49 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

DEV: Apply syntax_tree formatting to script/*

Browse Source
This commit is contained in:
David Taylor
2023-01-07 11:53:14 +00:00
parent ff508d1ae5
commit 436b3b392b
143 changed files with 8905 additions and 7353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
script/import_scripts/base/csv_helper.rb
View File

@ -13,65 +13,69 @@ module ImportScripts
def initialize(cols)
cols.each_with_index do |col, idx|
self.class.public_send(:define_method, col.downcase.gsub(/[\W]/, '_').squeeze('_')) do
@row[idx]
end
self
.class
.public_send(:define_method, col.downcase.gsub(/[\W]/, "_").squeeze("_")) { @row[idx] }
end
end
end
def csv_parse(filename, col_sep = ',')
def csv_parse(filename, col_sep = ",")
first = true
row = nil
current_row = +""
double_quote_count = 0
File.open(filename).each_line do |line|
File
.open(filename)
.each_line do |line|
line.strip!
line.strip!
current_row << "\n" unless current_row.empty?
current_row << line
current_row << "\n" unless current_row.empty?
current_row << line
double_quote_count += line.scan('"').count
double_quote_count += line.scan('"').count
next if double_quote_count % 2 == 1 # this row continues on a new line. don't parse until we have the whole row.
next if double_quote_count % 2 == 1 # this row continues on a new line. don't parse until we have the whole row.
raw =
begin
CSV.parse(current_row, col_sep: col_sep)
rescue CSV::MalformedCSVError => e
puts e.message
puts "*" * 100
puts "Bad row skipped, line is: #{line}"
puts
puts current_row
puts
puts "double quote count is : #{double_quote_count}"
puts "*" * 100
raw = begin
CSV.parse(current_row, col_sep: col_sep)
rescue CSV::MalformedCSVError => e
puts e.message
puts "*" * 100
puts "Bad row skipped, line is: #{line}"
puts
puts current_row
puts
puts "double quote count is : #{double_quote_count}"
puts "*" * 100
current_row = ""
double_quote_count = 0
current_row = ""
double_quote_count = 0
next
end[
0
]
next
end[0]
if first
row = RowResolver.create(raw)
if first
row = RowResolver.create(raw)
current_row = ""
double_quote_count = 0
first = false
next
end
row.load(raw)
yield row
current_row = ""
double_quote_count = 0
first = false
next
end
row.load(raw)
yield row
current_row = ""
double_quote_count = 0
end
end
end
end

38
script/import_scripts/base/generic_database.rb
View File

@ -1,6 +1,6 @@
# frozen_string_literal: true
require 'sqlite3'
require "sqlite3"
module ImportScripts
class GenericDatabase
@ -80,24 +80,20 @@ module ImportScripts
VALUES (:id, :raw, :topic_id, :user_id, :created_at, :reply_to_post_id, :url, :upload_count)
SQL
attachments&.each do |attachment|
@db.execute(<<-SQL, post_id: post[:id], path: attachment)
attachments&.each { |attachment| @db.execute(<<-SQL, post_id: post[:id], path: attachment) }
INSERT OR REPLACE INTO post_upload (post_id, path)
VALUES (:post_id, :path)
SQL
end
like_user_ids&.each do |user_id|
@db.execute(<<-SQL, post_id: post[:id], user_id: user_id)
like_user_ids&.each { |user_id| @db.execute(<<-SQL, post_id: post[:id], user_id: user_id) }
INSERT OR REPLACE INTO like (post_id, user_id)
VALUES (:post_id, :user_id)
SQL
end
end
end
def sort_posts_by_created_at
@db.execute 'DELETE FROM post_order'
@db.execute "DELETE FROM post_order"
@db.execute <<-SQL
INSERT INTO post_order (post_id)
@ -146,7 +142,7 @@ module ImportScripts
LIMIT #{@batch_size}
SQL
add_last_column_value(rows, 'id')
add_last_column_value(rows, "id")
end
def get_user_id(username)
@ -173,7 +169,7 @@ module ImportScripts
LIMIT #{@batch_size}
SQL
add_last_column_value(rows, 'id')
add_last_column_value(rows, "id")
end
def fetch_topic_attachments(topic_id)
@ -200,7 +196,7 @@ module ImportScripts
LIMIT #{@batch_size}
SQL
add_last_column_value(rows, 'rowid')
add_last_column_value(rows, "rowid")
end
def fetch_sorted_posts(last_row_id)
@ -213,7 +209,7 @@ module ImportScripts
LIMIT #{@batch_size}
SQL
add_last_column_value(rows, 'rowid')
add_last_column_value(rows, "rowid")
end
def fetch_post_attachments(post_id)
@ -240,7 +236,7 @@ module ImportScripts
LIMIT #{@batch_size}
SQL
add_last_column_value(rows, 'rowid')
add_last_column_value(rows, "rowid")
end
def execute_sql(sql)
@ -254,12 +250,12 @@ module ImportScripts
private
def configure_database
@db.execute 'PRAGMA journal_mode = OFF'
@db.execute 'PRAGMA locking_mode = EXCLUSIVE'
@db.execute "PRAGMA journal_mode = OFF"
@db.execute "PRAGMA locking_mode = EXCLUSIVE"
end
def key_data_type
@numeric_keys ? 'INTEGER' : 'TEXT'
@numeric_keys ? "INTEGER" : "TEXT"
end
def create_category_table
@ -299,7 +295,7 @@ module ImportScripts
)
SQL
@db.execute 'CREATE INDEX IF NOT EXISTS user_by_username ON user (username)'
@db.execute "CREATE INDEX IF NOT EXISTS user_by_username ON user (username)"
end
def create_topic_table
@ -317,7 +313,7 @@ module ImportScripts
)
SQL
@db.execute 'CREATE INDEX IF NOT EXISTS topic_by_user_id ON topic (user_id)'
@db.execute "CREATE INDEX IF NOT EXISTS topic_by_user_id ON topic (user_id)"
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_upload (
@ -326,7 +322,7 @@ module ImportScripts
)
SQL
@db.execute 'CREATE UNIQUE INDEX IF NOT EXISTS topic_upload_unique ON topic_upload(topic_id, path)'
@db.execute "CREATE UNIQUE INDEX IF NOT EXISTS topic_upload_unique ON topic_upload(topic_id, path)"
end
def create_post_table
@ -343,7 +339,7 @@ module ImportScripts
)
SQL
@db.execute 'CREATE INDEX IF NOT EXISTS post_by_user_id ON post (user_id)'
@db.execute "CREATE INDEX IF NOT EXISTS post_by_user_id ON post (user_id)"
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS post_order (
@ -358,7 +354,7 @@ module ImportScripts
)
SQL
@db.execute 'CREATE UNIQUE INDEX IF NOT EXISTS post_upload_unique ON post_upload(post_id, path)'
@db.execute "CREATE UNIQUE INDEX IF NOT EXISTS post_upload_unique ON post_upload(post_id, path)"
end
def prepare(hash)

42
script/import_scripts/base/lookup_container.rb
View File

@ -3,27 +3,26 @@
module ImportScripts
class LookupContainer
def initialize
puts 'Loading existing groups...'
@groups = GroupCustomField.where(name: 'import_id').pluck(:value, :group_id).to_h
puts "Loading existing groups..."
@groups = GroupCustomField.where(name: "import_id").pluck(:value, :group_id).to_h
puts 'Loading existing users...'
@users = UserCustomField.where(name: 'import_id').pluck(:value, :user_id).to_h
puts "Loading existing users..."
@users = UserCustomField.where(name: "import_id").pluck(:value, :user_id).to_h
puts 'Loading existing categories...'
@categories = CategoryCustomField.where(name: 'import_id').pluck(:value, :category_id).to_h
puts "Loading existing categories..."
@categories = CategoryCustomField.where(name: "import_id").pluck(:value, :category_id).to_h
puts 'Loading existing posts...'
@posts = PostCustomField.where(name: 'import_id').pluck(:value, :post_id).to_h
puts "Loading existing posts..."
@posts = PostCustomField.where(name: "import_id").pluck(:value, :post_id).to_h
puts 'Loading existing topics...'
puts "Loading existing topics..."
@topics = {}
Post.joins(:topic).pluck('posts.id, posts.topic_id, posts.post_number, topics.slug').each do |p|
@topics[p[0]] = {
topic_id: p[1],
post_number: p[2],
url: Post.url(p[3], p[1], p[2])
}
end
Post
.joins(:topic)
.pluck("posts.id, posts.topic_id, posts.post_number, topics.slug")
.each do |p|
@topics[p[0]] = { topic_id: p[1], post_number: p[2], url: Post.url(p[3], p[1], p[2]) }
end
end
# Get the Discourse Post id based on the id of the source record
@ -44,7 +43,7 @@ module ImportScripts
# Get the Discourse Group based on the id of the source group
def find_group_by_import_id(import_id)
GroupCustomField.where(name: 'import_id', value: import_id.to_s).first.try(:group)
GroupCustomField.where(name: "import_id", value: import_id.to_s).first.try(:group)
end
# Get the Discourse User id based on the id of the source user
@ -54,7 +53,7 @@ module ImportScripts
# Get the Discourse User based on the id of the source user
def find_user_by_import_id(import_id)
UserCustomField.where(name: 'import_id', value: import_id.to_s).first.try(:user)
UserCustomField.where(name: "import_id", value: import_id.to_s).first.try(:user)
end
def find_username_by_import_id(import_id)
@ -84,11 +83,7 @@ module ImportScripts
end
def add_topic(post)
@topics[post.id] = {
post_number: post.post_number,
topic_id: post.topic_id,
url: post.url,
}
@topics[post.id] = { post_number: post.post_number, topic_id: post.topic_id, url: post.url }
end
def user_already_imported?(import_id)
@ -98,6 +93,5 @@ module ImportScripts
def post_already_imported?(import_id)
@posts.has_key?(import_id) || @posts.has_key?(import_id.to_s)
end
end
end

20
script/import_scripts/base/uploader.rb
View File

@ -13,8 +13,16 @@ module ImportScripts
STDERR.puts "Failed to create upload: #{e}"
nil
ensure
tmp.close rescue nil
tmp.unlink rescue nil
begin
tmp.close
rescue StandardError
nil
end
begin
tmp.unlink
rescue StandardError
nil
end
end
def create_avatar(user, avatar_path)
@ -30,7 +38,7 @@ module ImportScripts
STDERR.puts "Failed to upload avatar for user #{user.username}: #{avatar_path}"
STDERR.puts upload.errors.inspect if upload
end
rescue
rescue StandardError
STDERR.puts "Failed to create avatar for user #{user.username}: #{avatar_path}"
ensure
tempfile.close! if tempfile
@ -52,11 +60,9 @@ module ImportScripts
def copy_to_tempfile(source_path)
extension = File.extname(source_path)
tmp = Tempfile.new(['discourse-upload', extension])
tmp = Tempfile.new(["discourse-upload", extension])
File.open(source_path) do |source_stream|
IO.copy_stream(source_stream, tmp)
end
File.open(source_path) { |source_stream| IO.copy_stream(source_stream, tmp) }
tmp.rewind
tmp