github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-21 18:12:32 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

REFACTOR: upload workflow creation into UploadCreator

Browse Source 
- Automatically convert large-ish PNG/BMP to JPEG
- Updated fast_image to latest version
This commit is contained in:
Régis Hanol
2017-05-11 00:16:57 +02:00
parent a5c4ddd334
commit 9641d2413d
27 changed files with 391 additions and 483 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
script/import_scripts/base/uploader.rb
View File

@ -15,7 +15,7 @@ module ImportScripts
src.close
tmp.rewind
Upload.create_for(user_id, tmp, source_filename, tmp.size)
UploadCreator.new(tmp, source_filename).create_for(user_id)
rescue => e
Rails.logger.error("Failed to create upload: #{e}")
nil

6
script/import_scripts/lithium.rb
View File

@ -144,7 +144,7 @@ class ImportScripts::Lithium < ImportScripts::Base
file.write(picture["filedata"].encode("ASCII-8BIT").force_encoding("UTF-8"))
file.rewind
upload = Upload.create_for(imported_user.id, file, picture["filename"], file.size)
upload = UploadCreator.new(file, picture["filename"]).create_for(imported_user.id)
return if !upload.persisted?
@ -173,7 +173,7 @@ class ImportScripts::Lithium < ImportScripts::Base
file.write(background["filedata"].encode("ASCII-8BIT").force_encoding("UTF-8"))
file.rewind
upload = Upload.create_for(imported_user.id, file, background["filename"], file.size)
upload = UploadCreator.new(file, background["filename"]).create_for(imported_user.id)
return if !upload.persisted?
@ -807,7 +807,7 @@ SQL
if image
File.open(image) do |file|
upload = Upload.create_for(user_id, file, "image." + (image =~ /.png$/ ? "png": "jpg"), File.size(image))
upload = UploadCreator.new(file, "image." + (image.ends_with?(".png") ? "png" : "jpg")).create_for(user_id)
l["src"] = upload.url
end
else

4
script/import_scripts/mbox.rb
View File

@ -439,7 +439,7 @@ p end
# read attachment
File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded }
# create the upload for the user
upload = Upload.create_for(user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID, tmp, attachment.filename, tmp.size )
upload = UploadCreator.new(tmp, attachment.filename).create_for(user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID)
if upload && upload.errors.empty?
raw << "\n\n#{receiver.attachment_markdown(upload)}\n\n"
end
@ -530,7 +530,7 @@ p end
# read attachment
File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded }
# create the upload for the user
upload = Upload.create_for(user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID, tmp, attachment.filename, tmp.size )
upload = UploadCreator.new(tmp, attachment.filename).create_for(user_id_from_imported_user_id(from_email) || Discourse::SYSTEM_USER_ID)
if upload && upload.errors.empty?
raw << "\n\n#{receiver.attachment_markdown(upload)}\n\n"
end

2
script/import_scripts/sfn.rb
View File

@ -101,7 +101,7 @@ class ImportScripts::Sfn < ImportScripts::Base
avatar.write(user["avatar"].encode("ASCII-8BIT").force_encoding("UTF-8"))
avatar.rewind
upload = Upload.create_for(newuser.id, avatar, "avatar.jpg", avatar.size)
upload = UploadCreator.new(avatar, "avatar.jpg").create_for(newuser.id)
if upload.persisted?
newuser.create_user_avatar
newuser.user_avatar.update(custom_upload_id: upload.id)

4
script/import_scripts/vbulletin.rb
View File

@ -201,7 +201,7 @@ EOM
file.write(picture["filedata"].encode("ASCII-8BIT").force_encoding("UTF-8"))
file.rewind
upload = Upload.create_for(imported_user.id, file, picture["filename"], file.size)
upload = UploadCreator.new(file, picture["filename"]).create_for(imported_user.id)
return if !upload.persisted?
@ -231,7 +231,7 @@ EOM
file.write(background["filedata"].encode("ASCII-8BIT").force_encoding("UTF-8"))
file.rewind
upload = Upload.create_for(imported_user.id, file, background["filename"], file.size)
upload = UploadCreator.new(file, background["filename"]).create_for(imported_user.id)
return if !upload.persisted?

46
script/import_scripts/vbulletin5.rb
View File

@ -9,7 +9,7 @@ class ImportScripts::VBulletin < ImportScripts::Base
# CHANGE THESE BEFORE RUNNING THE IMPORTER
DATABASE = "yourforum"
TIMEZONE = "America/Los_Angeles"
TIMEZONE = "America/Los_Angeles"
ATTACHMENT_DIR = '/home/discourse/yourforum/customattachments/'
AVATAR_DIR = '/home/discourse/yourforum/avatars/'
@ -25,7 +25,7 @@ class ImportScripts::VBulletin < ImportScripts::Base
@client = Mysql2::Client.new(
host: "localhost",
username: "root",
database: DATABASE,
database: DATABASE,
password: "password"
)
@ -123,7 +123,7 @@ class ImportScripts::VBulletin < ImportScripts::Base
file = Tempfile.new("profile-picture")
file.write(picture["filedata"].encode("ASCII-8BIT").force_encoding("UTF-8"))
file.rewind
upload = Upload.create_for(imported_user.id, file, picture["filename"], file.size)
upload = UploadCreator.new(file, picture["filename"]).create_for(imported_user.id)
else
filename = File.join(AVATAR_DIR, picture['filename'])
unless File.exists?(filename)
@ -160,7 +160,7 @@ class ImportScripts::VBulletin < ImportScripts::Base
file.write(background["filedata"].encode("ASCII-8BIT").force_encoding("UTF-8"))
file.rewind
upload = Upload.create_for(imported_user.id, file, background["filename"], file.size)
upload = UploadCreator.new(file, background["filename"]).create_for(imported_user.id)
return if !upload.persisted?
@ -173,13 +173,13 @@ class ImportScripts::VBulletin < ImportScripts::Base
def import_categories
puts "", "importing top level categories..."
categories = mysql_query("SELECT nodeid AS forumid, title, description, displayorder, parentid
FROM #{DBPREFIX}node
WHERE parentid=#{ROOT_NODE}
UNION
SELECT nodeid, title, description, displayorder, parentid
FROM #{DBPREFIX}node
WHERE contenttypeid = 23
categories = mysql_query("SELECT nodeid AS forumid, title, description, displayorder, parentid
FROM #{DBPREFIX}node
WHERE parentid=#{ROOT_NODE}
UNION
SELECT nodeid, title, description, displayorder, parentid
FROM #{DBPREFIX}node
WHERE contenttypeid = 23
AND parentid IN (SELECT nodeid FROM #{DBPREFIX}node WHERE parentid=#{ROOT_NODE})").to_a
top_level_categories = categories.select { |c| c["parentid"] == ROOT_NODE }
@ -222,17 +222,17 @@ class ImportScripts::VBulletin < ImportScripts::Base
# keep track of closed topics
@closed_topic_ids = []
topic_count = mysql_query("select count(nodeid) cnt from #{DBPREFIX}node where parentid in (
topic_count = mysql_query("select count(nodeid) cnt from #{DBPREFIX}node where parentid in (
select nodeid from #{DBPREFIX}node where contenttypeid=23 ) and contenttypeid=22;").first["cnt"]
batches(BATCH_SIZE) do |offset|
topics = mysql_query <<-SQL
SELECT t.nodeid AS threadid, t.title, t.parentid AS forumid,t.open,t.userid AS postuserid,t.publishdate AS dateline,
nv.count views, 1 AS visible, t.sticky,
nv.count views, 1 AS visible, t.sticky,
CONVERT(CAST(rawtext AS BINARY)USING utf8) AS raw
FROM #{DBPREFIX}node t
LEFT JOIN #{DBPREFIX}nodeview nv ON nv.nodeid=t.nodeid
LEFT JOIN #{DBPREFIX}text txt ON txt.nodeid=t.nodeid
FROM #{DBPREFIX}node t
LEFT JOIN #{DBPREFIX}nodeview nv ON nv.nodeid=t.nodeid
LEFT JOIN #{DBPREFIX}text txt ON txt.nodeid=t.nodeid
WHERE t.parentid in ( select nodeid from #{DBPREFIX}node where contenttypeid=23 )
AND t.contenttypeid = 22
ORDER BY t.nodeid
@ -275,17 +275,17 @@ class ImportScripts::VBulletin < ImportScripts::Base
rescue
end
post_count = mysql_query("SELECT COUNT(nodeid) cnt FROM #{DBPREFIX}node WHERE parentid NOT IN (
post_count = mysql_query("SELECT COUNT(nodeid) cnt FROM #{DBPREFIX}node WHERE parentid NOT IN (
SELECT nodeid FROM #{DBPREFIX}node WHERE contenttypeid=23 ) AND contenttypeid=22;").first["cnt"]
batches(BATCH_SIZE) do |offset|
posts = mysql_query <<-SQL
SELECT p.nodeid AS postid, p.userid AS userid, p.parentid AS threadid,
SELECT p.nodeid AS postid, p.userid AS userid, p.parentid AS threadid,
CONVERT(CAST(rawtext AS BINARY)USING utf8) AS raw, p.publishdate AS dateline,
1 AS visible, p.parentid AS parentid
FROM #{DBPREFIX}node p
LEFT JOIN #{DBPREFIX}nodeview nv ON nv.nodeid=p.nodeid
LEFT JOIN #{DBPREFIX}text txt ON txt.nodeid=p.nodeid
FROM #{DBPREFIX}node p
LEFT JOIN #{DBPREFIX}nodeview nv ON nv.nodeid=p.nodeid
LEFT JOIN #{DBPREFIX}text txt ON txt.nodeid=p.nodeid
WHERE p.parentid NOT IN ( select nodeid from #{DBPREFIX}node where contenttypeid=23 )
AND p.contenttypeid = 22
ORDER BY postid
@ -299,7 +299,7 @@ class ImportScripts::VBulletin < ImportScripts::Base
# next if all_records_exist? :posts, posts.map {|p| p["postid"] }
create_posts(posts, total: post_count, offset: offset) do |post|
raw = preprocess_post_raw(post["raw"])
raw = preprocess_post_raw(post["raw"])
next if raw.blank?
next unless topic = topic_lookup_from_imported_post_id("thread-#{post["threadid"]}")
p = {
@ -336,7 +336,7 @@ class ImportScripts::VBulletin < ImportScripts::Base
real_filename.prepend SecureRandom.hex if real_filename[0] == '.'
unless File.exists?(filename)
if row['dbsize'].to_i == 0
if row['dbsize'].to_i == 0
puts "Attachment file #{row['filedataid']} doesn't exist"
return nil
end