mirror of
https://github.com/discourse/discourse.git
synced 2025-05-30 07:11:34 +08:00
DEV: Apply syntax_tree formatting to script/*
This commit is contained in:
David Taylor
@ -1,7 +1,7 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require File.expand_path(File.dirname(__FILE__) + "/base.rb")
|
||||
require 'mongo'
|
||||
require "mongo"
|
||||
|
||||
# Import YahooGroups data as exported into MongoDB by:
|
||||
# https://github.com/jonbartlett/yahoo-groups-export
|
||||
@ -13,14 +13,13 @@ require 'mongo'
|
||||
# =end
|
||||
|
||||
class ImportScripts::YahooGroup < ImportScripts::Base
|
||||
|
||||
MONGODB_HOST = '192.168.10.1:27017'
|
||||
MONGODB_DB = 'syncro'
|
||||
MONGODB_HOST = "192.168.10.1:27017"
|
||||
MONGODB_DB = "syncro"
|
||||
|
||||
def initialize
|
||||
super
|
||||
|
||||
client = Mongo::Client.new([ MONGODB_HOST ], database: MONGODB_DB)
|
||||
client = Mongo::Client.new([MONGODB_HOST], database: MONGODB_DB)
|
||||
db = client.database
|
||||
Mongo::Logger.logger.level = Logger::FATAL
|
||||
puts "connected to db...."
|
||||
@ -28,7 +27,6 @@ class ImportScripts::YahooGroup < ImportScripts::Base
|
||||
@collection = client[:posts]
|
||||
|
||||
@user_profile_map = {}
|
||||
|
||||
end
|
||||
|
||||
def execute
|
||||
@ -41,43 +39,42 @@ class ImportScripts::YahooGroup < ImportScripts::Base
|
||||
end
|
||||
|
||||
def import_users
|
||||
|
||||
puts '', "Importing users"
|
||||
puts "", "Importing users"
|
||||
|
||||
# fetch distinct list of Yahoo "profile" names
|
||||
profiles = @collection.aggregate(
|
||||
[
|
||||
{ "$group": { "_id": { profile: "$ygData.profile" } } }
|
||||
]
|
||||
)
|
||||
profiles = @collection.aggregate([{ "$group": { _id: { profile: "$ygData.profile" } } }])
|
||||
|
||||
user_id = 0
|
||||
|
||||
create_users(profiles.to_a) do |u|
|
||||
|
||||
user_id = user_id + 1
|
||||
|
||||
# fetch last message for profile to pickup latest user info as this may have changed
|
||||
user_info = @collection.find("ygData.profile": u["_id"]["profile"]).sort("ygData.msgId": -1).limit(1).to_a[0]
|
||||
user_info =
|
||||
@collection
|
||||
.find("ygData.profile": u["_id"]["profile"])
|
||||
.sort("ygData.msgId": -1)
|
||||
.limit(1)
|
||||
.to_a[
|
||||
0
|
||||
]
|
||||
|
||||
# Store user_id to profile lookup
|
||||
@user_profile_map.store(user_info["ygData"]["profile"], user_id)
|
||||
|
||||
puts "User created: #{user_info["ygData"]["profile"]}"
|
||||
|
||||
user =
|
||||
{
|
||||
id: user_id, # yahoo "userId" sequence appears to have changed mid forum life so generate this
|
||||
user = {
|
||||
id: user_id, # yahoo "userId" sequence appears to have changed mid forum life so generate this
|
||||
username: user_info["ygData"]["profile"],
|
||||
name: user_info["ygData"]["authorName"],
|
||||
email: user_info["ygData"]["from"], # mandatory
|
||||
created_at: Time.now
|
||||
created_at: Time.now,
|
||||
}
|
||||
user
|
||||
end
|
||||
|
||||
puts "#{user_id} users created"
|
||||
|
||||
end
|
||||
|
||||
def import_discussions
|
||||
@ -86,21 +83,16 @@ class ImportScripts::YahooGroup < ImportScripts::Base
|
||||
topics_count = 0
|
||||
posts_count = 0
|
||||
|
||||
topics = @collection.aggregate(
|
||||
[
|
||||
{ "$group": { "_id": { topicId: "$ygData.topicId" } } }
|
||||
]
|
||||
).to_a
|
||||
topics = @collection.aggregate([{ "$group": { _id: { topicId: "$ygData.topicId" } } }]).to_a
|
||||
|
||||
# for each distinct topicId found
|
||||
topics.each_with_index do |t, tidx|
|
||||
|
||||
# create "topic" post first.
|
||||
# fetch topic document
|
||||
topic_post = @collection.find("ygData.msgId": t["_id"]["topicId"]).to_a[0]
|
||||
next if topic_post.nil?
|
||||
|
||||
puts "Topic: #{tidx + 1} / #{topics.count()} (#{sprintf('%.2f', ((tidx + 1).to_f / topics.count().to_f) * 100)}%) Subject: #{topic_post["ygData"]["subject"]}"
|
||||
puts "Topic: #{tidx + 1} / #{topics.count()} (#{sprintf("%.2f", ((tidx + 1).to_f / topics.count().to_f) * 100)}%) Subject: #{topic_post["ygData"]["subject"]}"
|
||||
|
||||
if topic_post["ygData"]["subject"].to_s.empty?
|
||||
topic_title = "No Subject"
|
||||
@ -115,8 +107,10 @@ class ImportScripts::YahooGroup < ImportScripts::Base
|
||||
created_at: Time.at(topic_post["ygData"]["postDate"].to_i),
|
||||
cook_method: Post.cook_methods[:raw_html],
|
||||
title: topic_title,
|
||||
category: ENV['CATEGORY_ID'],
|
||||
custom_fields: { import_id: topic_post["ygData"]["msgId"] }
|
||||
category: ENV["CATEGORY_ID"],
|
||||
custom_fields: {
|
||||
import_id: topic_post["ygData"]["msgId"],
|
||||
},
|
||||
}
|
||||
|
||||
topics_count += 1
|
||||
@ -128,34 +122,31 @@ class ImportScripts::YahooGroup < ImportScripts::Base
|
||||
posts = @collection.find("ygData.topicId": topic_post["ygData"]["topicId"]).to_a
|
||||
|
||||
posts.each_with_index do |p, pidx|
|
||||
|
||||
# skip over first post as this is created by topic above
|
||||
next if p["ygData"]["msgId"] == topic_post["ygData"]["topicId"]
|
||||
|
||||
puts " Post: #{pidx + 1} / #{posts.count()}"
|
||||
|
||||
post = {
|
||||
id: pidx + 1,
|
||||
topic_id: parent_post[:topic_id],
|
||||
user_id: @user_profile_map[p["ygData"]["profile"]] || -1,
|
||||
raw: p["ygData"]["messageBody"],
|
||||
created_at: Time.at(p["ygData"]["postDate"].to_i),
|
||||
cook_method: Post.cook_methods[:raw_html],
|
||||
custom_fields: { import_id: p["ygData"]["msgId"] }
|
||||
id: pidx + 1,
|
||||
topic_id: parent_post[:topic_id],
|
||||
user_id: @user_profile_map[p["ygData"]["profile"]] || -1,
|
||||
raw: p["ygData"]["messageBody"],
|
||||
created_at: Time.at(p["ygData"]["postDate"].to_i),
|
||||
cook_method: Post.cook_methods[:raw_html],
|
||||
custom_fields: {
|
||||
import_id: p["ygData"]["msgId"],
|
||||
},
|
||||
}
|
||||
|
||||
child_post = create_post(post, post[:id])
|
||||
|
||||
posts_count += 1
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
puts "", "Imported #{topics_count} topics with #{topics_count + posts_count} posts."
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
ImportScripts::YahooGroup.new.perform
|
||||
|
||||
Reference in New Issue
Block a user