From 888b635cfc65ca1822801e39bed612cbf0ad136f Mon Sep 17 00:00:00 2001 From: Gerhard Schlager Date: Tue, 13 Aug 2019 17:24:22 +0200 Subject: [PATCH] Import avatars and likes in the Zendesk AP importer Co-authored-by: Justin DiRose --- .../import_scripts/base/generic_database.rb | 93 +++++++++++--- script/import_scripts/zendesk_api.rb | 121 +++++++++++++++--- 2 files changed, 176 insertions(+), 38 deletions(-) diff --git a/script/import_scripts/base/generic_database.rb b/script/import_scripts/base/generic_database.rb index 885d5bb7a4e..2b10e0c25ad 100644 --- a/script/import_scripts/base/generic_database.rb +++ b/script/import_scripts/base/generic_database.rb @@ -14,6 +14,7 @@ module ImportScripts configure_database create_category_table + create_like_table create_user_table create_topic_table create_post_table @@ -34,37 +35,64 @@ module ImportScripts SQL end + def insert_like(like) + @db.execute(<<-SQL, prepare(like)) + INSERT OR REPLACE INTO like (id, user_id, post_id, topic) + VALUES (:id, :user_id, :post_id, :topic) + SQL + end + def insert_topic(topic) + like_user_ids = topic.delete(:like_user_ids) attachments = topic.delete(:attachments) topic[:upload_count] = attachments&.size || 0 - @db.execute(<<-SQL, prepare(topic)) - INSERT OR REPLACE INTO topic (id, title, raw, category_id, closed, user_id, created_at, url, upload_count) - VALUES (:id, :title, :raw, :category_id, :closed, :user_id, :created_at, :url, :upload_count) - SQL - - attachments&.each do |attachment| - @db.execute(<<-SQL, topic_id: topic[:id], path: attachment) - INSERT OR REPLACE INTO topic_upload (topic_id, path) - VALUES (:topic_id, :path) + @db.transaction do + @db.execute(<<-SQL, prepare(topic)) + INSERT OR REPLACE INTO topic (id, title, raw, category_id, closed, user_id, created_at, url, upload_count) + VALUES (:id, :title, :raw, :category_id, :closed, :user_id, :created_at, :url, :upload_count) SQL + + attachments&.each do |attachment| + @db.execute(<<-SQL, topic_id: topic[:id], path: attachment) + INSERT OR REPLACE INTO topic_upload (topic_id, path) + VALUES (:topic_id, :path) + SQL + end + + like_user_ids&.each do |user_id| + @db.execute(<<-SQL, topic_id: topic[:id], user_id: user_id) + INSERT OR REPLACE INTO like (topic_id, user_id) + VALUES (:topic_id, :user_id) + SQL + end end end def insert_post(post) + like_user_ids = post.delete(:like_user_ids) attachments = post.delete(:attachments) post[:upload_count] = attachments&.size || 0 - @db.execute(<<-SQL, prepare(post)) - INSERT OR REPLACE INTO post (id, raw, topic_id, user_id, created_at, reply_to_post_id, url, upload_count) - VALUES (:id, :raw, :topic_id, :user_id, :created_at, :reply_to_post_id, :url, :upload_count) - SQL - - attachments&.each do |attachment| - @db.execute(<<-SQL, post_id: post[:id], path: attachment) - INSERT OR REPLACE INTO post_upload (post_id, path) - VALUES (:post_id, :path) + @db.transaction do + @db.execute(<<-SQL, prepare(post)) + INSERT OR REPLACE INTO post (id, raw, topic_id, user_id, created_at, reply_to_post_id, url, upload_count) + VALUES (:id, :raw, :topic_id, :user_id, :created_at, :reply_to_post_id, :url, :upload_count) SQL + + attachments&.each do |attachment| + @db.execute(<<-SQL, post_id: post[:id], path: attachment) + INSERT OR REPLACE INTO post_upload (post_id, path) + VALUES (:post_id, :path) + SQL + end + + like_user_ids&.each do |user_id| + @db.execute(<<-SQL, post_id: post[:id], user_id: user_id) + INSERT OR REPLACE INTO like (post_id, user_id) + VALUES (:post_id, :user_id) + SQL + end end end @@ -196,6 +224,25 @@ module ImportScripts SQL end + def count_likes + @db.get_first_value(<<-SQL) + SELECT COUNT(*) + FROM like + SQL + end + + def fetch_likes(last_row_id) + rows = @db.execute(<<-SQL, last_row_id) + SELECT ROWID AS rowid, * + FROM like + WHERE ROWID > :last_row_id + ORDER BY ROWID + LIMIT #{@batch_size} + SQL + + add_last_column_value(rows, 'rowid') + end + def execute_sql(sql) @db.execute(sql) end @@ -227,6 +274,16 @@ module ImportScripts SQL end + def create_like_table + @db.execute <<-SQL + CREATE TABLE IF NOT EXISTS like ( + user_id #{key_data_type} NOT NULL, + topic_id #{key_data_type}, + post_id #{key_data_type} + ) + SQL + end + def create_user_table @db.execute <<-SQL CREATE TABLE IF NOT EXISTS user ( diff --git a/script/import_scripts/zendesk_api.rb b/script/import_scripts/zendesk_api.rb index 43749dc2712..7d16be87e0d 100644 --- a/script/import_scripts/zendesk_api.rb +++ b/script/import_scripts/zendesk_api.rb @@ -30,12 +30,22 @@ class ImportScripts::ZendeskApi < ImportScripts::Base import_users import_topics import_posts + import_likes end def fetch_from_api + fetch_categories + fetch_topics + fetch_posts + fetch_users + + @db.sort_posts_by_created_at + end + + def fetch_categories puts '', 'fetching categories...' - get_from_api('/api/v2/community/topics.json', 'topics') do |row| + get_from_api('/api/v2/community/topics.json', 'topics', show_status: true) do |row| @db.insert_category( id: row['id'], name: row['name'], @@ -44,10 +54,16 @@ class ImportScripts::ZendeskApi < ImportScripts::Base url: row['html_url'] ) end + end + def fetch_topics puts '', 'fetching topics...' - get_from_api('/api/v2/community/posts.json', 'posts') do |row| + get_from_api('/api/v2/community/posts.json', 'posts', show_status: true) do |row| + if row['vote_count'] > 0 + like_user_ids = fetch_likes("/api/v2/community/posts/#{row['id']}/votes.json") + end + @db.insert_topic( id: row['id'], title: row['title'], @@ -56,11 +72,15 @@ class ImportScripts::ZendeskApi < ImportScripts::Base closed: row['closed'], user_id: row['author_id'], created_at: row['created_at'], - url: row['html_url'] + url: row['html_url'], + like_user_ids: like_user_ids ) end + end + def fetch_posts puts '', 'fetching posts...' + current_count = 0 total_count = @db.count_topics start_time = Time.now last_id = '' @@ -69,49 +89,72 @@ class ImportScripts::ZendeskApi < ImportScripts::Base rows, last_id = @db.fetch_topics(last_id) break if rows.empty? - print_status(offset, total_count, start_time) - rows.each do |topic_row| - get_from_api("/api/v2/community/posts/#{topic_row['id']}/comments.json", 'comments', show_status: false) do |row| + get_from_api("/api/v2/community/posts/#{topic_row['id']}/comments.json", 'comments') do |row| + if row['vote_count'] > 0 + like_user_ids = fetch_likes("/api/v2/community/posts/#{topic_row['id']}/comments/#{row['id']}/votes.json") + end + @db.insert_post( id: row['id'], raw: row['body'], topic_id: topic_row['id'], user_id: row['author_id'], created_at: row['created_at'], - url: row['html_url'] + url: row['html_url'], + like_user_ids: like_user_ids ) end + + current_count += 1 + print_status(current_count, total_count, start_time) end end + end + def fetch_users puts '', 'fetching users...' - results = @db.execute_sql("SELECT user_id FROM topic") - user_ids = results.map { |h| h['user_id']&.to_i } - results = @db.execute_sql("SELECT user_id FROM post") - user_ids += results.map { |h| h['user_id']&.to_i } - user_ids.uniq! - user_ids.sort! + user_ids = @db.execute_sql(<<~SQL).map { |row| row['user_id'] } + SELECT user_id FROM topic + UNION + SELECT user_id FROM post + UNION + SELECT user_id FROM like + SQL - total_users = user_ids.size + current_count = 0 + total_count = user_ids.size start_time = Time.now while !user_ids.empty? - print_status(total_users - user_ids.size, total_users, start_time) - get_from_api("/api/v2/users/show_many.json?ids=#{user_ids.shift(50).join(',')}", 'users', show_status: false) do |row| + get_from_api("/api/v2/users/show_many.json?ids=#{user_ids.shift(50).join(',')}", 'users') do |row| @db.insert_user( id: row['id'], email: row['email'], name: row['name'], created_at: row['created_at'], last_seen_at: row['last_login_at'], - active: row['active'] + active: row['active'], + avatar_path: row['photo'].present? ? row['photo']['content_url'] : nil ) + + current_count += 1 + print_status(current_count, total_count, start_time) + end + end + end + + def fetch_likes(url) + user_ids = [] + + get_from_api(url, 'votes') do |row| + if row['id'].present? && row['value'] == 1 + user_ids << row['user_id'] end end - @db.sort_posts_by_created_at + user_ids end def import_categories @@ -150,7 +193,12 @@ class ImportScripts::ZendeskApi < ImportScripts::Base name: row['name'], created_at: row['created_at'], last_seen_at: row['last_seen_at'], - active: row['active'] == 1 + active: row['active'] == 1, + post_create_action: proc do |user| + if row['avatar_path'].present? + UserAvatar.import_url_for_user(row['avatar_path'], user) rescue nil + end + end } end end @@ -222,6 +270,38 @@ class ImportScripts::ZendeskApi < ImportScripts::Base end end + def import_likes + puts "", "importing likes..." + start_time = Time.now + current_count = 0 + total_count = @db.count_likes + last_row_id = 0 + + batches do |offset| + rows, last_row_id = @db.fetch_likes(last_row_id) + break if rows.empty? + + rows.each do |row| + import_id = row['topic_id'] ? import_topic_id(row['topic_id']) : row['post_id'] + post = Post.find_by(id: post_id_from_imported_post_id(import_id)) if import_id + user = User.find_by(id: user_id_from_imported_user_id(row['user_id'])) + + if post && user + begin + PostActionCreator.like(user, post) if user && post + rescue => e + puts "error acting on post #{e}" + end + else + puts "Skipping Like from #{row['user_id']} on topic #{row['topic_id']} / post #{row['post_id']}" + end + + current_count += 1 + print_status(current_count, total_count, start_time) + end + end + end + def normalize_raw(raw) raw = raw.gsub('\n', '') raw = ReverseMarkdown.convert(raw) @@ -256,7 +336,7 @@ class ImportScripts::ZendeskApi < ImportScripts::Base end end - def get_from_api(path, array_name, show_status: true) + def get_from_api(path, array_name, show_status: false) url = "#{@source_url}#{path}" start_time = Time.now @@ -298,6 +378,7 @@ class ImportScripts::ZendeskApi < ImportScripts::Base end end end + end unless ARGV.length == 4 && Dir.exist?(ARGV[1])