DEV: speed up posts base imports

lib/post_creator.rb

@@ -174,15 +174,14 @@ class PostCreator
         update_user_counts
         create_embedded_topic
         link_post_uploads
-
         ensure_in_allowed_users if guardian.is_staff?
         unarchive_message
-        @post.advance_draft_sequence
+        @post.advance_draft_sequence unless @opts[:import_mode]
         @post.save_reply_relationships
       end
     end
 
-    if @post && errors.blank?
+    if @post && errors.blank? && !@opts[:import_mode]
       # update counters etc.
       @post.topic.reload
 
@@ -194,12 +193,10 @@ class PostCreator
 
       trigger_after_events unless opts[:skip_events]
 
-      auto_close unless @opts[:import_mode]
+      auto_close
     end
 
-    if @post || @spam
+    handle_spam if !opts[:import_mode] && (@post || @spam)
-      handle_spam unless @opts[:import_mode]
-    end
 
     @post
   end
@@ -428,6 +425,8 @@ class PostCreator
   end
 
   def update_topic_auto_close
+    return if @opts[:import_mode]
+
     if @topic.closed?
       @topic.delete_topic_timer(TopicTimer.types[:close])
     else
@@ -510,9 +509,7 @@ class PostCreator
   end
 
   def publish
-    return if @opts[:import_mode]
+    return if @opts[:import_mode] || @post.post_number == 1
-    return unless @post.post_number > 1
-
     @post.publish_change_to_clients! :created
   end
 
@@ -522,7 +519,7 @@ class PostCreator
   end
 
   def track_topic
-    return if @opts[:auto_track] == false
+    return if @opts[:import_mode] || @opts[:auto_track] == false
 
     unless @user.user_option.disable_jump_reply?
       TopicUser.change(@post.user_id,
@@ -540,8 +537,7 @@ class PostCreator
 
     if @user.staged
       TopicUser.auto_notification_for_staging(@user.id, @topic.id, TopicUser.notification_reasons[:auto_watch])
-    else
+    elsif !@topic.private_message?
-      return if @topic.private_message?
       notification_level = @user.user_option.notification_level_when_replying || NotificationLevels.topic_levels[:tracking]
       TopicUser.auto_notification(@user.id, @topic.id, TopicUser.notification_reasons[:created_post], notification_level)
     end

lib/upload_creator.rb

@@ -80,8 +80,8 @@ class UploadCreator
       end
 
       fixed_original_filename = nil
-      if is_image
 
+      if is_image
         current_extension = File.extname(@filename).downcase.sub("jpeg", "jpg")
         expected_extension = ".#{image_type}".downcase.sub("jpeg", "jpg")
 
@@ -89,11 +89,7 @@ class UploadCreator
         # otherwise validation will fail and we can not save
         # TODO decide if we only run the validation on the extension
         if current_extension != expected_extension
-          basename = File.basename(@filename, current_extension)
+          basename = File.basename(@filename, current_extension).presence || "image"
-
-          if basename.length == 0
-            basename = "image"
-          end
           fixed_original_filename = "#{basename}#{expected_extension}"
         end
       end
@@ -173,10 +169,7 @@ class UploadCreator
   MIN_CONVERT_TO_JPEG_SAVING_RATIO = 0.70
 
   def convert_to_jpeg!
-
+    return if filesize < MIN_CONVERT_TO_JPEG_BYTES_SAVED
-    if filesize < MIN_CONVERT_TO_JPEG_BYTES_SAVED
-      return
-    end
 
     jpeg_tempfile = Tempfile.new(["image", ".jpg"])
 
@@ -290,7 +283,6 @@ class UploadCreator
 
   def crop!
     max_pixel_ratio = Discourse::PIXEL_RATIOS.max
-
     filename_with_correct_ext = "image.#{@image_info.type}"
 
     case @opts[:type]

script/import_scripts/base.rb

@@ -54,6 +54,8 @@ class ImportScripts::Base
       update_last_posted_at
       update_last_seen_at
       update_user_stats
+      update_topic_users
+      update_post_timings
       update_feature_topic_users
       update_category_featured_topics
       update_topic_count_replies
@@ -313,7 +315,7 @@ class ImportScripts::Base
 
     unless opts[:email][EmailValidator.email_regex]
       opts[:email] = fake_email
-      puts "Invalid email #{original_email} for #{opts[:username]}. Using: #{opts[:email]}"
+      puts "Invalid email '#{original_email}' for '#{opts[:username]}'. Using '#{opts[:email]}'"
     end
 
     opts[:name] = original_username if original_name.blank? && opts[:username] != original_username
@@ -326,7 +328,7 @@ class ImportScripts::Base
     u = User.new(opts)
     (opts[:custom_fields] || {}).each { |k, v| u.custom_fields[k] = v }
     u.custom_fields["import_id"] = import_id
-    u.custom_fields["import_username"] = opts[:username] if original_username.present?
+    u.custom_fields["import_username"] = original_username if original_username.present? && original_username != opts[:username]
     u.custom_fields["import_avatar_url"] = avatar_url if avatar_url.present?
     u.custom_fields["import_pass"] = opts[:password] if opts[:password].present?
     u.custom_fields["import_email"] = original_email if original_email != opts[:email]
@@ -506,21 +508,19 @@ class ImportScripts::Base
         import_id = params.delete(:id).to_s
 
         if post_id_from_imported_post_id(import_id)
-          skipped += 1 # already imported this post
+          skipped += 1
         else
           begin
             new_post = create_post(params, import_id)
             if new_post.is_a?(Post)
               add_post(import_id, new_post)
               add_topic(new_post)
-
               created_post(new_post)
-
               created += 1
             else
               skipped += 1
               puts "Error creating post #{import_id}. Skipping."
-              puts new_post.inspect
+              p new_post
             end
           rescue Discourse::InvalidAccess => e
             skipped += 1
@@ -632,7 +632,7 @@ class ImportScripts::Base
   def update_topic_status
     puts "", "Updating topic status"
 
-    DB.exec(<<~SQL)
+    DB.exec <<~SQL
       UPDATE topics AS t
       SET closed = TRUE
       WHERE EXISTS(
@@ -642,7 +642,7 @@ class ImportScripts::Base
       )
     SQL
 
-    DB.exec(<<~SQL)
+    DB.exec <<~SQL
       UPDATE topics AS t
       SET archived = TRUE
       WHERE EXISTS(
@@ -652,7 +652,7 @@ class ImportScripts::Base
       )
     SQL
 
-    DB.exec(<<~SQL)
+    DB.exec <<~SQL
       DELETE FROM topic_custom_fields
       WHERE name IN ('import_closed', 'import_archived')
     SQL
@@ -660,13 +660,16 @@ class ImportScripts::Base
 
   def update_bumped_at
     puts "", "Updating bumped_at on topics"
-    DB.exec("update topics t set bumped_at = COALESCE((select max(created_at) from posts where topic_id = t.id and post_type = #{Post.types[:regular]}), bumped_at)")
+    DB.exec <<~SQL
+      UPDATE topics t
+         SET bumped_at = COALESCE((SELECT MAX(created_at) FROM posts WHERE topic_id = t.id AND post_type = #{Post.types[:regular]}), bumped_at)
+    SQL
   end
 
   def update_last_posted_at
     puts "", "Updating last posted at on users"
 
-    sql = <<-SQL
+    DB.exec <<~SQL
       WITH lpa AS (
         SELECT user_id, MAX(posts.created_at) AS last_posted_at
         FROM posts
@@ -679,8 +682,6 @@ class ImportScripts::Base
       WHERE u1.id = users.id
         AND users.last_posted_at <> lpa.last_posted_at
     SQL
-
-    DB.exec(sql)
   end
 
   def update_user_stats
@@ -699,7 +700,7 @@ class ImportScripts::Base
 
     puts "", "Updating first_post_created_at..."
 
-    sql = <<-SQL
+    DB.exec <<~SQL
       WITH sub AS (
         SELECT user_id, MIN(posts.created_at) AS first_post_created_at
         FROM posts
@@ -713,11 +714,9 @@ class ImportScripts::Base
         AND user_stats.first_post_created_at <> sub.first_post_created_at
     SQL
 
-    DB.exec(sql)
-
     puts "", "Updating user post_count..."
 
-    sql = <<-SQL
+    DB.exec <<~SQL
       WITH sub AS (
         SELECT user_id, COUNT(*) AS post_count
         FROM posts
@@ -731,11 +730,9 @@ class ImportScripts::Base
         AND user_stats.post_count <> sub.post_count
     SQL
 
-    DB.exec(sql)
-
     puts "", "Updating user topic_count..."
 
-    sql = <<-SQL
+    DB.exec <<~SQL
       WITH sub AS (
         SELECT user_id, COUNT(*) AS topic_count
         FROM topics
@@ -748,8 +745,6 @@ class ImportScripts::Base
       WHERE u1.user_id = user_stats.user_id
         AND user_stats.topic_count <> sub.topic_count
     SQL
-
-    DB.exec(sql)
   end
 
   # scripts that are able to import last_seen_at from the source data should override this method
@@ -760,6 +755,31 @@ class ImportScripts::Base
     DB.exec("UPDATE users SET last_seen_at = last_posted_at WHERE last_posted_at IS NOT NULL")
   end
 
+  def update_topic_users
+    puts "", "Updating topic users"
+
+    DB.exec <<~SQL
+      INSERT INTO topic_users (user_id, topic_id, posted, last_read_post_number, highest_seen_post_number, first_visited_at, last_visited_at, total_msecs_viewed)
+           SELECT user_id, topic_id, 't' , MAX(post_number), MAX(post_number), MIN(created_at), MAX(created_at), COUNT(id) * 5000
+             FROM posts
+            WHERE user_id > 0
+         GROUP BY user_id, topic_id
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def update_post_timings
+    puts "", "Updating post timings"
+
+    DB.exec <<~SQL
+      INSERT INTO post_timings (topic_id, post_number, user_id, msecs)
+           SELECT topic_id, post_number, user_id, 5000
+             FROM posts
+            WHERE user_id > 0
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
   def update_feature_topic_users
     puts "", "Updating featured topic users"