update bbcode dialect and fix vBulletin importer

script/import_scripts/base.rb

@@ -235,9 +235,11 @@ class ImportScripts::Base
 
   def create_user(opts, import_id)
     opts.delete(:id)
+    merge = opts.delete(:merge)
     post_create_action = opts.delete(:post_create_action)
+
     existing = User.where(email: opts[:email].downcase, username: opts[:username]).first
-    return existing if existing && existing.custom_fields["import_id"].to_i == import_id.to_i
+    return existing if existing && (merge || existing.custom_fields["import_id"].to_i == import_id.to_i)
 
     bio_raw = opts.delete(:bio_raw)
     website = opts.delete(:website)
@@ -434,12 +436,12 @@ class ImportScripts::Base
   end
 
   def update_bumped_at
-    puts "updating bumped_at on topics"
+    puts "", "updating bumped_at on topics"
     Post.exec_sql("update topics t set bumped_at = (select max(created_at) from posts where topic_id = t.id and post_type != #{Post.types[:moderator_action]})")
   end
 
   def update_feature_topic_users
-    puts "updating featured topic users"
+    puts "", "updating featured topic users"
 
     total_count = Topic.count
     progress_count = 0
@@ -452,7 +454,7 @@ class ImportScripts::Base
   end
 
   def update_category_featured_topics
-    puts "updating featured topics in categories"
+    puts "", "updating featured topics in categories"
 
     total_count = Category.count
     progress_count = 0
@@ -465,7 +467,7 @@ class ImportScripts::Base
   end
 
   def update_topic_count_replies
-    puts "updating user topic reply counts"
+    puts "", "updating user topic reply counts"
 
     total_count = User.real.count
     progress_count = 0

script/import_scripts/vbulletin.rb

@@ -243,6 +243,9 @@ class ImportScripts::VBulletin < ImportScripts::Base
     def import_groups
       puts "", "Importing groups..."
 
+      # sort the groups
+      @groups.sort_by! { |group| group[:usergroupid].to_i }
+
       create_groups(@groups) do |group|
         {
           id: group[:usergroupid],
@@ -255,10 +258,13 @@ class ImportScripts::VBulletin < ImportScripts::Base
     def import_users
       puts "", "Importing users..."
 
+      # sort the users
+      @users.sort_by! { |user| user[:userid].to_i }
+
       @old_username_to_new_usernames = {}
 
       create_users(@users) do |user|
-        @old_username_to_new_usernames[user[:username]] = UserNameSuggester.suggest(user[:username])
+        @old_username_to_new_usernames[user[:username]] = UserNameSuggester.fix_username(user[:username])
 
         {
           id: user[:userid],
@@ -267,6 +273,7 @@ class ImportScripts::VBulletin < ImportScripts::Base
           website: user[:homepage],
           title: user[:usertitle],
           primary_group_id: group_id_from_imported_group_id(user[:usergroupid]),
+          merge: true,
         }
       end
 
@@ -300,6 +307,9 @@ class ImportScripts::VBulletin < ImportScripts::Base
     def import_categories
       puts "", "Importing categories..."
 
+      # sort categories
+      @categories.sort_by! { |category| category[:forumid].to_i }
+
       create_categories(@categories) do |category|
         {
           id: category[:forumid],
@@ -343,22 +353,18 @@ class ImportScripts::VBulletin < ImportScripts::Base
       raw = raw.gsub(/\[mention\](.+?)\[\/mention\]/i) do
         old_username = $1
         if @old_username_to_new_usernames.has_key?(old_username)
-          username = @old_username_to_new_usernames[old_username]
-          "@#{username}"
-        else
-          $&
+          old_username = @old_username_to_new_usernames[old_username]
         end
+        "@#{old_username}"
       end
 
-      # [MENTION=<user_id>]...[/MENTION]
-      raw = raw.gsub(/\[mention=(\d+)\].+?\[\/mention\]/i) do
-        user_id = $1
+      # [MENTION=<user_id>]<username>[/MENTION]
+      raw = raw.gsub(/\[mention=(\d+)\](.+?)\[\/mention\]/i) do
+        user_id, old_username = $1, $2
         if user = @users.select { |u| u[:userid] == user_id }.first
-          username = @old_username_to_new_usernames[user[:username]] || user[:username]
-          "@#{username}"
-        else
-          $&
+          old_username = @old_username_to_new_usernames[user[:username]] || user[:username]
         end
+        "@#{old_username}"
       end
 
       # [QUOTE]...[/QUOTE]
@@ -368,11 +374,9 @@ class ImportScripts::VBulletin < ImportScripts::Base
       raw = raw.gsub(/\[quote=([^;\]]+)\](.+?)\[\/quote\]/im) do
         old_username, quote = $1, $2
         if @old_username_to_new_usernames.has_key?(old_username)
-          username = @old_username_to_new_usernames[old_username]
-          "\n[quote=\"#{username}\"]\n#{quote}\n[/quote]\n"
-        else
-          $&
+          old_username = @old_username_to_new_usernames[old_username]
         end
+        "\n[quote=\"#{old_username}\"]\n#{quote}\n[/quote]\n"
       end
 
       # [HTML]...[/HTML]
@@ -399,12 +403,24 @@ class ImportScripts::VBulletin < ImportScripts::Base
       # [MP3]<url>[/MP3]
       raw = raw.gsub(/\[MP3\](.+?)\[\/MP3\]/i) { "\n#{$1}\n" }
 
+      # replace all chevrons with HTML entities
+      raw = raw.gsub(/`([^`]+)`/im) { "`" + $1.gsub("<", "\u2603") + "`" }
+               .gsub("<", "&lt;")
+               .gsub("\u2603", "<")
+
+      raw = raw.gsub(/`([^`]+)`/im) { "`" + $1.gsub(">", "\u2603") + "`" }
+               .gsub(">", "&gt;")
+               .gsub("\u2603", ">")
+
       raw
     end
 
     def import_topics
       puts "", "Importing topics..."
 
+      # sort the topics
+      @topics.sort_by! { |topic| topic[:threadid].to_i }
+
       create_posts(@topics) do |topic|
         next unless post = @posts.select { |p| p[:postid] == topic[:firstpostid] }.first
 
@@ -441,8 +457,11 @@ class ImportScripts::VBulletin < ImportScripts::Base
       first_post_ids = Set.new(@topics.map { |t| t[:firstpostid] })
       @posts.reject! { |post| first_post_ids.include?(post[:postid]) }
 
+      # sort the posts
+      @posts.sort_by! { |post| post[:postid].to_i }
+
       create_posts(@posts) do |post|
-        t = topic_lookup_from_imported_post_id("thread#" + post[:threadid])
+        next unless t = topic_lookup_from_imported_post_id("thread#" + post[:threadid])
 
         p = {
           id: post[:postid],
@@ -487,13 +506,17 @@ class ImportScripts::VBulletin < ImportScripts::Base
       # [QUOTE=<username>;<post_id>]...[/QUOTE]
       raw = raw.gsub(/\[quote=([^;]+);(\d+)\](.+?)\[\/quote\]/im) do
         old_username, post_id, quote = $1, $2, $3
-        if @old_username_to_new_usernames.has_key?(old_username) && topic_lookup = topic_lookup_from_imported_post_id(post_id)
+
+        if @old_username_to_new_usernames.has_key?(old_username)
+          old_username = @old_username_to_new_usernames[old_username]
+        end
+
+        if topic_lookup = topic_lookup_from_imported_post_id(post_id)
           post_number = topic_lookup[:post_number]
           topic_id    = topic_lookup[:topic_id]
-          username    = @old_username_to_new_usernames[old_username]
-          "\n[quote=\"#{username},post:#{post_number},topic:#{topic_id}\"]\n#{quote}\n[/quote]\n"
+          "\n[quote=\"#{old_username},post:#{post_number},topic:#{topic_id}\"]\n#{quote}\n[/quote]\n"
         else
-          $&
+          "\n[quote=\"#{old_username}\"]\n#{quote}\n[/quote]\n"
         end
       end