PERF: finalize porting to new incoming links structure

app/controllers/embed_controller.rb

@@ -1,7 +1,6 @@
 class EmbedController < ApplicationController
   skip_before_filter :check_xhr
   skip_before_filter :preload_json
-  skip_before_filter :store_incoming_links
   skip_before_filter :verify_authenticity_token
 
   before_filter :ensure_embeddable

app/controllers/posts_controller.rb

@@ -7,7 +7,6 @@ class PostsController < ApplicationController
   # Need to be logged in for all actions here
   before_filter :ensure_logged_in, except: [:show, :replies, :by_number, :short_link, :reply_history, :revisions, :expand_embed, :markdown, :raw, :cooked]
 
-  skip_before_filter :store_incoming_links, only: [:short_link]
   skip_before_filter :check_xhr, only: [:markdown_id, :markdown_num, :short_link]
 
   def markdown_id

app/controllers/static_controller.rb

@@ -63,7 +63,7 @@ class StaticController < ApplicationController
     )
   end
 
-  skip_before_filter :store_incoming_links, :verify_authenticity_token, only: [:cdn_asset]
+  skip_before_filter :verify_authenticity_token, only: [:cdn_asset]
   def cdn_asset
     path = File.expand_path(Rails.root + "public/assets/" + params[:path])
 

app/controllers/uploads_controller.rb

@@ -1,6 +1,6 @@
 class UploadsController < ApplicationController
   before_filter :ensure_logged_in, except: [:show]
-  skip_before_filter :store_incoming_links, :check_xhr, only: [:show]
+  skip_before_filter :check_xhr, only: [:show]
 
   def create
     file = params[:file] || params[:files].first

app/controllers/user_avatars_controller.rb

@@ -3,7 +3,7 @@ require_dependency 'letter_avatar'
 class UserAvatarsController < ApplicationController
   DOT = Base64.decode64("R0lGODlhAQABALMAAAAAAIAAAACAAICAAAAAgIAAgACAgMDAwICAgP8AAAD/AP//AAAA//8A/wD//wBiZCH5BAEAAA8ALAAAAAABAAEAAAQC8EUAOw==")
 
-  skip_before_filter :store_incoming_links, :redirect_to_login_if_required, :check_xhr, :verify_authenticity_token, only: [:show, :show_letter]
+  skip_before_filter :redirect_to_login_if_required, :check_xhr, :verify_authenticity_token, only: [:show, :show_letter]
 
   def refresh_gravatar
     user = User.find_by(username_lower: params[:username].downcase)

app/models/badge.rb

@@ -86,18 +86,17 @@ SQL
 SQL
 
     FirstShare = <<SQL
-    SELECT views.user_id, p2.id post_id, i2.created_at granted_at
+    SELECT views.user_id, i2.post_id, i2.created_at granted_at
     FROM
     (
       SELECT i.user_id, MIN(i.id) i_id
       FROM incoming_links i
       JOIN topics t on t.id = i.topic_id
-      JOIN badge_posts p on p.topic_id = t.id AND p.post_number = i.post_number
+      JOIN badge_posts p on p.id = i.post_id
       WHERE i.user_id IS NOT NULL
       GROUP BY i.user_id
     ) as views
     JOIN incoming_links i2 ON i2.id = views.i_id
-    JOIN posts p2 on p2.topic_id = i2.topic_id AND p2.post_number = i2.post_number
 SQL
 
     FirstFlag = <<SQL

app/models/incoming_domain.rb

@@ -1,4 +1,32 @@
 class IncomingDomain < ActiveRecord::Base
+  def self.add!(uri)
+    name = uri.host
+    https = uri.scheme == "https"
+    port = uri.port
+
+    current = find_by(name: name, https: https, port: port)
+    return current if current
+
+    # concurrency ...
+
+    begin
+      current = create!(name: name, https: https, port: port)
+    rescue
+      # duplicate key is just ignored
+    end
+
+    current || find_by(name: name, https: https, port: port)
+  end
+
+  def to_url
+    url = "http#{https ? "s" : ""}://#{name}"
+
+    if https && port != 443 || !https && port != 80
+      url << ":#{port}"
+    end
+
+    url
+  end
 end
 
 # == Schema Information

app/models/incoming_link.rb

@@ -1,11 +1,11 @@
 class IncomingLink < ActiveRecord::Base
-  belongs_to :topic
+  belongs_to :post
   belongs_to :user
+  belongs_to :incoming_referer
 
   validate :referer_valid
   validate :post_id, presence: true
 
-  before_validation :extract_domain
   after_create :update_link_counts
 
   attr_accessor :url
@@ -52,15 +52,40 @@ class IncomingLink < ActiveRecord::Base
   end
 
 
-  # Internal: Extract the domain from link.
+  def referer=(referer)
-  def extract_domain
+    self.incoming_referer_id = nil
-    if referer.present?
+
-      # We may get a junk URI, just deal with it
+    # will set incoming_referer_id
-      self.domain = URI.parse(self.referer).host rescue nil
+    unless referer.present?
-      self.referer = nil unless self.domain
+      return
+    end
+
+    parsed = URI.parse(referer)
+
+    if parsed.scheme == "http" || parsed.scheme == "https"
+      domain = IncomingDomain.add!(parsed)
+
+      referer = IncomingReferer.add!(path: parsed.path, incoming_domain: domain) if domain
+      self.incoming_referer_id = referer.id if referer
+    end
+
+  rescue URI::InvalidURIError
+    # ignore
+  end
+
+  def referer
+    if self.incoming_referer
+      self.incoming_referer.incoming_domain.to_url << self.incoming_referer.path
     end
   end
 
+  def domain
+    if incoming_referer
+      incoming_referer.incoming_domain.name
+    end
+  end
+
+
   # Internal: Update appropriate link counts.
   def update_link_counts
     exec_sql("UPDATE topics

app/models/incoming_links_report.rb

@@ -43,7 +43,11 @@ class IncomingLinksReport
   end
 
   def self.per_user
-    @per_user_query ||= IncomingLink.where('incoming_links.created_at > ? AND incoming_links.user_id IS NOT NULL', 30.days.ago).joins(:user).group('users.username')
+    @per_user_query ||= IncomingLink
+        .where('incoming_links.created_at > ? AND incoming_links.user_id IS NOT NULL', 30.days.ago)
+        .joins(:user)
+        .joins(:post)
+        .group('users.username')
   end
 
   def self.link_count_per_user
@@ -51,7 +55,7 @@ class IncomingLinksReport
   end
 
   def self.topic_count_per_user
-    per_user.count('incoming_links.topic_id', distinct: true)
+    per_user.count('topic_id', distinct: true)
   end
 
 
@@ -71,11 +75,19 @@ class IncomingLinksReport
   end
 
   def self.link_count_per_domain(limit=10)
-    IncomingLink.where('created_at > ? AND domain IS NOT NULL', 30.days.ago).group('domain').order('count_all DESC').limit(limit).count
+    IncomingLink.where('incoming_links.created_at > ?', 30.days.ago)
+                .joins(:incoming_referer => :incoming_domain)
+                .group('incoming_domains.name')
+                .order('count_all DESC')
+                .limit(limit).count
   end
 
   def self.per_domain(domains)
-    IncomingLink.where('created_at > ? AND domain IN (?)', 30.days.ago, domains).group('domain')
+    IncomingLink
+        .joins(:incoming_referer => :incoming_domain)
+        .joins(:post)
+        .where('incoming_links.created_at > ? AND incoming_domains.name IN (?)', 30.days.ago, domains)
+        .group('incoming_domains.name')
   end
 
   def self.topic_count_per_domain(domains)
@@ -100,6 +112,9 @@ class IncomingLinksReport
   end
 
   def self.link_count_per_topic
-    IncomingLink.where('created_at > ? AND topic_id IS NOT NULL', 30.days.ago).group('topic_id').count
+    IncomingLink.joins(:post)
+                .where('incoming_links.created_at > ? AND topic_id IS NOT NULL', 30.days.ago)
+                .group('topic_id')
+                .count
   end
 end

app/models/incoming_referer.rb

@@ -1,4 +1,22 @@
 class IncomingReferer < ActiveRecord::Base
+  belongs_to :incoming_domain
+
+  def self.add!(opts)
+    domain_id = opts[:incoming_domain_id]
+    domain_id ||= opts[:incoming_domain].id
+    path = opts[:path]
+
+    current = find_by(path: path, incoming_domain_id: domain_id)
+    return current if current
+
+    begin
+      current = create!(path: path, incoming_domain_id: domain_id)
+    rescue
+      # duplicates
+    end
+
+    current || find_by(path: path, incoming_domain_id: domain_id)
+  end
 end
 
 # == Schema Information

db/migrate/20140804030041_remove_url_from_incoming_referer.rb

@@ -0,0 +1,9 @@
+class RemoveUrlFromIncomingReferer < ActiveRecord::Migration
+  def up
+    remove_column :incoming_referers, :url
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20140804060439_drop_topic_id_from_incoming_links.rb

@@ -0,0 +1,5 @@
+class DropTopicIdFromIncomingLinks < ActiveRecord::Migration
+  def change
+    remove_column :incoming_links, :topic_id
+  end
+end

spec/models/incoming_link_spec.rb

@@ -2,8 +2,6 @@ require 'spec_helper'
 
 describe IncomingLink do
 
-  it { should belong_to :topic }
-
   let(:post) { Fabricate(:post) }
   let(:topic) { post.topic }
 

spec/models/incoming_links_report_spec.rb

@@ -2,6 +2,33 @@ require 'spec_helper'
 
 describe IncomingLinksReport do
 
+  describe 'integration' do
+    it 'runs correctly' do
+      p1 = create_post
+
+      IncomingLink.add(
+        referer: 'http://test.com',
+        host: 'http://boo.com',
+        topic_id: p1.topic.id,
+        ip_address: '10.0.0.2',
+        username: p1.user.username
+      )
+
+
+      c = IncomingLinksReport.link_count_per_topic
+      c[p1.topic_id].should == 1
+
+      c = IncomingLinksReport.link_count_per_domain
+      c["test.com"].should == 1
+
+      c = IncomingLinksReport.topic_count_per_domain(['test.com', 'foo.com'])
+      c["test.com"].should == 1
+
+      c = IncomingLinksReport.topic_count_per_user()
+      c[p1.username].should == 1
+    end
+  end
+
   describe 'top_referrers' do
     subject(:top_referrers) { IncomingLinksReport.find('top_referrers').as_json }