From 10ae7ef44af00b67507856291e1ecd2ce1b86df3 Mon Sep 17 00:00:00 2001 From: Osama Sayegh Date: Wed, 21 Aug 2024 00:03:42 +0300 Subject: [PATCH] FEATURE: Add estimated number of global and EU visitors to the about page (#28382) This commit implements 2 new metrics/stats in the /about page for the _estimated_ numbers of unique visitors from the EU and the rest of the world. This new feature is currently off by default, but it can be enabled by turning on the hidden `display_eu_visitor_stats` site settings via the rails console. There are a number of assumptions that we're making here in order to estimate the number of unique visitors, specifically: 1. we're assuming that the average of page views per anonymous visitor is similar to the average number of page views that a logged-in visitor makes, and 2. we're assuming that the ratio of logged in visitors from the EU is similar to the ratio of anonymous visitors from the EU Discourse keeps track of the number of both logged-in and anonymous page views, and also the number of unique logged-in visitors and where they're from. So with those numbers and the assumptions above, we can estimate the number of unique anonymous visitors from the EU and the rest of the world. Internal topic: t/128480. --- .../discourse/app/components/about-page.gjs | 17 ++ .../discourse/app/controllers/about.js | 11 + .../discourse/app/templates/about.hbs | 20 ++ app/assets/stylesheets/common/base/about.scss | 4 + app/models/application_request.rb | 11 + app/models/stat.rb | 15 +- config/locales/client.en.yml | 18 ++ config/locales/server.en.yml | 1 + config/site_settings.yml | 4 + lib/statistics.rb | 105 ++++++++++ spec/lib/statistics_spec.rb | 188 +++++++++++++++++- spec/system/about_page_spec.rb | 25 +++ .../components/about_page_site_activity.rb | 15 ++ .../about_page_site_activity_item.rb | 4 + 14 files changed, 436 insertions(+), 2 deletions(-) diff --git a/app/assets/javascripts/discourse/app/components/about-page.gjs b/app/assets/javascripts/discourse/app/components/about-page.gjs index 5d929aaa44c..42be96476b9 100644 --- a/app/assets/javascripts/discourse/app/components/about-page.gjs +++ b/app/assets/javascripts/discourse/app/components/about-page.gjs @@ -1,5 +1,6 @@ import Component from "@glimmer/component"; import { hash } from "@ember/helper"; +import { service } from "@ember/service"; import { htmlSafe } from "@ember/template"; import AboutPageUsers from "discourse/components/about-page-users"; import PluginOutlet from "discourse/components/plugin-outlet"; @@ -20,6 +21,8 @@ export function clearAboutPageActivities() { } export default class AboutPage extends Component { + @service siteSettings; + get moderatorsCount() { return this.args.model.moderators.length; } @@ -115,6 +118,20 @@ export default class AboutPage extends Component { }, ]; + if (this.siteSettings.display_eu_visitor_stats) { + list.splice(2, 0, { + icon: "user-secret", + class: "visitors", + activityText: I18n.messageFormat("about.activities.visitors_MF", { + total_count: this.args.model.stats.visitors_7_days, + eu_count: this.args.model.stats.eu_visitors_7_days, + total_formatted_number: number(this.args.model.stats.visitors_7_days), + eu_formatted_number: number(this.args.model.stats.eu_visitors_7_days), + }), + period: I18n.t("about.activities.periods.last_7_days"), + }); + } + return list.concat(this.siteActivitiesFromPlugins()); } diff --git a/app/assets/javascripts/discourse/app/controllers/about.js b/app/assets/javascripts/discourse/app/controllers/about.js index aa203fc2932..6d211c17ed8 100644 --- a/app/assets/javascripts/discourse/app/controllers/about.js +++ b/app/assets/javascripts/discourse/app/controllers/about.js @@ -23,4 +23,15 @@ export default class AboutController extends Controller { return null; } } + + @discourseComputed( + "model.stats.visitors_30_days", + "model.stats.eu_visitors_30_days" + ) + statsTableFooter(all, eu) { + return I18n.messageFormat("about.traffic_info_footer_MF", { + total_visitors: all, + eu_visitors: eu, + }); + } } diff --git a/app/assets/javascripts/discourse/app/templates/about.hbs b/app/assets/javascripts/discourse/app/templates/about.hbs index 023ebf79d27..135aa123cc5 100644 --- a/app/assets/javascripts/discourse/app/templates/about.hbs +++ b/app/assets/javascripts/discourse/app/templates/about.hbs @@ -139,6 +139,22 @@ {{number this.model.stats.active_users_30_days}} — + {{#if this.siteSettings.display_eu_visitor_stats}} + + {{i18n "about.visitor_count"}} + {{number this.model.stats.visitors_last_day}} + {{number this.model.stats.visitors_7_days}} + {{number this.model.stats.visitors_30_days}} + — + + + {{i18n "about.eu_visitor_count"}} + {{number this.model.stats.eu_visitors_last_day}} + {{number this.model.stats.eu_visitors_7_days}} + {{number this.model.stats.eu_visitors_30_days}} + — + + {{/if}} {{i18n "about.like_count"}} {{number this.model.stats.likes_last_day}} @@ -170,6 +186,10 @@ {{/each}} + {{#if this.siteSettings.display_eu_visitor_stats}} +

{{this.statsTableFooter}}

+ {{/if}} {{/if}} diff --git a/app/assets/stylesheets/common/base/about.scss b/app/assets/stylesheets/common/base/about.scss index b70f50a5510..e3a015c5e34 100644 --- a/app/assets/stylesheets/common/base/about.scss +++ b/app/assets/stylesheets/common/base/about.scss @@ -113,5 +113,9 @@ section.about { } } } + + .stats-table-footer { + max-width: 30em; + } } } diff --git a/app/models/application_request.rb b/app/models/application_request.rb index a789ceb3346..b471314a1a0 100644 --- a/app/models/application_request.rb +++ b/app/models/application_request.rb @@ -59,6 +59,17 @@ class ApplicationRequest < ActiveRecord::Base s end + + def self.request_type_count_for_period(type, since) + id = self.req_types[type] + if !id + raise ArgumentError.new( + "unknown request type #{type.inspect} in ApplicationRequest.req_types", + ) + end + + self.where(req_type: id).where("date >= ?", since).sum(:count) + end end # == Schema Information diff --git a/app/models/stat.rb b/app/models/stat.rb index e425f7e5e26..0ab2a5c09d2 100644 --- a/app/models/stat.rb +++ b/app/models/stat.rb @@ -40,7 +40,7 @@ class Stat end def self.core_stats - [ + list = [ Stat.new("topics", show_in_ui: true, expose_via_api: true) { Statistics.topics }, Stat.new("posts", show_in_ui: true, expose_via_api: true) { Statistics.posts }, Stat.new("users", show_in_ui: true, expose_via_api: true) { Statistics.users }, @@ -50,6 +50,19 @@ class Stat Statistics.participating_users end, ] + + if SiteSetting.display_eu_visitor_stats + list.concat( + [ + Stat.new("visitors", show_in_ui: true, expose_via_api: true) { Statistics.visitors }, + Stat.new("eu_visitors", show_in_ui: true, expose_via_api: true) do + Statistics.eu_visitors + end, + ], + ) + end + + list end def self._api_stats diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index 5ac999b5a54..4692c81bb36 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -342,6 +342,16 @@ en: post_count: "Posts" user_count: "Sign-ups" active_user_count: "Active users" + visitor_count: "Visitors" + eu_visitor_count: "Visitors from European Union" + traffic_info_footer_MF: | + In the last 6 months, this site has served content to an average of approximately { total_visitors, plural, + one {# people} + other {# people} + } each month, with an average of approximately { eu_visitors, plural, + one {# people} + other {# people} + } from the European Union. contact: "Contact us" contact_info: "In the event of a critical issue or urgent matter affecting this site, please contact %{contact_info}." site_activity: "Site activity" @@ -363,6 +373,14 @@ en: likes: one: "%{formatted_number} like" other: "%{formatted_number} likes" + visitors_MF: | + { total_count, plural, + one {{total_formatted_number} visitor} + other {{total_formatted_number} visitors} + }, about { eu_count, plural, + one {{eu_formatted_number}} + other {{eu_formatted_number}} + } from the EU periods: last_7_days: "in the last 7 days" today: "today" diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 88b44f595ba..8a8a48a0539 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -2209,6 +2209,7 @@ en: tos_url: "If you have a Terms of Service document hosted elsewhere that you want to use, provide the full URL here." privacy_policy_url: "If you have a Privacy Policy document hosted elsewhere that you want to use, provide the full URL here." log_anonymizer_details: "Whether to keep a user's details in the log after being anonymized." + display_eu_visitor_stats: "Show number of global and EU visitors on the /about page." newuser_spam_host_threshold: "How many times a new user can post a link to the same host within their `newuser_spam_host_threshold` posts before being considered spam." diff --git a/config/site_settings.yml b/config/site_settings.yml index ccd5a440256..3d5c6e80375 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -2503,6 +2503,10 @@ legal: default: false hidden: true client: true + display_eu_visitor_stats: + default: false + client: true + hidden: true backups: enable_backups: diff --git a/lib/statistics.rb b/lib/statistics.rb index c366cb13366..1beda7cd6b7 100644 --- a/lib/statistics.rb +++ b/lib/statistics.rb @@ -1,6 +1,36 @@ # frozen_string_literal: true class Statistics + EU_COUNTRIES = %w[ + AT + BE + BG + CY + CZ + DE + DK + EE + ES + FI + FR + GR + HR + HU + IE + IT + LT + LU + LV + MT + NL + PL + PT + RO + SE + SI + SK + ] + def self.active_users { last_day: User.where("last_seen_at > ?", 1.day.ago).count, @@ -56,6 +86,58 @@ class Statistics } end + def self.visitors + periods = [[1.day.ago, :last_day], [7.days.ago, :"7_days"], [30.days.ago, :"30_days"]] + + periods + .map do |(period, key)| + anon_page_views = + ApplicationRequest.request_type_count_for_period(:page_view_anon_browser, period) + + logged_in_visitors = logged_in_visitors_count(period) + next key, anon_page_views if logged_in_visitors == 0 + + logged_in_page_views = + ApplicationRequest.request_type_count_for_period(:page_view_logged_in_browser, period) + next key, anon_page_views + logged_in_visitors if logged_in_page_views == 0 + + total_visitors = logged_in_visitors + avg_logged_in_page_view_per_user = logged_in_page_views.to_f / logged_in_visitors + anon_visitors = (anon_page_views / avg_logged_in_page_view_per_user).round + total_visitors += anon_visitors + [key, total_visitors] + end + .to_h + end + + def self.eu_visitors + periods = [[1.day.ago, :last_day], [7.days.ago, :"7_days"], [30.days.ago, :"30_days"]] + + periods + .map do |(period, key)| + logged_in_page_views = + ApplicationRequest.request_type_count_for_period(:page_view_logged_in_browser, period) + anon_page_views = + ApplicationRequest.request_type_count_for_period(:page_view_anon_browser, period) + + all_logged_in_visitors = logged_in_visitors_count(period) + eu_logged_in_visitors = eu_logged_in_visitors_count(period) + + next key, 0 if all_logged_in_visitors == 0 || eu_logged_in_visitors == 0 + next key, eu_logged_in_visitors if logged_in_page_views == 0 + + avg_logged_in_page_view_per_user = logged_in_page_views / all_logged_in_visitors.to_f + + eu_logged_in_visitors_ratio = eu_logged_in_visitors / all_logged_in_visitors.to_f + + eu_anon_visitors = + ((anon_page_views / avg_logged_in_page_view_per_user) * eu_logged_in_visitors_ratio).round + eu_visitors = eu_logged_in_visitors + eu_anon_visitors + [key, eu_visitors] + end + .to_h + end + private def self.participating_users_count(date) @@ -75,4 +157,27 @@ class Statistics DB.query_single(sql, date: date, action_types: UserAction::USER_ACTED_TYPES).first end + + def self.logged_in_visitors_count(since) + DB.query_single(<<~SQL, since:).first + SELECT COUNT(DISTINCT user_id) + FROM user_visits + WHERE visited_at >= :since + SQL + end + + def self.eu_logged_in_visitors_count(since) + results = DB.query_hash(<<~SQL, since:) + SELECT DISTINCT(user_id), ip_address + FROM user_visits uv + INNER JOIN users u + ON u.id = uv.user_id + WHERE visited_at >= :since AND ip_address IS NOT NULL + SQL + + results.reduce(0) do |sum, hash| + ip_info = DiscourseIpInfo.get(hash["ip_address"].to_s) + sum + (EU_COUNTRIES.include?(ip_info[:country_code]) ? 1 : 0) + end + end end diff --git a/spec/lib/statistics_spec.rb b/spec/lib/statistics_spec.rb index 657eec99abf..02c568c8d1c 100644 --- a/spec/lib/statistics_spec.rb +++ b/spec/lib/statistics_spec.rb @@ -1,7 +1,59 @@ # frozen_string_literal: true RSpec.describe Statistics do - describe "#participating_users" do + def create_page_views_and_user_visit_records(date, users) + freeze_time(date - 50.minutes) do + 2.times { ApplicationRequest.increment!(:page_view_anon_browser) } + ApplicationRequest.increment!(:page_view_logged_in_browser) + end + + freeze_time(date - 3.days) do + ApplicationRequest.increment!(:page_view_anon_browser) + 5.times { ApplicationRequest.increment!(:page_view_logged_in_browser) } + end + + freeze_time(date - 6.days) do + 3.times { ApplicationRequest.increment!(:page_view_anon_browser) } + 4.times { ApplicationRequest.increment!(:page_view_logged_in_browser) } + end + + freeze_time(date - 8.days) do + ApplicationRequest.increment!(:page_view_anon_browser) + ApplicationRequest.increment!(:page_view_logged_in_browser) + end + + freeze_time(date - 15.days) do + 4.times { ApplicationRequest.increment!(:page_view_anon_browser) } + 3.times { ApplicationRequest.increment!(:page_view_logged_in_browser) } + end + + freeze_time(date - 31.days) do + ApplicationRequest.increment!(:page_view_anon_browser) + ApplicationRequest.increment!(:page_view_logged_in_browser) + end + + UserVisit.create!(user_id: users[0].id, visited_at: date - 50.minute) + + UserVisit.create!(user_id: users[0].id, visited_at: date - 36.hours) + UserVisit.create!(user_id: users[1].id, visited_at: date - 2.day) + UserVisit.create!(user_id: users[0].id, visited_at: date - 4.days) + UserVisit.create!(user_id: users[2].id, visited_at: date - 6.days) + UserVisit.create!(user_id: users[3].id, visited_at: date - 3.days) + UserVisit.create!(user_id: users[3].id, visited_at: date - 5.days) + UserVisit.create!(user_id: users[1].id, visited_at: date - 66.hours) + + UserVisit.create!(user_id: users[2].id, visited_at: date - 8.days) + UserVisit.create!(user_id: users[3].id, visited_at: date - 13.days) + UserVisit.create!(user_id: users[0].id, visited_at: date - 24.days) + UserVisit.create!(user_id: users[4].id, visited_at: date - 19.days) + + UserVisit.create!(user_id: users[2].id, visited_at: date - 31.days) + end + + fab!(:users) { Fabricate.times(5, :user) } + let(:date) { DateTime.parse("2024-03-01 13:00") } + + describe ".participating_users" do it "returns no participating users by default" do pu = described_class.participating_users expect(pu[:last_day]).to eq(0) @@ -29,4 +81,138 @@ RSpec.describe Statistics do expect(described_class.participating_users[:last_day]).to eq(1) end end + + describe ".visitors" do + before do + ApplicationRequest.enable + create_page_views_and_user_visit_records(date, users) + end + + after { ApplicationRequest.disable } + + it "estimates the number of visitors for each of the previous 1 day, 7 days and 30 days periods" do + freeze_time(date) do + visitors = described_class.visitors + + # anon page views: 2 + # logged-in page views: 1 + # logged-in visitors: 1 + # we can estimate the number of unique anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor. + # in this case, the estimated number of anon visitors is 2 / (1 / 1) = 2. + # total visitors = logged-in visitors (1) + estimated anon visitors (2) = 3 + expect(visitors[:last_day]).to eq(3) + + # anon page views: 6 + # logged-in page views: 10 + # logged-in visitors: 4 + # we can estimate the number of unique anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor. + # in this case, the estimated number of anon visitors is 6 / (10 / 4) ~= 2. + # total visitors = logged-in visitors (4) + estimated anon visitors (2) = 6 + expect(visitors[:"7_days"]).to eq(6) + + # anon page views: 11 + # logged-in page views: 14 + # logged-in visitors: 5 + # we can estimate the number of unique anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor. + # in this case, the estimated number of anon visitors is 11 / (14 / 5) ~= 4. + # total visitors = logged-in visitors (5) + estimated anon visitors (4) = 9 + expect(visitors[:"30_days"]).to eq(9) + end + end + + it "is the same as the number of anon page views when there are no logged in visitors" do + freeze_time(date) do + UserVisit.delete_all + + visitors = described_class.visitors + + expect(visitors[:last_day]).to eq(2) + expect(visitors[:"7_days"]).to eq(6) + expect(visitors[:"30_days"]).to eq(11) + end + end + end + + describe ".eu_visitors" do + before do + ApplicationRequest.enable + create_page_views_and_user_visit_records(date, users) + + users[0].update!(ip_address: IPAddr.new("60.23.1.42")) + users[1].update!(ip_address: IPAddr.new("90.19.255.63")) + users[2].update!(ip_address: IPAddr.new("8.33.134.244")) + users[3].update!(ip_address: IPAddr.new("2.74.0.98")) + users[4].update!(ip_address: IPAddr.new("88.82.3.101")) + + # EU IP addresses + DiscourseIpInfo.stubs(:get).with("60.23.1.42").returns({ country_code: "FR" }) # users[0] + DiscourseIpInfo.stubs(:get).with("2.74.0.98").returns({ country_code: "NL" }) # users[3] + DiscourseIpInfo.stubs(:get).with("88.82.3.101").returns({ country_code: "DE" }) # users[4] + + # non-EU IP addresses + DiscourseIpInfo.stubs(:get).with("90.19.255.63").returns({ country_code: "US" }) # users[1] + DiscourseIpInfo.stubs(:get).with("8.33.134.244").returns({ country_code: "SA" }) # users[2] + end + + after { ApplicationRequest.disable } + + it "estimates the number of EU visitors for each of the previous 1 day, 7 days and 30 days periods" do + freeze_time(date) do + eu_visitors = described_class.eu_visitors + + # anon page views: 2 + # logged-in page views: 1 + # logged-in visitors: 1 + # EU logged-in visitors: 1 (users[0]) + # we can estimate the number of unique EU anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor, then multiplying the result by the ratio + # of EU logged-in visitors to all logged-in visitors. + # in this case, the estimated number of EU anon visitors is 2 / (1 / 1) * (1 / 1) = 2 + # total EU visitors = EU logged-in visitors (1) + estimated EU anon visitors (2) = 3 + expect(eu_visitors[:last_day]).to eq(3) + + # anon page views: 6 + # logged-in page views: 10 + # logged-in visitors: 4 + # EU logged-in visitors: 2 (users[0], users[3]) + # we can estimate the number of unique EU anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor, then multiplying the result by the ratio + # of EU logged-in visitors to all logged-in visitors. + # in this case, the estimated number of EU anon visitors is 6 / (10 / 4) * (2 / 4) ~= 1 + # total EU visitors = EU logged-in visitors (2) + estimated EU anon visitors (1) = 3 + expect(eu_visitors[:"7_days"]).to eq(3) + + # anon page views: 11 + # logged-in page views: 14 + # logged-in visitors: 5 + # EU logged-in visitors: 3 (users[0], users[3], users[4]) + # we can estimate the number of unique EU anon visitors by dividing the + # number of anon page views by the average number of logged-in page + # views per logged-in visitor, then multiplying the result by the ratio + # of EU logged-in visitors to all logged-in visitors. + # in this case, the estimated number of EU anon visitors is 11 / (14 / 5) * (3 / 5) ~= 1 + # total EU visitors = EU logged-in visitors (3) + estimated EU anon visitors (2) = 5 + expect(eu_visitors[:"30_days"]).to eq(5) + end + end + + it "returns 0 for EU visitors when there are no logged-in users" do + freeze_time(date) do + UserVisit.delete_all + + eu_visitors = described_class.eu_visitors + expect(eu_visitors[:last_day]).to eq(0) + expect(eu_visitors[:"7_days"]).to eq(0) + expect(eu_visitors[:"30_days"]).to eq(0) + end + end + end end diff --git a/spec/system/about_page_spec.rb b/spec/system/about_page_spec.rb index fd1c17440be..02c92f0c7bf 100644 --- a/spec/system/about_page_spec.rb +++ b/spec/system/about_page_spec.rb @@ -127,6 +127,31 @@ describe "About page", type: :system do end end + describe "visitors" do + context "when the display_eu_visitor_stats setting is disabled" do + before { SiteSetting.display_eu_visitor_stats = false } + + it "doesn't show the row" do + about_page.visit + + expect(about_page.site_activities).to have_no_activity_item("visitors") + end + end + + context "when the display_eu_visitor_stats setting is enabled" do + before { SiteSetting.display_eu_visitor_stats = true } + + it "shows the row" do + about_page.visit + + expect(about_page.site_activities).to have_activity_item("visitors") + expect(about_page.site_activities.visitors).to have_text( + "1 visitor, about 0 from the EU", + ) + end + end + end + describe "active users" do before do User.update_all(last_seen_at: 1.month.ago) diff --git a/spec/system/page_objects/components/about_page_site_activity.rb b/spec/system/page_objects/components/about_page_site_activity.rb index 5e0a9a5d7d0..794aca0a25c 100644 --- a/spec/system/page_objects/components/about_page_site_activity.rb +++ b/spec/system/page_objects/components/about_page_site_activity.rb @@ -23,6 +23,13 @@ module PageObjects ) end + def visitors + AboutPageSiteActivityItem.new( + container.find(".about__activities-item.visitors"), + translation_key: nil, + ) + end + def active_users AboutPageSiteActivityItem.new( container.find(".about__activities-item.active-users"), @@ -51,6 +58,14 @@ module PageObjects translation_key:, ) end + + def has_activity_item?(name) + container.has_css?(".about__activities-item.#{name}") + end + + def has_no_activity_item?(name) + container.has_no_css?(".about__activities-item.#{name}") + end end end end diff --git a/spec/system/page_objects/components/about_page_site_activity_item.rb b/spec/system/page_objects/components/about_page_site_activity_item.rb index b0f9b07c644..7ae821fd304 100644 --- a/spec/system/page_objects/components/about_page_site_activity_item.rb +++ b/spec/system/page_objects/components/about_page_site_activity_item.rb @@ -16,6 +16,10 @@ module PageObjects ) end + def has_text?(text) + container.find(".about__activities-item-count").has_text?(text) + end + def has_1_day_period? period_element.has_text?(I18n.t("js.about.activities.periods.today")) end