PERF: Reuse existing core JS build where possible (#32311)

Building the Discourse JS app is very resource-intensive. This commit
introduces an `assemble_ember_build` script which will check the
existing content of the `dist/` directory and re-use the core build if
possible. Plugins will always be rebuilt.

For now, this functionality is only useful for multi-stage (i.e.
non-standard) Discourse deployments. But in future, this script may be
extended to pull the contents of the `dist/` directory from a remote
location.

.github/workflows/tests.yml

@@ -45,6 +45,7 @@ jobs:
       MINIO_RUNNER_INSTALL_DIR: /home/discourse/.minio_runner
       TESTEM_BROWSER: ${{ (startsWith(matrix.browser, 'Firefox') && 'Firefox') || matrix.browser }}
       TESTEM_FIREFOX_PATH: ${{ (matrix.browser == 'Firefox Evergreen') && '/opt/firefox-evergreen/firefox' }}
+      EMBER_ENV: development
 
     strategy:
       fail-fast: false
@@ -262,7 +263,7 @@ jobs:
 
       - name: Ember Build for System Tests
         if: matrix.build_type == 'system'
-        run: bin/ember-cli --build
+        run: script/assemble_ember_build.rb
 
       - name: Core System Tests
         if: matrix.build_type == 'system' && matrix.target == 'core'

.gitignore

@@ -53,7 +53,7 @@
 
 /spec/fixtures/plugins/my_plugin/auto_generated
 
-/vendor/bundle/*
+/vendor/bundle
 /vendor/data/GeoLite2-City.mmdb
 /vendor/data/GeoLite2-ASN.mmdb
 

app/assets/javascripts/discourse/ember-cli-build.js

@@ -89,6 +89,12 @@ module.exports = function (defaults) {
   const terserPlugin = app.project.findAddonByName("ember-cli-terser");
   const applyTerser = (tree) => terserPlugin.postprocessTree("all", tree);
 
+  const pluginTrees = applyTerser(discoursePluginsTree);
+
+  if (process.env.SKIP_CORE_BUILD) {
+    return pluginTrees;
+  }
+
   let extraPublicTrees = [
     parsePluginClientSettings(discourseRoot, vendorJs, app),
     funnel(`${discourseRoot}/public/javascripts`, { destDir: "javascripts" }),
@@ -99,7 +105,7 @@ module.exports = function (defaults) {
       })
     ),
     applyTerser(generateScriptsTree(app)),
-    applyTerser(discoursePluginsTree),
+    pluginTrees,
   ];
 
   const assetCachebuster = process.env["DISCOURSE_ASSET_URL_SALT"] || "";

lib/tasks/assets.rake

@@ -12,26 +12,7 @@ task "assets:precompile:build" do
 
     raise "Unknown ember version '#{ember_version}'" if !%w[5].include?(ember_version)
 
-    # If `JOBS` env is not set, `thread-loader` defaults to the number of CPUs - 1 on the machine but we want to cap it
-    # at 2 because benchmarking has shown that anything beyond 2 does not improve build times or the increase is marginal.
-    # Therefore, we cap it so that we don't spawn more processes than necessary.
-    jobs_env_count = (2 if !ENV["JOBS"].present? && Etc.nprocessors > 2)
-
-    compile_command = "CI=1 pnpm --dir=app/assets/javascripts/discourse ember build"
-
-    heap_size_limit = check_node_heap_size_limit
-
-    if heap_size_limit < 2048
-      STDERR.puts "Node.js heap_size_limit (#{heap_size_limit}) is less than 2048MB. Setting --max-old-space-size=2048 and CHEAP_SOURCE_MAPS=1"
-      jobs_env_count = 1
-
-      compile_command =
-        "NODE_OPTIONS='--max-old-space-size=2048' CHEAP_SOURCE_MAPS=1 #{compile_command}"
-    end
-
-    ember_env = ENV["EMBER_ENV"] || "production"
-    compile_command = "#{compile_command} -prod" if ember_env == "production"
-    compile_command = "JOBS=#{jobs_env_count} #{compile_command}" if jobs_env_count
+    compile_command = "#{Rails.root}/script/assemble_ember_build.rb"
 
     only_ember_precompile_build_remaining = (ARGV.last == "assets:precompile:build")
     only_assets_precompile_remaining = (ARGV.last == "assets:precompile")
@@ -130,13 +111,6 @@ task "assets:flush_sw" => "environment" do
   end
 end
 
-def check_node_heap_size_limit
-  output, status =
-    Open3.capture2("node", "-e", "console.log(v8.getHeapStatistics().heap_size_limit/1024/1024)")
-  raise "Failed to fetch node memory limit" if status != 0
-  output.to_f
-end
-
 def assets_path
   "#{Rails.root}/public/assets"
 end

script/assemble_ember_build.rb

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+# rubocop:disable Discourse/NoChdir
+
+require "fileutils"
+require "tempfile"
+require "open3"
+require "json"
+
+BUILD_INFO_FILE = "dist/BUILD_INFO.json"
+
+Dir.chdir("#{__dir__}/../app/assets/javascripts/discourse")
+
+def capture(*args)
+  output, status = Open3.capture2(*args)
+  raise "Command failed: #{args.inspect}" if status != 0
+  output
+end
+
+# Returns a git tree-hash representing the current state of Discourse core.
+# If the working directory is clean, it will match the tree hash (note: different to the commit hash) of the HEAD commit.
+def core_tree_hash
+  Tempfile.create do |f|
+    f.close
+
+    git_dir = capture("git", "rev-parse", "--git-dir").strip
+    FileUtils.cp "#{git_dir}/index", f.path
+
+    env = { "GIT_INDEX_FILE" => f.path }
+    system(env, "git", "add", "-A", exception: true)
+    return capture(env, "git", "write-tree").strip
+  end
+end
+
+def node_heap_size_limit
+  capture("node", "-e", "console.log(v8.getHeapStatistics().heap_size_limit/1024/1024)").to_f
+end
+
+def low_memory_environment?
+  node_heap_size_limit < 2048
+end
+
+def resolved_ember_env
+  ENV["EMBER_ENV"] || "production"
+end
+
+def build_info
+  { "ember_env" => resolved_ember_env, "core_tree_hash" => core_tree_hash }
+end
+
+def existing_core_build_usable?
+  if !File.exist?(BUILD_INFO_FILE)
+    STDERR.puts "No existing build info file found."
+    return false
+  end
+
+  existing = JSON.parse(File.read(BUILD_INFO_FILE))
+  expected = build_info
+
+  if existing == expected
+    true
+  else
+    STDERR.puts <<~MSG
+      Existing build is not reusable.
+      - Existing: #{existing.inspect}
+      - Current: #{expected.inspect}
+    MSG
+    false
+  end
+end
+
+build_cmd = %w[pnpm ember build]
+build_env = { "CI" => "1" }
+
+if Etc.nprocessors > 2
+  # Anything more than 2 doesn't seem to improve build times
+  build_env["JOBS"] ||= "2"
+end
+
+if low_memory_environment?
+  STDERR.puts "Node.js heap_size_limit is less than 2048MB. Setting --max-old-space-size=2048 and CHEAP_SOURCE_MAPS=1"
+  build_env["NODE_OPTIONS"] = "--max_old_space_size=2048"
+  build_env["CHEAP_SOURCE_MAPS"] = "1"
+end
+
+build_cmd << "-prod" if resolved_ember_env == "production"
+
+if existing_core_build_usable?
+  STDERR.puts "Reusing existing core ember build. Only building plugins..."
+  build_env["SKIP_CORE_BUILD"] = "1"
+  build_cmd << "-o" << "dist/_plugin_only_build"
+  begin
+    system(build_env, *build_cmd, exception: true)
+    FileUtils.rm_rf("dist/assets/plugins")
+    FileUtils.mv("dist/_plugin_only_build/assets/plugins", "dist/assets/plugins")
+  ensure
+    FileUtils.rm_rf("dist/_plugin_only_build")
+  end
+  STDERR.puts "Plugin build successfully integrated into dist"
+else
+  STDERR.puts "Running full core build..."
+  system(build_env, *build_cmd, exception: true)
+  File.write(BUILD_INFO_FILE, JSON.pretty_generate(build_info))
+end