Support for transpiling .js files (#9160)

* Remove some `.es6` from comments where it does not matter

* Use a post processor for transpilation

This will allow us to eventually use the directory structure to
transpile rather than the extension.

* FIX: Some errors and clean up in confirm-new-email

It would throw an error if the webauthn element wasn't present.
Also I changed things so that no-module is not explicitly
referenced.

* Remove `no-module`

Instead we allow a magic comment: `// discourse-skip-module` to prevent
the asset pipeline from creating a module.

* DEV: Enable babel transpilation based on directory

If it's in `app/assets/javascripts/dicourse` it will be transpiled
even without the `.es6` extension.

* REFACTOR: Remove Tilt/ES6ModuleTranspiler

lib/discourse.rb

@@ -655,7 +655,7 @@ module Discourse
     # in case v8 was initialized we want to make sure it is nil
     PrettyText.reset_context
 
-    Tilt::ES6ModuleTranspilerTemplate.reset_context if defined? Tilt::ES6ModuleTranspilerTemplate
+    DiscourseJsProcessor::Transpiler.reset_context if defined? DiscourseJsProcessor::Transpiler
     JsLocaleHelper.reset_context if defined? JsLocaleHelper
     nil
   end

lib/discourse_iife.rb

@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-class DiscourseIIFE
-  def initialize(options = {}, &block)
-  end
-
-  def self.instance
-    @instance ||= new
-  end
-
-  def self.call(input)
-    instance.call(input)
-  end
-
-  # Add a IIFE around our javascript
-  def call(input)
-    path = input[:environment].context_class.new(input).pathname.to_s
-    data = input[:data]
-
-    # Only discourse or admin paths
-    return data unless (path =~ /\/javascripts\/discourse/ || path =~ /\/javascripts\/admin/ || path =~ /\/test\/javascripts/)
-
-    # Ignore the js helpers
-    return data if (path =~ /test\_helper\.js/)
-    return data if (path =~ /javascripts\/helpers\//)
-
-    # Ignore ES6 files
-    return data if (path =~ /\.es6/)
-
-    # Ignore translations
-    return data if (path =~ /\/translations/)
-
-    # We don't add IIFEs to handlebars
-    return data if path =~ /\.handlebars/
-    return data if path =~ /\.shbrs/
-    return data if path =~ /\.hbrs/
-    return data if path =~ /\.hbs/
-    return data if path =~ /\.hbr/
-
-    return data if path =~ /discourse-loader/
-
-    "(function () {\n\nvar $ = window.jQuery;\n// IIFE Wrapped Content Begins:\n\n#{data}\n\n// IIFE Wrapped Content Ends\n\n })(this);"
-  end
-
-end

lib/discourse_js_processor.rb

@@ -1,28 +1,54 @@
 # frozen_string_literal: true
-
 require 'execjs'
 require 'mini_racer'
 
-module Tilt
+class DiscourseJsProcessor
 
-  class ES6ModuleTranspilerTemplate < Tilt::Template
-    self.default_mime_type = 'application/javascript'
+  def self.call(input)
+    root_path = input[:load_path] || ''
+    logical_path = (input[:filename] || '').sub(root_path, '').gsub(/\.(js|es6).*$/, '').sub(/^\//, '')
+    data = input[:data]
 
+    if should_transpile?(input[:filename])
+      data = transpile(data, root_path, logical_path)
+    end
+
+    # add sourceURL until we can do proper source maps
+    unless Rails.env.production?
+      data = "eval(#{data.inspect} + \"\\n//# sourceURL=#{logical_path}\");\n"
+    end
+
+    { data: data }
+  end
+
+  def self.transpile(data, root_path, logical_path)
+    transpiler = Transpiler.new(skip_module: skip_module?(data))
+    transpiler.perform(data, root_path, logical_path)
+  end
+
+  def self.should_transpile?(filename)
+    filename ||= ''
+
+    # es6 is always transpiled
+    return true if filename.end_with?(".es6") || filename.end_with?(".es6.erb")
+
+    # For .js check the path...
+    return false unless filename.end_with?(".js") || filename.end_with?(".js.erb")
+
+    relative_path = filename.sub(Rails.root.to_s, '').sub(/^\/*/, '')
+    relative_path.start_with?("app/assets/javascripts/discourse/")
+  end
+
+  def self.skip_module?(data)
+    !!(data.present? && data =~ /^\/\/ discourse-skip-module$/)
+  end
+
+  class Transpiler
     @mutex = Mutex.new
     @ctx_init = Mutex.new
 
-    def self.call(input)
-      filename = input[:filename]
-      source = input[:data]
-      context = input[:environment].context_class.new(input)
-
-      result = new(filename) { source }.render(context)
-      context.metadata.merge(data: result)
-    end
-
-    def prepare
-      # intentionally left empty
-      # Tilt requires this method to be defined
+    def self.mutex
+      @mutex
     end
 
     def self.create_new_context
@@ -66,33 +92,13 @@ JS
       @ctx
     end
 
-    class JavaScriptError < StandardError
-      attr_accessor :message, :backtrace
-
-      def initialize(message, backtrace)
-        @message = message
-        @backtrace = backtrace
-      end
-
+    def initialize(skip_module: false)
+      @skip_module = skip_module
     end
 
-    def self.protect
-      @mutex.synchronize do
-        yield
-      end
-    end
-
-    def babel_transpile(source)
+    def perform(source, root_path = nil, logical_path = nil)
       klass = self.class
-      klass.protect do
-        klass.v8.eval("console.prefix = 'BABEL: babel-eval: ';")
-        @output = klass.v8.eval(babel_source(source))
-      end
-    end
-
-    def module_transpile(source, root_path, logical_path)
-      klass = self.class
-      klass.protect do
+      klass.mutex.synchronize do
         klass.v8.eval("console.prefix = 'BABEL: babel-eval: ';")
         transpiled = babel_source(
           source,
@@ -103,31 +109,12 @@ JS
       end
     end
 
-    def evaluate(scope, locals, &block)
-      return @output if @output
-
-      klass = self.class
-      klass.protect do
-        klass.v8.eval("console.prefix = 'BABEL: #{scope.logical_path}: ';")
-
-        source = babel_source(
-          data,
-          module_name: module_name(scope.root_path, scope.logical_path),
-          filename: scope.logical_path
-        )
-
-        @output = klass.v8.eval(source)
-      end
-
-      @output
-    end
-
     def babel_source(source, opts = nil)
       opts ||= {}
 
       js_source = ::JSON.generate(source, quirks_mode: true)
 
-      if opts[:module_name] && transpile_into_module?
+      if opts[:module_name] && !@skip_module
         filename = opts[:filename] || 'unknown'
         "Babel.transform(#{js_source}, { moduleId: '#{opts[:module_name]}', filename: '#{filename}', ast: false, presets: ['es2015'], plugins: [['transform-es2015-modules-amd', {noInterop: true}], 'transform-decorators-legacy', exports.WidgetHbsCompiler] }).code"
       else
@@ -135,12 +122,6 @@ JS
       end
     end
 
-    private
-
-    def transpile_into_module?
-      file.nil? || file.exclude?('.no-module')
-    end
-
     def module_name(root_path, logical_path)
       path = nil
 
@@ -153,26 +134,8 @@ JS
         path = "discourse/plugins/#{plugin.name}/#{logical_path.sub(/javascripts\//, '')}" if plugin
       end
 
-      path ||= logical_path
-      if ES6ModuleTranspiler.transform
-        path = ES6ModuleTranspiler.transform.call(path)
-      end
-
-      path
+      path || logical_path
     end
 
-    def compiler_method
-      type = {
-        amd: 'AMD',
-        cjs: 'CJS',
-        globals: 'Globals'
-      }[ES6ModuleTranspiler.compile_to.to_sym]
-
-      "to#{type}"
-    end
-
-    def compiler_options
-      ::JSON.generate(ES6ModuleTranspiler.compiler_options, quirks_mode: true)
-    end
   end
 end

lib/discourse_plugin_registry.rb

@@ -151,7 +151,7 @@ class DiscoursePluginRegistry
     end
   end
 
-  JS_REGEX = /\.js$|\.js\.erb$|\.js\.es6|\.js\.no-module\.es6$/
+  JS_REGEX = /\.js$|\.js\.erb$|\.js\.es6$/
   HANDLEBARS_REGEX = /\.(hb[rs]|js\.handlebars)$/
 
   def self.register_asset(asset, opts = nil, plugin_directory_name = nil)

lib/es6_module_transpiler/rails.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-require 'es6_module_transpiler/rails/version'
-require 'es6_module_transpiler/tilt'
-require 'es6_module_transpiler/sprockets'
-
-module ES6ModuleTranspiler
-  def self.compile_to
-    @compile_to || :amd
-  end
-
-  def self.compile_to=(target)
-    @compile_to = target
-  end
-
-  def self.transform=(transform)
-    @transform = transform
-  end
-
-  def self.transform
-    @transform
-  end
-
-  def self.compiler_options
-    @compiler_options ||= {}
-  end
-end

lib/es6_module_transpiler/rails/version.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module ES6ModuleTranspiler
-  module Rails
-    VERSION = '0.4.0'
-  end
-end

lib/es6_module_transpiler/sprockets.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-require 'sprockets'
-
-Sprockets.register_mime_type 'application/ecmascript6', extensions: ['.es6', '.js.es6', '.js.no-module.es6'], charset: :unicode
-Sprockets.register_transformer 'application/ecmascript6', 'application/javascript', Tilt::ES6ModuleTranspilerTemplate

lib/es6_module_transpiler/tilt.rb

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-require 'tilt'
-require 'es6_module_transpiler/tilt/es6_module_transpiler_template'
-
-Tilt.register Tilt::ES6ModuleTranspilerTemplate, 'es6'

lib/freedom_patches/raw_handlebars.rb

@@ -12,8 +12,8 @@ class Barber::Precompiler
     if !@precompiler
 
       source = File.read("#{Rails.root}/app/assets/javascripts/discourse-common/lib/raw-handlebars.js.es6")
-      template = Tilt::ES6ModuleTranspilerTemplate.new {}
-      transpiled = template.babel_transpile(source)
+      transpiler = DiscourseJsProcessor::Transpiler.new(skip_module: true)
+      transpiled = transpiler.perform(source)
 
       # very hacky but lets us use ES6. I'm ashamed of this code -RW
       transpiled = transpiled[0...transpiled.index('export ')]

lib/freedom_patches/zeitwerk.rb

@@ -12,7 +12,6 @@ module ActiveSupport::Dependencies::ZeitwerkIntegration::Inflector
     'postgresql_fallback_adapter' => 'PostgreSQLFallbackHandler',
     'regular' => 'Jobs',
     'scheduled' => 'Jobs',
-    'source_url' => 'SourceURL',
     'topic_query_sql' => 'TopicQuerySQL',
     'version' => 'Discourse',
   }

lib/pretty_text.rb

@@ -29,13 +29,10 @@ module PrettyText
     filename = find_file(root_path, part_name)
     if filename
       source = File.read("#{root_path}#{filename}")
+      source = ERB.new(source).result(binding) if filename =~ /\.erb$/
 
-      if filename =~ /\.erb$/
-        source = ERB.new(source).result(binding)
-      end
-
-      template = Tilt::ES6ModuleTranspilerTemplate.new {}
-      transpiled = template.module_transpile(source, "#{Rails.root}/app/assets/javascripts/", part_name)
+      transpiler = DiscourseJsProcessor::Transpiler.new
+      transpiled = transpiler.perform(source, "#{Rails.root}/app/assets/javascripts/", part_name)
       ctx.eval(transpiled)
     else
       # Look for vendored stuff

lib/source_url.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-class SourceURL < Tilt::Template
-  self.default_mime_type = 'application/javascript'
-
-  def self.call(input)
-    filename = input[:filename]
-    source = input[:data]
-    context = input[:environment].context_class.new(input)
-
-    result = new(filename) { source }.render(context)
-    context.metadata.merge(data: result)
-  end
-
-  def prepare
-  end
-
-  def evaluate(scope, locals, &block)
-    code = +"eval("
-    code << data.inspect
-    code << " + \"\\n//# sourceURL=#{scope.logical_path}\""
-    code << ");\n"
-  end
-end

lib/theme_javascript_compiler.rb

@@ -214,8 +214,8 @@ class ThemeJavascriptCompiler
 
   def append_module(script, name, include_variables: true)
     script = "#{theme_variables}#{script}" if include_variables
-    template = Tilt::ES6ModuleTranspilerTemplate.new {}
-    @content << template.module_transpile(script, "", name)
+    transpiler = DiscourseJsProcessor::Transpiler.new
+    @content << transpiler.perform(script, "", name)
   rescue MiniRacer::RuntimeError => ex
     raise CompileError.new ex.message
   end
@@ -237,7 +237,7 @@ class ThemeJavascriptCompiler
   end
 
   def transpile(es6_source, version)
-    template = Tilt::ES6ModuleTranspilerTemplate.new {}
+    transpiler = DiscourseJsProcessor::Transpiler.new(skip_module: true)
     wrapped = <<~PLUGIN_API_JS
       (function() {
         if ('Discourse' in window && typeof Discourse._registerPluginCode === 'function') {
@@ -254,7 +254,7 @@ class ThemeJavascriptCompiler
       })();
     PLUGIN_API_JS
 
-    template.babel_transpile(wrapped)
+    transpiler.perform(wrapped)
   rescue MiniRacer::RuntimeError => ex
     raise CompileError.new ex.message
   end