FIX: crawler requests exceptions for non UTF-8 user agents with invalid bytes

lib/http_user_agent_encoder.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module HttpUserAgentEncoder
+  def self.ensure_utf8(user_agent)
+    return "" unless user_agent
+
+    if user_agent.encoding != Encoding::UTF_8
+      user_agent = user_agent.encode("utf-8", invalid: :replace, undef: :replace)
+      user_agent.scrub!
+    end
+
+    user_agent || ""
+  end
+end