From 2d05a82f52e2358bc3c21ee3df224937fb19cf7c Mon Sep 17 00:00:00 2001
From: Gerhard Schlager <gerhard.schlager@discourse.org>
Date: Mon, 20 Jan 2025 23:16:30 +0100
Subject: [PATCH] DEV: Add `Upload` to IntermediateDB (#29780)

---
 Gemfile                                       |  3 +
 Gemfile.lock                                  |  2 +
 .../db/intermediate_db_schema/003-uploads.sql | 12 +++
 .../100-base-schema.sql                       |  4 +-
 migrations/lib/common/id.rb                   | 15 +++
 .../lib/database/intermediate_db/upload.rb    | 92 +++++++++++++++++++
 migrations/lib/migrations.rb                  |  7 +-
 7 files changed, 132 insertions(+), 3 deletions(-)
 create mode 100644 migrations/db/intermediate_db_schema/003-uploads.sql
 create mode 100644 migrations/lib/common/id.rb
 create mode 100644 migrations/lib/database/intermediate_db/upload.rb

diff --git a/Gemfile b/Gemfile
index bdb7e1134d2..308aeec881f 100644
--- a/Gemfile
+++ b/Gemfile
@@ -284,6 +284,9 @@ group :migrations, optional: true do
 
   # CLI
   gem "ruby-progressbar"
+
+  # non-cryptographic hashing algorithm for generating placeholder IDs
+  gem "digest-xxhash"
 end
 
 gem "dry-initializer", "~> 3.1"
diff --git a/Gemfile.lock b/Gemfile.lock
index 50723e03617..6b5ada66791 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -124,6 +124,7 @@ GEM
     diff-lcs (1.5.1)
     diffy (3.4.3)
     digest (3.1.1)
+    digest-xxhash (0.2.9)
     discourse-fonts (0.0.14)
     discourse-seed-fu (2.3.12)
       activerecord (>= 3.1)
@@ -630,6 +631,7 @@ DEPENDENCIES
   csv
   diffy
   digest
+  digest-xxhash
   discourse-fonts
   discourse-seed-fu
   discourse_dev_assets
diff --git a/migrations/db/intermediate_db_schema/003-uploads.sql b/migrations/db/intermediate_db_schema/003-uploads.sql
new file mode 100644
index 00000000000..deda4841749
--- /dev/null
+++ b/migrations/db/intermediate_db_schema/003-uploads.sql
@@ -0,0 +1,12 @@
+CREATE TABLE uploads
+(
+    id          TEXT NOT NULL PRIMARY KEY,
+    filename    TEXT NOT NULL,
+    path        TEXT,
+    data        BLOB,
+    url         TEXT,
+    type        TEXT,
+    description TEXT,
+    origin      TEXT,
+    user_id     NUMERIC
+);
diff --git a/migrations/db/intermediate_db_schema/100-base-schema.sql b/migrations/db/intermediate_db_schema/100-base-schema.sql
index aadd6abda46..bd86d41cb1c 100644
--- a/migrations/db/intermediate_db_schema/100-base-schema.sql
+++ b/migrations/db/intermediate_db_schema/100-base-schema.sql
@@ -33,5 +33,5 @@ CREATE TABLE users
     suspended_at              DATETIME,
     suspended_till            DATETIME,
     title                     TEXT,
-    uploaded_avatar_id        INTEGER
-);
\ No newline at end of file
+    uploaded_avatar_id        TEXT
+);
diff --git a/migrations/lib/common/id.rb b/migrations/lib/common/id.rb
new file mode 100644
index 00000000000..f0cf6c20ba8
--- /dev/null
+++ b/migrations/lib/common/id.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "digest/xxhash"
+
+module Migrations
+  module ID
+    def self.hash(value)
+      Digest::XXH3_128bits.base64digest(value)
+    end
+
+    def self.build(part1, part2, *others)
+      [part1, part2, *others].join("-")
+    end
+  end
+end
diff --git a/migrations/lib/database/intermediate_db/upload.rb b/migrations/lib/database/intermediate_db/upload.rb
new file mode 100644
index 00000000000..4da5aee8db8
--- /dev/null
+++ b/migrations/lib/database/intermediate_db/upload.rb
@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Migrations::Database::IntermediateDB
+  module Upload
+    SQL = <<~SQL
+      INSERT OR IGNORE INTO uploads (
+          id,
+          filename,
+          path,
+          data,
+          url,
+          type,
+          description,
+          origin,
+          user_id
+      )
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    SQL
+
+    class << self
+      def create_for_file!(
+        path:,
+        filename: nil,
+        type: nil,
+        description: nil,
+        origin: nil,
+        user_id: nil
+      )
+        create!(
+          id: ::Migrations::ID.hash(path),
+          filename: filename || File.basename(path),
+          path:,
+          type:,
+          description:,
+          origin:,
+          user_id:,
+        )
+      end
+
+      def create_for_url!(url:, filename:, type: nil, description: nil, origin: nil, user_id: nil)
+        create!(
+          id: ::Migrations::ID.hash(url),
+          filename:,
+          url:,
+          type:,
+          description:,
+          origin:,
+          user_id:,
+        )
+      end
+
+      def create_for_data!(data:, filename:, type: nil, description: nil, origin: nil, user_id: nil)
+        create!(
+          id: ::Migrations::ID.hash(data),
+          filename:,
+          data: ::Migrations::Database.to_blob(data),
+          type:,
+          description:,
+          origin:,
+          user_id:,
+        )
+      end
+
+      private
+
+      def create!(
+        id:,
+        filename:,
+        path: nil,
+        data: nil,
+        url: nil,
+        type: nil,
+        description: nil,
+        origin: nil,
+        user_id: nil
+      )
+        ::Migrations::Database::IntermediateDB.insert(
+          SQL,
+          id,
+          filename,
+          path,
+          data,
+          url,
+          type,
+          description,
+          origin,
+          user_id,
+        )
+      end
+    end
+  end
+end
diff --git a/migrations/lib/migrations.rb b/migrations/lib/migrations.rb
index ddd829d098e..eb67a56da45 100644
--- a/migrations/lib/migrations.rb
+++ b/migrations/lib/migrations.rb
@@ -45,7 +45,12 @@ module Migrations
     loader.log! if ENV["DEBUG"]
 
     loader.inflector.inflect(
-      { "cli" => "CLI", "intermediate_db" => "IntermediateDB", "uploads_db" => "UploadsDB" },
+      {
+        "cli" => "CLI",
+        "id" => "ID",
+        "intermediate_db" => "IntermediateDB",
+        "uploads_db" => "UploadsDB",
+      },
     )
 
     loader.push_dir(File.join(::Migrations.root_path, "lib"), namespace: ::Migrations)