DEV: Support nullable column property modification (#32978)

By default, rails makes timestamp columns (`created_at` and
`updated_at`) non-nullable, we also have some required core and plugins
columns we wouldn't necessarily want to enforce in the intermediate DB
schema. It'll be better to set the default values for these during
import instead of enforcing these at the converter level.

This change adds support for globally modifying a column’s `nullable`
state, defaulting all `created_at` columns to be `nullable` while
allowing for table level overrides.

---------

Co-authored-by: Gerhard Schlager <gerhard.schlager@discourse.org>

migrations/config/intermediate_db.yml

@@ -41,6 +41,8 @@ schema:
         - name: "id"
           datatype: numeric
           rename_to: "original_id"
+        - name: "created_at"
+          nullable: true
         - name_regex: ".*upload.*_id$"
           datatype: text
         - name_regex: ".*_id$"

migrations/config/json_schemas/db_schema.json

@@ -83,12 +83,26 @@
                         },
                         "datatype": {
                           "$ref": "#/$defs/datatypes"
+                        },
+                        "nullable": {
+                          "type": "boolean"
                         }
                       },
                       "additionalProperties": false,
                       "required": [
-                        "name",
+                        "name"
-                        "datatype"
+                      ],
+                      "anyOf": [
+                        {
+                          "required": [
+                            "datatype"
+                          ]
+                        },
+                        {
+                          "required": [
+                            "nullable"
+                          ]
+                        }
                       ]
                     }
                   },
@@ -191,6 +205,9 @@
                       },
                       "rename_to": {
                         "type": "string"
+                      },
+                      "nullable": {
+                        "type": "boolean"
                       }
                     },
                     "additionalProperties": false,
@@ -204,6 +221,11 @@
                         "required": [
                           "rename_to"
                         ]
+                      },
+                      {
+                        "required": [
+                          "nullable"
+                        ]
                       }
                     ],
                     "oneOf": [

migrations/db/intermediate_db_schema/100-base-schema.sql

@@ -5,7 +5,7 @@
 CREATE TABLE user_emails
 (
     email      TEXT     NOT NULL PRIMARY KEY,
-    created_at DATETIME NOT NULL,
+    created_at DATETIME,
     "primary"  BOOLEAN,
     user_id    NUMERIC  NOT NULL
 );
@@ -80,7 +80,7 @@ CREATE TABLE users
     approved                  BOOLEAN,
     approved_at               DATETIME,
     approved_by_id            NUMERIC,
-    created_at                DATETIME  NOT NULL,
+    created_at                DATETIME,
     date_of_birth             DATE,
     first_seen_at             DATETIME,
     flair_group_id            NUMERIC,

migrations/lib/database/intermediate_db/user.rb

@@ -48,7 +48,7 @@ module Migrations::Database::IntermediateDB
       approved: nil,
       approved_at: nil,
       approved_by_id: nil,
-      created_at:,
+      created_at: nil,
       date_of_birth: nil,
       first_seen_at: nil,
       flair_group_id: nil,

migrations/lib/database/intermediate_db/user_email.rb

@@ -18,7 +18,7 @@ module Migrations::Database::IntermediateDB
       )
     SQL
 
-    def self.create(email:, created_at:, primary: nil, user_id:)
+    def self.create(email:, created_at: nil, primary: nil, user_id:)
       ::Migrations::Database::IntermediateDB.insert(
         SQL,
         email,

migrations/lib/database/schema/global_config.rb

@@ -27,6 +27,12 @@ module Migrations::Database::Schema
       end
     end
 
+    def modified_nullable(column_name)
+      if (modified_column = find_modified_column(column_name))
+        modified_column[:nullable]
+      end
+    end
+
     private
 
     def find_modified_column(column_name)

migrations/lib/database/schema/loader.rb

@@ -45,7 +45,7 @@ module Migrations::Database::Schema
           Column.new(
             name: name_for(column),
             datatype: datatype_for(column),
-            nullable: column.null || column.default,
+            nullable: nullable_for(column, config),
             max_length: column.type == :text ? column.limit : nil,
             is_primary_key: primary_key_column_names.include?(column.name),
           )
@@ -102,6 +102,16 @@ module Migrations::Database::Schema
       end
     end
 
+    def nullable_for(column, config)
+      modified_column = config.dig(:columns, :modify)&.find { |col| col[:name] == column.name }
+      return modified_column[:nullable] if modified_column&.key?(:nullable)
+
+      global_nullable = @global.modified_nullable(column.name)
+      return global_nullable unless global_nullable.nil?
+
+      column.null || column.default.present?
+    end
+
     def indexes(config)
       config[:indexes]&.map do |index|
         Index.new(