github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-29 11:48:08 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

REFACTOR: Code generator for migrations IntemerdiateDB

Browse Source 
* Splits the existing script into multiple classes
* Adds command for generating IntermediateDB schema (`migrations/bin/cli schema generate`)
* Changes the syntax of the IntermediateDB schema config
* Adds validation for the schema config
* It uses YAML schema aka JSON schema to validate the config file
* It generates the SQL schema file and Ruby classes for storing data in the IntermediateDB
This commit is contained in:
Gerhard Schlager
2025-04-07 17:12:18 +02:00
committed by Gerhard Schlager
parent 71a90dcba2
commit 17ba19c7ae
33 changed files with 2359 additions and 864 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
.github/workflows/migration-tests.yml vendored
View File

@ -111,11 +111,16 @@ jobs:
if: steps.app-cache.outputs.cache-hit != 'true'
run: rm -rf tmp/app-cache/uploads && cp -r public/uploads tmp/app-cache/uploads
# - name: Check core database drift
# run: |
# mkdir /tmp/intermediate_db
# ./migrations/scripts/schema_generator /tmp/intermediate_db/base_migration.sql
# diff -u migrations/common/intermediate_db_schema/000_base_schema.sql /tmp/intermediate_db/base_migration.sql
- name: Validate IntermediateDB schema
run: |
migrations/bin/cli schema generate --db=intermediate_db
if [ ! -z "$(git status --porcelain migrations/db/intermediate_db_schema/)" ]; then
echo "IntermediateDB schema is not up to date."
echo "---------------------------------------------"
git -c color.ui=always diff migrations/db/intermediate_db_schema/
exit 1
fi
- name: RSpec
run: bin/rspec --default-path migrations/spec

3
migrations/bin/cli
View File

@ -37,6 +37,9 @@ module Migrations
::Migrations::CLI::UploadCommand.new(options).execute
end
desc "schema [COMMAND]", "Manage database schema"
subcommand "schema", ::Migrations::CLI::SchemaSubCommand
def self.exit_on_failure?
true
end

333
migrations/config/intermediate_db.yml
View File

@ -1,50 +1,291 @@
## Configuration options for the base intermediate schema generator
##
## After modifying this file, regenerate the base intermediate schema
## by running the `generate_schema` script.
# $schema: ./json_schemas/db_schema.json
# Default relative path for generated base schema file.
# An absolute path can also be provided to the script as the first CLI argument.
# If the CLI argument is present, it takes precedence over the value specified here.
output_file_path: "../db/schema/100-base-schema.sql"
output:
schema_file: "db/intermediate_db_schema/100-base-schema.sql"
models_directory: "lib/database/intermediate_db"
models_namespace: Migrations::Database::IntermediateDB
## Tables to include in the generated base intermediate schema.
##
## Available table options:
## virtual: Boolean. Enables the inclusion of a table in the schema solely based.
## on the provided configuration. A virtual table does not need to be available in the core schema.
## ignore: List of columns to ignore. Convenient if most of the table's column are needed.
## Usage is mutually exclusive with the `include` option. Only one should be used at a time.
## include: List of columns to include. Convenient if only a few columns are needed.
## Usage is mutually exclusive with the `include`` option. Only one should be used at a time.
## primary_key: Literal or list of columns to use as primary key.
## extend: List of objects describing columns to be added/extended.
## The following options are available for an "extend" object:
## name: Required. The name of the column being extended.
## is_null: Specifies if the column can be null.
## type: Column type. Defaults to TEXT.
## indexes: List of indexes to create. The following options are available for an "index" object:
## name: Index name.
## columns: List of column(s) to index.
schema:
tables:
users:
ignore:
- flag_level
- last_emailed_at
- last_posted_at
- last_seen_reviewable_id
- password_algorithm
- password_hash
- salt
- secure_identifier
- seen_notification_id
- username_lower
## Schema-wide column configuration options. These options apply to all tables.
## See table specific column configuration options above.
##
## Available Options:
## ignore: List of core/plugin table columns to ignore and exclude from intermediate schema.
user_emails:
columns:
ignore:
- updated_at
include:
- "email"
- "primary"
- "user_id"
- "created_at"
primary_key_column_names: [ "email" ]
user_options:
primary_key_column_names: [ "user_id" ]
users:
columns:
add:
- name: "original_username"
datatype: text
exclude:
- "flag_level"
- "last_emailed_at"
- "last_posted_at"
- "last_seen_reviewable_id"
- "previous_visit_at"
- "required_fields_version"
- "secure_identifier"
- "seen_notification_id"
- "suspended_at"
- "suspended_till"
- "username_lower"
global:
columns:
modify:
- name: "id"
datatype: numeric
rename_to: "original_id"
- name_regex: ".*upload.*_id$"
datatype: text
- name_regex: ".*_id$"
datatype: numeric
exclude:
- "updated_at"
tables:
exclude:
- "admin_notices"
- "allowed_pm_users"
- "anonymous_users"
- "api_key_scopes"
- "api_keys"
- "application_requests"
- "ar_internal_metadata"
- "associated_groups"
- "backup_draft_posts"
- "backup_draft_topics"
- "backup_metadata"
- "badge_groupings"
- "badge_types"
- "badges"
- "bookmarks"
- "categories"
- "categories_web_hooks"
- "category_custom_fields"
- "category_featured_topics"
- "category_form_templates"
- "category_groups"
- "category_moderation_groups"
- "category_required_tag_groups"
- "category_search_data"
- "category_settings"
- "category_tag_groups"
- "category_tag_stats"
- "category_tags"
- "category_users"
- "chat_channel_archives"
- "chat_channel_custom_fields"
- "chat_channels"
- "chat_drafts"
- "chat_mention_notifications"
- "chat_mentions"
- "chat_message_custom_fields"
- "chat_message_interactions"
- "chat_message_reactions"
- "chat_message_revisions"
- "chat_messages"
- "chat_thread_custom_fields"
- "chat_threads"
- "chat_webhook_events"
- "child_themes"
- "color_scheme_colors"
- "color_schemes"
- "custom_emojis"
- "developers"
- "direct_message_channels"
- "direct_message_users"
- "directory_columns"
- "directory_items"
- "discourse_automation_automations"
- "discourse_automation_fields"
- "discourse_automation_pending_automations"
- "discourse_automation_pending_pms"
- "discourse_automation_stats"
- "discourse_automation_user_global_notices"
- "dismissed_topic_users"
- "do_not_disturb_timings"
- "draft_sequences"
- "drafts"
- "email_change_requests"
- "email_logs"
- "email_tokens"
- "embeddable_host_tags"
- "embeddable_hosts"
- "external_upload_stubs"
- "flags"
- "form_templates"
- "given_daily_likes"
- "group_archived_messages"
- "group_associated_groups"
- "group_category_notification_defaults"
- "group_custom_fields"
- "group_histories"
- "group_mentions"
- "group_requests"
- "group_tag_notification_defaults"
- "group_users"
- "groups"
- "groups_web_hooks"
- "ignored_users"
- "imap_sync_logs"
- "incoming_chat_webhooks"
- "incoming_domains"
- "incoming_emails"
- "incoming_links"
- "incoming_referers"
- "invited_groups"
- "invited_users"
- "invites"
- "javascript_caches"
- "linked_topics"
- "message_bus"
- "moved_posts"
- "muted_users"
- "notifications"
- "oauth2_user_infos"
- "onceoff_logs"
- "optimized_images"
- "permalinks"
- "plugin_store_rows"
- "poll_options"
- "poll_votes"
- "polls"
- "post_action_types"
- "post_actions"
- "post_custom_fields"
- "post_details"
- "post_hotlinked_media"
- "post_replies"
- "post_reply_keys"
- "post_revisions"
- "post_search_data"
- "post_stats"
- "post_timings"
- "posts"
- "problem_check_trackers"
- "published_pages"
- "push_subscriptions"
- "quoted_posts"
- "redelivering_webhook_events"
- "remote_themes"
- "reviewable_claimed_topics"
- "reviewable_histories"
- "reviewable_scores"
- "reviewables"
- "scheduler_stats"
- "schema_migration_details"
- "schema_migrations"
- "screened_emails"
- "screened_ip_addresses"
- "screened_urls"
- "search_logs"
- "shared_drafts"
- "shelved_notifications"
- "sidebar_section_links"
- "sidebar_sections"
- "sidebar_urls"
- "single_sign_on_records"
- "site_settings"
- "sitemaps"
- "skipped_email_logs"
- "stylesheet_cache"
- "summary_sections"
- "tag_group_memberships"
- "tag_group_permissions"
- "tag_groups"
- "tag_search_data"
- "tag_users"
- "tags"
- "tags_web_hooks"
- "theme_color_schemes"
- "theme_fields"
- "theme_modifier_sets"
- "theme_settings"
- "theme_settings_migrations"
- "theme_svg_sprites"
- "theme_translation_overrides"
- "themes"
- "top_topics"
- "topic_allowed_groups"
- "topic_allowed_users"
- "topic_custom_fields"
- "topic_embeds"
- "topic_groups"
- "topic_hot_scores"
- "topic_invites"
- "topic_link_clicks"
- "topic_links"
- "topic_search_data"
- "topic_tags"
- "topic_thumbnails"
- "topic_timers"
- "topic_users"
- "topic_view_stats"
- "topic_views"
- "topics"
- "translation_overrides"
- "unsubscribe_keys"
- "upload_references"
- "uploads"
- "user_actions"
- "user_api_key_client_scopes"
- "user_api_key_clients"
- "user_api_key_scopes"
- "user_api_keys"
- "user_archived_messages"
- "user_associated_accounts"
- "user_associated_groups"
- "user_auth_token_logs"
- "user_auth_tokens"
- "user_avatars"
- "user_badges"
- "user_chat_channel_memberships"
- "user_chat_thread_memberships"
- "user_custom_fields"
- "user_exports"
- "user_field_options"
- "user_fields"
- "user_histories"
- "user_ip_address_histories"
- "user_notification_schedules"
- "user_open_ids"
- "user_passwords"
- "user_profile_views"
- "user_profiles"
- "user_required_fields_versions"
- "user_search_data"
- "user_second_factors"
- "user_security_keys"
- "user_stats"
- "user_statuses"
- "user_uploads"
- "user_visits"
- "user_warnings"
- "watched_word_groups"
- "watched_words"
- "web_crawler_requests"
- "web_hook_event_types"
- "web_hook_event_types_hooks"
- "web_hook_events"
- "web_hook_events_daily_aggregates"
- "web_hooks"
plugins:
- "automation"
- "chat"
- "checklist"
- "discourse-details"
- "discourse-lazy-videos"
- "discourse-local-dates"
- "discourse-narrative-bot"
- "discourse-presence"
- "footnote"
- "poll"
- "spoiler-alert"
- "styleguide"

258
migrations/config/json_schemas/db_schema.json Normal file
View File

@ -0,0 +1,258 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$defs": {
"datatypes": {
"type": "string",
"enum": [
"blob",
"boolean",
"date",
"datetime",
"float",
"inet",
"integer",
"json",
"numeric",
"text"
]
}
},
"type": "object",
"required": [
"output",
"schema",
"plugins"
],
"properties": {
"output": {
"type": "object",
"additionalProperties": false,
"properties": {
"schema_file": {
"type": "string"
},
"models_directory": {
"type": "string"
},
"models_namespace": {
"type": "string"
}
},
"required": [
"schema_file",
"models_directory",
"models_namespace"
]
},
"schema": {
"type": "object",
"required": [
"tables",
"global"
],
"properties": {
"tables": {
"type": "object",
"additionalProperties": {
"type": "object",
"properties": {
"columns": {
"type": "object",
"properties": {
"exclude": {
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
},
"include": {
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
},
"modify": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"datatype": {
"$ref": "#/$defs/datatypes"
}
},
"additionalProperties": false,
"required": [
"name",
"datatype"
]
}
},
"add": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"datatype": {
"$ref": "#/$defs/datatypes"
},
"nullable": {
"type": "boolean"
},
"max_length": {
"type": "integer"
}
},
"additionalProperties": false,
"required": [
"name",
"datatype"
]
}
}
},
"additionalProperties": false,
"not": {
"required": [
"exclude",
"include"
]
}
},
"indexes": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"columns": {
"type": "array",
"items": {
"type": "string"
},
"uniqueItems": true
},
"unique": {
"type": "boolean"
},
"condition": {
"type": "string"
}
},
"additionalProperties": false,
"required": [
"name",
"columns"
]
}
},
"primary_key_column_names": {
"type": "array",
"items": {
"type": "string"
},
"uniqueItems": true
},
"copy_of": {
"type": "string"
}
},
"additionalProperties": false
}
},
"global": {
"type": "object",
"properties": {
"columns": {
"properties": {
"modify": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"name_regex": {
"type": "string",
"format": "regex"
},
"datatype": {
"$ref": "#/$defs/datatypes"
},
"rename_to": {
"type": "string"
}
},
"additionalProperties": false,
"anyOf": [
{
"required": [
"datatype"
]
},
{
"required": [
"rename_to"
]
}
],
"oneOf": [
{
"required": [
"name"
]
},
{
"required": [
"name_regex"
]
}
]
}
},
"exclude": {
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
}
}
},
"tables": {
"properties": {
"exclude": {
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
}
}
}
},
"additionalProperties": false
}
},
"additionalProperties": false
},
"plugins": {
"type": "array",
"uniqueItems": true,
"items": {
"type": "string"
}
}
},
"additionalProperties": false
}

32
migrations/config/locales/migrations.en.yml
View File

@ -19,3 +19,35 @@ en:
default_step_title: "Converting %{type}"
max_progress_calculation: "Calculating items took %{duration}"
schema:
validator:
include_exclude_not_allowed: "Cannot use `include` and `exclude` together at %{path}"
invalid_name_regex: "Invalid `name_regex`: %{message}"
output:
schema_file_directory_not_found: "Schema file directory not found"
models_directory_not_found: "Models directory not found"
models_namespace_undefined: "Models namespace not defined"
global:
excluded_tables_missing: "Missing globally excluded tables: %{table_names}"
excluded_columns_missing: "Missing globally excluded columns: %{column_names}"
modified_columns_missing: "Missing globally modified columns: %{column_names}"
tables:
excluded_tables_configured: "Excluded tables configured: %{table_names}"
not_configured: "Tables not configured or excluded: %{table_names}"
copy_table_not_found: "Can't copy table '%{table_name}' because it doesn't exist"
added_columns_exist: "Added columns already exist in '%{table_name}': %{column_names}"
included_columns_missing: "Included columns do not exist in '%{table_name}': %{column_names}"
excluded_columns_missing: "Excluded columns do not exist in '%{table_name}': %{column_names}"
modified_columns_missing: "Missing modified columns in '%{table_name}': %{column_names}"
modified_columns_included: "Modified columns included in '%{table_name}': %{column_names}"
modified_columns_excluded: "Modified columns excluded in '%{table_name}': %{column_names}"
modified_columns_globally_excluded: "Modified columns globally excluded in '%{table_name}': %{column_names}"
no_columns_configured: "No columns configured in '%{table_name}'"
plugins:
not_installed: "Plugins not installed: %{plugin_names}"
additional_installed: "Unconfigured plugins installed: %{plugin_names}"

9
migrations/db/intermediate_db_schema/004-user_suspensions.sql Normal file
View File

@ -0,0 +1,9 @@
CREATE TABLE user_suspensions
(
user_id NUMERIC NOT NULL,
suspended_at DATETIME NOT NULL,
suspended_till DATETIME,
suspended_by_id NUMERIC,
reason TEXT,
PRIMARY KEY (user_id, suspended_at)
);

112
migrations/db/intermediate_db_schema/100-base-schema.sql
View File

@ -1,37 +1,105 @@
/*
This file is auto-generated from the Discourse core database schema. Instead of editing it directly,
please update the `schema.yml` configuration file and re-run the `generate_schema` script to update it.
*/
-- This file is auto-generated from the IntermediateDB schema. To make changes,
-- update the "config/intermediate_db.yml" configuration file and then run
-- `bin/cli schema generate` to regenerate this file.
CREATE TABLE user_emails
(
email TEXT NOT NULL PRIMARY KEY,
created_at DATETIME NOT NULL,
"primary" BOOLEAN,
user_id NUMERIC NOT NULL
);
CREATE TABLE user_options
(
user_id NUMERIC NOT NULL PRIMARY KEY,
allow_private_messages BOOLEAN,
auto_track_topics_after_msecs INTEGER,
automatically_unpin_topics BOOLEAN,
bookmark_auto_delete_preference INTEGER,
chat_email_frequency INTEGER,
chat_enabled BOOLEAN,
chat_header_indicator_preference INTEGER,
chat_quick_reaction_type INTEGER,
chat_quick_reactions_custom TEXT,
chat_send_shortcut INTEGER,
chat_separate_sidebar_mode INTEGER,
chat_sound TEXT,
color_scheme_id NUMERIC,
dark_scheme_id NUMERIC,
default_calendar INTEGER,
digest_after_minutes INTEGER,
dismissed_channel_retention_reminder BOOLEAN,
dismissed_dm_retention_reminder BOOLEAN,
dynamic_favicon BOOLEAN,
email_digests BOOLEAN,
email_in_reply_to BOOLEAN,
email_level INTEGER,
email_messages_level INTEGER,
email_previous_replies INTEGER,
enable_allowed_pm_users BOOLEAN,
enable_defer BOOLEAN,
enable_experimental_sidebar BOOLEAN,
enable_quoting BOOLEAN,
enable_smart_lists BOOLEAN,
external_links_in_new_tab BOOLEAN,
hide_presence BOOLEAN,
hide_profile BOOLEAN,
hide_profile_and_presence BOOLEAN,
homepage_id NUMERIC,
ignore_channel_wide_mention BOOLEAN,
include_tl0_in_digests BOOLEAN,
last_redirected_to_top_at DATETIME,
like_notification_frequency INTEGER,
mailing_list_mode BOOLEAN,
mailing_list_mode_frequency INTEGER,
new_topic_duration_minutes INTEGER,
notification_level_when_replying INTEGER,
oldest_search_log_date DATETIME,
only_chat_push_notifications BOOLEAN,
seen_popups INTEGER,
show_thread_title_prompts BOOLEAN,
sidebar_link_to_filtered_list BOOLEAN,
sidebar_show_count_of_new_items BOOLEAN,
skip_new_user_tips BOOLEAN,
text_size_key INTEGER,
text_size_seq INTEGER,
theme_ids INTEGER,
theme_key_seq INTEGER,
timezone TEXT,
title_count_mode_key INTEGER,
topics_unread_when_closed BOOLEAN,
watched_precedence_over_muted BOOLEAN
);
CREATE TABLE users
(
id INTEGER NOT NULL PRIMARY KEY,
active BOOLEAN NOT NULL,
admin BOOLEAN NOT NULL,
approved BOOLEAN NOT NULL,
created_at DATETIME NOT NULL,
staged BOOLEAN NOT NULL,
trust_level INTEGER NOT NULL,
username TEXT NOT NULL,
views INTEGER NOT NULL,
original_id NUMERIC NOT NULL PRIMARY KEY,
active BOOLEAN,
admin BOOLEAN,
approved BOOLEAN,
approved_at DATETIME,
approved_by_id INTEGER,
approved_by_id NUMERIC,
created_at DATETIME NOT NULL,
date_of_birth DATE,
first_seen_at DATETIME,
flair_group_id INTEGER,
flair_group_id NUMERIC,
group_locked_trust_level INTEGER,
ip_address TEXT,
ip_address INET_TEXT,
last_seen_at DATETIME,
locale TEXT,
manual_locked_trust_level INTEGER,
moderator BOOLEAN,
name TEXT,
previous_visit_at DATETIME,
primary_group_id INTEGER,
registration_ip_address TEXT,
original_username TEXT,
primary_group_id NUMERIC,
registration_ip_address INET_TEXT,
silenced_till DATETIME,
suspended_at DATETIME,
suspended_till DATETIME,
staged BOOLEAN,
title TEXT,
uploaded_avatar_id TEXT
trust_level INTEGER NOT NULL,
uploaded_avatar_id TEXT,
username TEXT NOT NULL,
views INTEGER
);

90
migrations/lib/cli/schema_sub_command.rb Normal file
View File

@ -0,0 +1,90 @@
# frozen_string_literal: true
module Migrations::CLI
class SchemaSubCommand < Thor
Schema = ::Migrations::Database::Schema
desc "generate", "Generates the database schema"
method_option :db, type: :string, default: "intermediate_db", desc: "Name of the database"
def generate
db = options[:db]
config = load_config_file(db)
puts "Generating schema for #{db.bold}..."
::Migrations.load_rails_environment(quiet: true)
validate_config(config)
loader = Schema::Loader.new(config[:schema])
schema = loader.load_schema
header = file_header(db)
write_db_schema(config, header, schema)
write_db_models(config, header, schema)
puts "Done"
end
private
def validate_config(config)
validator = Schema::ConfigValidator.new
validator.validate(config)
if validator.has_errors?
validator.errors.each { |error| print_error(error) }
exit(2)
end
end
def write_db_schema(config, header, schema)
schema_file_path = File.expand_path(config.dig(:output, :schema_file), ::Migrations.root_path)
File.open(schema_file_path, "w") do |schema_file|
writer = Schema::TableWriter.new(schema_file)
writer.output_file_header(header)
schema.each { |table| writer.output_table(table) }
end
end
def write_db_models(config, header, schema)
writer = Schema::ModelWriter.new(config.dig(:output, :models_namespace), header)
models_path = File.expand_path(config.dig(:output, :models_directory), ::Migrations.root_path)
schema.each do |table|
model_file_path = File.join(models_path, Schema::ModelWriter.filename_for(table))
File.open(model_file_path, "w") { |model_file| writer.output_table(table, model_file) }
end
Schema::ModelWriter.format_files(models_path)
end
def relative_config_path(db)
File.join("config", "#{db}.yml")
end
def file_header(db)
<<~HEADER
This file is auto-generated from the IntermediateDB schema. To make changes,
update the "#{relative_config_path(db)}" configuration file and then run
`bin/cli schema generate` to regenerate this file.
HEADER
end
def load_config_file(db)
config_path = File.join(::Migrations.root_path, relative_config_path(db))
if !File.exist?(config_path)
print_error("Configuration file for #{db} wasn't found at '#{config_path}'")
exit 1
end
YAML.load_file(config_path, symbolize_names: true)
end
def print_error(message)
$stderr.puts "ERROR: ".red + message
end
end
end

4
migrations/lib/common/date_helper.rb
View File

@ -9,9 +9,9 @@ module Migrations
end
def self.track_time
start_time = Time.now
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
yield
Time.now - start_time
Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
end
end
end

105
migrations/lib/database/intermediate_db/user.rb Normal file
View File

@ -0,0 +1,105 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
# update the "config/intermediate_db.yml" configuration file and then run
# `bin/cli schema generate` to regenerate this file.
module Migrations::Database::IntermediateDB
module User
SQL = <<~SQL
INSERT INTO users (
original_id,
active,
admin,
approved,
approved_at,
approved_by_id,
created_at,
date_of_birth,
first_seen_at,
flair_group_id,
group_locked_trust_level,
ip_address,
last_seen_at,
locale,
manual_locked_trust_level,
moderator,
name,
original_username,
primary_group_id,
registration_ip_address,
silenced_till,
staged,
title,
trust_level,
uploaded_avatar_id,
username,
views
)
VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
SQL
def self.create(
original_id:,
active: nil,
admin: nil,
approved: nil,
approved_at: nil,
approved_by_id: nil,
created_at:,
date_of_birth: nil,
first_seen_at: nil,
flair_group_id: nil,
group_locked_trust_level: nil,
ip_address: nil,
last_seen_at: nil,
locale: nil,
manual_locked_trust_level: nil,
moderator: nil,
name: nil,
original_username: nil,
primary_group_id: nil,
registration_ip_address: nil,
silenced_till: nil,
staged: nil,
title: nil,
trust_level:,
uploaded_avatar_id: nil,
username:,
views: nil
)
::Migrations::Database::IntermediateDB.insert(
SQL,
original_id,
::Migrations::Database.format_boolean(active),
::Migrations::Database.format_boolean(admin),
::Migrations::Database.format_boolean(approved),
::Migrations::Database.format_datetime(approved_at),
approved_by_id,
::Migrations::Database.format_datetime(created_at),
::Migrations::Database.format_date(date_of_birth),
::Migrations::Database.format_datetime(first_seen_at),
flair_group_id,
group_locked_trust_level,
::Migrations::Database.format_ip_address(ip_address),
::Migrations::Database.format_datetime(last_seen_at),
locale,
manual_locked_trust_level,
::Migrations::Database.format_boolean(moderator),
name,
original_username,
primary_group_id,
::Migrations::Database.format_ip_address(registration_ip_address),
::Migrations::Database.format_datetime(silenced_till),
::Migrations::Database.format_boolean(staged),
title,
trust_level,
uploaded_avatar_id,
username,
views,
)
end
end
end

31
migrations/lib/database/intermediate_db/user_email.rb Normal file
View File

@ -0,0 +1,31 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
# update the "config/intermediate_db.yml" configuration file and then run
# `bin/cli schema generate` to regenerate this file.
module Migrations::Database::IntermediateDB
module UserEmail
SQL = <<~SQL
INSERT INTO user_emails (
email,
created_at,
"primary",
user_id
)
VALUES (
?, ?, ?, ?
)
SQL
def self.create(email:, created_at:, primary: nil, user_id:)
::Migrations::Database::IntermediateDB.insert(
SQL,
email,
::Migrations::Database.format_datetime(created_at),
::Migrations::Database.format_boolean(primary),
user_id,
)
end
end
end

199
migrations/lib/database/intermediate_db/user_option.rb Normal file
View File

@ -0,0 +1,199 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
# update the "config/intermediate_db.yml" configuration file and then run
# `bin/cli schema generate` to regenerate this file.
module Migrations::Database::IntermediateDB
module UserOption
SQL = <<~SQL
INSERT INTO user_options (
user_id,
allow_private_messages,
auto_track_topics_after_msecs,
automatically_unpin_topics,
bookmark_auto_delete_preference,
chat_email_frequency,
chat_enabled,
chat_header_indicator_preference,
chat_quick_reaction_type,
chat_quick_reactions_custom,
chat_send_shortcut,
chat_separate_sidebar_mode,
chat_sound,
color_scheme_id,
dark_scheme_id,
default_calendar,
digest_after_minutes,
dismissed_channel_retention_reminder,
dismissed_dm_retention_reminder,
dynamic_favicon,
email_digests,
email_in_reply_to,
email_level,
email_messages_level,
email_previous_replies,
enable_allowed_pm_users,
enable_defer,
enable_experimental_sidebar,
enable_quoting,
enable_smart_lists,
external_links_in_new_tab,
hide_presence,
hide_profile,
hide_profile_and_presence,
homepage_id,
ignore_channel_wide_mention,
include_tl0_in_digests,
last_redirected_to_top_at,
like_notification_frequency,
mailing_list_mode,
mailing_list_mode_frequency,
new_topic_duration_minutes,
notification_level_when_replying,
oldest_search_log_date,
only_chat_push_notifications,
seen_popups,
show_thread_title_prompts,
sidebar_link_to_filtered_list,
sidebar_show_count_of_new_items,
skip_new_user_tips,
text_size_key,
text_size_seq,
theme_ids,
theme_key_seq,
timezone,
title_count_mode_key,
topics_unread_when_closed,
watched_precedence_over_muted
)
VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
SQL
def self.create(
user_id:,
allow_private_messages: nil,
auto_track_topics_after_msecs: nil,
automatically_unpin_topics: nil,
bookmark_auto_delete_preference: nil,
chat_email_frequency: nil,
chat_enabled: nil,
chat_header_indicator_preference: nil,
chat_quick_reaction_type: nil,
chat_quick_reactions_custom: nil,
chat_send_shortcut: nil,
chat_separate_sidebar_mode: nil,
chat_sound: nil,
color_scheme_id: nil,
dark_scheme_id: nil,
default_calendar: nil,
digest_after_minutes: nil,
dismissed_channel_retention_reminder: nil,
dismissed_dm_retention_reminder: nil,
dynamic_favicon: nil,
email_digests: nil,
email_in_reply_to: nil,
email_level: nil,
email_messages_level: nil,
email_previous_replies: nil,
enable_allowed_pm_users: nil,
enable_defer: nil,
enable_experimental_sidebar: nil,
enable_quoting: nil,
enable_smart_lists: nil,
external_links_in_new_tab: nil,
hide_presence: nil,
hide_profile: nil,
hide_profile_and_presence: nil,
homepage_id: nil,
ignore_channel_wide_mention: nil,
include_tl0_in_digests: nil,
last_redirected_to_top_at: nil,
like_notification_frequency: nil,
mailing_list_mode: nil,
mailing_list_mode_frequency: nil,
new_topic_duration_minutes: nil,
notification_level_when_replying: nil,
oldest_search_log_date: nil,
only_chat_push_notifications: nil,
seen_popups: nil,
show_thread_title_prompts: nil,
sidebar_link_to_filtered_list: nil,
sidebar_show_count_of_new_items: nil,
skip_new_user_tips: nil,
text_size_key: nil,
text_size_seq: nil,
theme_ids: nil,
theme_key_seq: nil,
timezone: nil,
title_count_mode_key: nil,
topics_unread_when_closed: nil,
watched_precedence_over_muted: nil
)
::Migrations::Database::IntermediateDB.insert(
SQL,
user_id,
::Migrations::Database.format_boolean(allow_private_messages),
auto_track_topics_after_msecs,
::Migrations::Database.format_boolean(automatically_unpin_topics),
bookmark_auto_delete_preference,
chat_email_frequency,
::Migrations::Database.format_boolean(chat_enabled),
chat_header_indicator_preference,
chat_quick_reaction_type,
chat_quick_reactions_custom,
chat_send_shortcut,
chat_separate_sidebar_mode,
chat_sound,
color_scheme_id,
dark_scheme_id,
default_calendar,
digest_after_minutes,
::Migrations::Database.format_boolean(dismissed_channel_retention_reminder),
::Migrations::Database.format_boolean(dismissed_dm_retention_reminder),
::Migrations::Database.format_boolean(dynamic_favicon),
::Migrations::Database.format_boolean(email_digests),
::Migrations::Database.format_boolean(email_in_reply_to),
email_level,
email_messages_level,
email_previous_replies,
::Migrations::Database.format_boolean(enable_allowed_pm_users),
::Migrations::Database.format_boolean(enable_defer),
::Migrations::Database.format_boolean(enable_experimental_sidebar),
::Migrations::Database.format_boolean(enable_quoting),
::Migrations::Database.format_boolean(enable_smart_lists),
::Migrations::Database.format_boolean(external_links_in_new_tab),
::Migrations::Database.format_boolean(hide_presence),
::Migrations::Database.format_boolean(hide_profile),
::Migrations::Database.format_boolean(hide_profile_and_presence),
homepage_id,
::Migrations::Database.format_boolean(ignore_channel_wide_mention),
::Migrations::Database.format_boolean(include_tl0_in_digests),
::Migrations::Database.format_datetime(last_redirected_to_top_at),
like_notification_frequency,
::Migrations::Database.format_boolean(mailing_list_mode),
mailing_list_mode_frequency,
new_topic_duration_minutes,
notification_level_when_replying,
::Migrations::Database.format_datetime(oldest_search_log_date),
::Migrations::Database.format_boolean(only_chat_push_notifications),
seen_popups,
::Migrations::Database.format_boolean(show_thread_title_prompts),
::Migrations::Database.format_boolean(sidebar_link_to_filtered_list),
::Migrations::Database.format_boolean(sidebar_show_count_of_new_items),
::Migrations::Database.format_boolean(skip_new_user_tips),
text_size_key,
text_size_seq,
theme_ids,
theme_key_seq,
timezone,
title_count_mode_key,
::Migrations::Database.format_boolean(topics_unread_when_closed),
::Migrations::Database.format_boolean(watched_precedence_over_muted),
)
end
end
end

29
migrations/lib/database/intermediate_db/user_suspension.rb Normal file
View File

@ -0,0 +1,29 @@
# frozen_string_literal: true
module Migrations::Database::IntermediateDB
module UserSuspension
SQL = <<~SQL
INSERT INTO user_suspensions (
user_id,
suspended_at,
suspended_till,
suspended_by_id,
reason
)
VALUES (
?, ?, ?, ?, ?
)
SQL
def self.create(user_id:, suspended_at:, suspended_till: nil, suspended_by_id: nil, reason: nil)
::Migrations::Database::IntermediateDB.insert(
SQL,
user_id,
::Migrations::Database.format_datetime(suspended_at),
::Migrations::Database.format_datetime(suspended_till),
suspended_by_id,
reason,
)
end
end
end

175
migrations/lib/database/schema.rb Normal file
View File

@ -0,0 +1,175 @@
# frozen_string_literal: true
module Migrations::Database
module Schema
Table =
Data.define(:name, :columns, :indexes, :primary_key_column_names) do
def sorted_columns
columns.sort_by { |c| [c.is_primary_key ? 0 : 1, c.name] }
end
end
Column = Data.define(:name, :datatype, :nullable, :max_length, :is_primary_key)
Index = Data.define(:name, :column_names, :unique, :condition)
class ConfigError < StandardError
end
SQLITE_KEYWORDS = %w[
abort
action
add
after
all
alter
always
analyze
and
as
asc
attach
autoincrement
before
begin
between
by
cascade
case
cast
check
collate
column
commit
conflict
constraint
create
cross
current
current_date
current_time
current_timestamp
database
default
deferrable
deferred
delete
desc
detach
distinct
do
drop
each
else
end
escape
except
exclude
exclusive
exists
explain
fail
filter
first
following
for
foreign
from
full
generated
glob
group
groups
having
if
ignore
immediate
in
index
indexed
initially
inner
insert
instead
intersect
into
is
isnull
join
key
last
left
like
limit
match
materialized
natural
no
not
nothing
notnull
null
nulls
of
offset
on
or
order
others
outer
over
partition
plan
pragma
preceding
primary
query
raise
range
recursive
references
regexp
reindex
release
rename
replace
restrict
returning
right
rollback
row
rows
savepoint
select
set
table
temp
temporary
then
ties
to
transaction
trigger
unbounded
union
unique
update
using
vacuum
values
view
virtual
when
where
window
with
without
]
def self.escape_identifier(identifier)
if SQLITE_KEYWORDS.include?(identifier)
%Q("#{identifier}")
else
identifier
end
end
end
end

46
migrations/lib/database/schema/config_validator.rb Normal file
View File

@ -0,0 +1,46 @@
# frozen_string_literal: true
module Migrations::Database::Schema
class ConfigValidator
attr_reader :errors
def initialize
@errors = []
end
def validate(config)
@errors.clear
validate_with_json_schema(config)
return self if has_errors?
validate_output_config(config)
validate_schema_config(config)
validate_plugins(config)
self
end
def has_errors?
@errors.any?
end
private
def validate_with_json_schema(config)
Validation::JsonSchemaValidator.new(config, @errors).validate
end
def validate_output_config(config)
Validation::OutputConfigValidator.new(config, @errors).validate
end
def validate_schema_config(config)
Validation::SchemaConfigValidator.new(config, @errors).validate
end
def validate_plugins(config)
Validation::PluginConfigValidator.new(config, @errors).validate
end
end
end

61
migrations/lib/database/schema/global_config.rb Normal file
View File

@ -0,0 +1,61 @@
# frozen_string_literal: true
module Migrations::Database::Schema
class GlobalConfig
attr_reader :excluded_column_names, :modified_columns
def initialize(schema_config)
@schema_config = schema_config
@excluded_table_names = load_globally_excluded_table_names.freeze
@excluded_column_names = load_globally_excluded_column_names.freeze
@modified_columns = load_globally_modified_columns.freeze
end
def excluded_table_name?(table_name)
@excluded_table_names.include?(table_name)
end
def modified_name(column_name)
if (modified_column = find_modified_column(column_name))
modified_column[:rename_to]
end
end
def modified_datatype(column_name)
if (modified_column = find_modified_column(column_name))
modified_column[:datatype]
end
end
private
def find_modified_column(column_name)
@modified_columns.find { |column| column[:name] == column_name } ||
@modified_columns.find { |column| column[:name_regex]&.match?(column_name) }
end
def load_globally_excluded_table_names
table_names = @schema_config.dig(:global, :tables, :exclude)
table_names.presence&.to_set || Set.new
end
def load_globally_excluded_column_names
column_names = @schema_config.dig(:global, :columns, :exclude)
column_names.presence || []
end
def load_globally_modified_columns
modified_columns = @schema_config.dig(:global, :columns, :modify)
return {} if modified_columns.blank?
modified_columns.map do |column|
if column[:name_regex]
column[:name_regex_original] = column[:name_regex]
column[:name_regex] = Regexp.new(column[:name_regex])
end
column[:datatype] = column[:datatype]&.to_sym
column
end
end
end
end

116
migrations/lib/database/schema/loader.rb Normal file
View File

@ -0,0 +1,116 @@
# frozen_string_literal: true
module Migrations::Database::Schema
class Loader
def initialize(schema_config)
@schema_config = schema_config
@global = GlobalConfig.new(@schema_config)
end
def load_schema
@db = ActiveRecord::Base.lease_connection
schema = []
existing_table_names = @db.tables.to_set
@schema_config[:tables].sort.each do |table_name, config|
table_name = table_name.to_s
if config[:copy_of].present?
table_alias = table_name
table_name = config[:copy_of]
else
next if @global.excluded_table_name?(table_name)
end
if existing_table_names.include?(table_name)
schema << table(table_name, config, table_alias)
end
end
@db = nil
ActiveRecord::Base.release_connection
schema
end
private
def table(table_name, config, table_alias = nil)
primary_key_column_names =
config[:primary_key_column_names].presence || @db.primary_keys(table_name)
columns =
filtered_columns_of(table_name, config).map do |column|
Column.new(
name: name_for(column),
datatype: datatype_for(column),
nullable: column.null || column.default,
max_length: column.type == :text ? column.limit : nil,
is_primary_key: primary_key_column_names.include?(column.name),
)
end + added_columns(config, primary_key_column_names)
Table.new(table_alias || table_name, columns, indexes(config), primary_key_column_names)
end
def filtered_columns_of(table_name, config)
columns_by_name = @db.columns(table_name).index_by(&:name)
columns_by_name.except!(*@global.excluded_column_names)
if (included_columns = config.dig(:columns, :include))
columns_by_name.slice!(*included_columns)
elsif (excluded_columns = config.dig(:columns, :exclude))
columns_by_name.except!(*excluded_columns)
end
columns_by_name.values
end
def added_columns(config, primary_key_column_names)
columns = config.dig(:columns, :add) || []
columns.map do |column|
datatype = column[:datatype].to_sym
Column.new(
name: column[:name],
datatype:,
nullable: column.fetch(:nullable, true),
max_length: datatype == :text ? column[:max_length] : nil,
is_primary_key: primary_key_column_names.include?(column[:name]),
)
end
end
def name_for(column)
@global.modified_name(column.name) || column.name
end
def datatype_for(column)
datatype = @global.modified_datatype(column.name) || column.type
case datatype
when :binary
:blob
when :string, :enum, :uuid
:text
when :jsonb
:json
when :boolean, :date, :datetime, :float, :inet, :integer, :numeric, :json, :text
datatype
else
raise "Unknown datatype: #{datatype}"
end
end
def indexes(config)
config[:indexes]&.map do |index|
Index.new(
name: index[:name],
column_names: Array.wrap(index[:columns]),
unique: index.fetch(:unique, false),
condition: index[:condition],
)
end
end
end
end

139
migrations/lib/database/schema/model_writer.rb Normal file
View File

@ -0,0 +1,139 @@
# frozen_string_literal: true
require "rake"
require "syntax_tree/rake_tasks"
module Migrations::Database::Schema
class ModelWriter
def initialize(namespace, header)
@namespace = namespace
@header = header.gsub(/^/, "# ")
end
def self.filename_for(table)
"#{table.name.singularize}.rb"
end
def self.format_files(path)
glob_pattern = File.join(path, "**/*.rb")
system(
"bundle",
"exec",
"stree",
"write",
glob_pattern,
exception: true,
out: File::NULL,
err: File::NULL,
)
rescue StandardError
raise "Failed to run `bundle exec stree write '#{glob_pattern}'`"
end
def output_table(table, output_stream)
columns = table.sorted_columns
output_stream.puts "# frozen_string_literal: true"
output_stream.puts
output_stream.puts @header
output_stream.puts
output_stream.puts "module #{@namespace}"
output_stream.puts " module #{to_singular_classname(table.name)}"
output_stream.puts " SQL = <<~SQL"
output_stream.puts " INSERT INTO #{escape_identifier(table.name)} ("
output_stream.puts column_names(columns)
output_stream.puts " )"
output_stream.puts " VALUES ("
output_stream.puts value_placeholders(columns)
output_stream.puts " )"
output_stream.puts " SQL"
output_stream.puts
output_stream.puts " def self.create("
output_stream.puts method_parameters(columns)
output_stream.puts " )"
output_stream.puts " ::Migrations::Database::IntermediateDB.insert("
output_stream.puts " SQL,"
output_stream.puts insertion_arguments(columns)
output_stream.puts " )"
output_stream.puts " end"
output_stream.puts " end"
output_stream.puts "end"
end
private
def to_singular_classname(snake_case_string)
snake_case_string.singularize.camelize
end
def column_names(columns)
columns.map { |c| " #{escape_identifier(c.name)}" }.join(",\n")
end
def value_placeholders(columns)
indentation = " "
max_length = 100 - indentation.length
placeholder = "?, "
placeholder_count = columns.size
current_length = 0
placeholders = indentation.dup
(1..placeholder_count).each do |index|
placeholder = "?" if index == placeholder_count
if current_length + placeholder.length > max_length
placeholders.rstrip!
placeholders << "\n" << indentation
current_length = 0
end
placeholders << placeholder
current_length += placeholder.length
end
placeholders
end
def method_parameters(columns)
columns
.map do |c|
default_value = !c.is_primary_key && c.nullable ? " nil" : ""
" #{c.name}:#{default_value}"
end
.join(",\n")
end
def insertion_arguments(columns)
columns
.map do |c|
argument =
case c.datatype
when :datetime
"::Migrations::Database.format_datetime(#{c.name})"
when :date
"::Migrations::Database.format_date(#{c.name})"
when :boolean
"::Migrations::Database.format_boolean(#{c.name})"
when :inet
"::Migrations::Database.format_ip_address(#{c.name})"
when :blob
"::Migrations::Database.to_blob(#{c.name})"
when :json
"::Migrations::Database.to_json(#{c.name})"
when :float, :integer, :numeric, :text
c.name
else
raise "Unknown dataype: #{type}"
end
" #{argument},"
end
.join("\n")
end
def escape_identifier(identifier)
::Migrations::Database::Schema.escape_identifier(identifier)
end
end
end

104
migrations/lib/database/schema/table_writer.rb Normal file
View File

@ -0,0 +1,104 @@
# frozen_string_literal: true
module Migrations::Database::Schema
class TableWriter
def initialize(output_stream)
@output = output_stream
end
def output_file_header(header)
@output.puts header.gsub(/^/, "-- ")
@output.puts
end
def output_table(table)
output_create_table_statement(table)
output_columns(table)
output_primary_key(table)
output_indexes(table)
@output.puts ""
end
private
def output_create_table_statement(table)
@output.puts "CREATE TABLE #{escape_identifier(table.name)}"
@output.puts "("
end
def output_columns(table)
column_definitions = create_column_definitions(table)
@output.puts column_definitions.join(",\n")
end
def output_primary_key(table)
if table.primary_key_column_names.size > 1
pk_definition =
table.primary_key_column_names.map { |name| escape_identifier(name) }.join(", ")
@output.puts " PRIMARY KEY (#{pk_definition})"
end
@output.puts ");"
end
def create_column_definitions(table)
columns = table.sorted_columns
has_composite_primary_key = table.primary_key_column_names.size > 1
max_column_name_length = columns.map { |c| escape_identifier(c.name).length }.max
max_datatype_length = columns.map { |c| convert_datatype(c.datatype).length }.max
columns.map do |c|
definition = [
escape_identifier(c.name).ljust(max_column_name_length),
convert_datatype(c.datatype).ljust(max_datatype_length),
]
if c.is_primary_key && !has_composite_primary_key
definition << "NOT NULL" if c.datatype != :integer
definition << "PRIMARY KEY"
else
definition << "NOT NULL" unless c.nullable
end
definition = definition.join(" ")
definition.strip!
" #{definition}"
end
end
def convert_datatype(type)
case type
when :blob, :boolean, :date, :datetime, :float, :integer, :numeric, :text
type.to_s.upcase
when :inet
"INET_TEXT"
when :json
"JSON_TEXT"
else
raise "Unknown datatype: #{type}"
end
end
def escape_identifier(identifier)
::Migrations::Database::Schema.escape_identifier(identifier)
end
def output_indexes(table)
return unless table.indexes
table.indexes.each do |index|
index_name = escape_identifier(index.name)
table_name = escape_identifier(table.name)
column_names = index.column_names.map { |name| escape_identifier(name) }
@output.puts ""
@output.print "CREATE "
@output.print "UNIQUE " if index.unique
@output.print "INDEX #{index_name} ON #{table_name} (#{column_names.join(", ")})"
@output.print " #{index.condition}" if index.condition.present?
@output.puts ";"
end
end
end
end

18
migrations/lib/database/schema/validation/base_validator.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class BaseValidator
def initialize(config, errors, db)
@config = config
@schema_config ||= @config[:schema]
@errors = errors
@db = db
end
private
def sort_and_join(values)
values.sort.join(", ")
end
end
end

115
migrations/lib/database/schema/validation/columns_validator.rb Normal file
View File

@ -0,0 +1,115 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class ColumnsValidator < BaseValidator
def validate(table_name, copy_of_table_name = nil)
@table_name = table_name
columns = @schema_config[:tables][table_name.to_sym][:columns] || {}
@existing_column_names = find_existing_column_names(table_name, copy_of_table_name)
@added_column_names = columns[:add]&.map { |column| column[:name] } || []
@included_column_names = columns[:include] || []
@excluded_column_names = columns[:exclude] || []
@modified_column_names = columns[:modify]&.map { |column| column[:name] } || []
@global = ::Migrations::Database::Schema::GlobalConfig.new(@schema_config)
validated_added_columns
validate_included_columns
validate_excluded_columns
validate_modified_columns
validate_any_columns_configured
end
private
def find_existing_column_names(table_name, copy_of_table_name)
table_name = copy_of_table_name if copy_of_table_name.present?
@db.columns(table_name).map { |c| c.name }
end
def validated_added_columns
if (column_names = @existing_column_names & @added_column_names).any?
@errors << I18n.t(
"schema.validator.tables.added_columns_exist",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
end
def validate_included_columns
if (column_names = @included_column_names - @existing_column_names).any?
@errors << I18n.t(
"schema.validator.tables.included_columns_missing",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
end
def validate_excluded_columns
if (column_names = @excluded_column_names - @existing_column_names).any?
@errors << I18n.t(
"schema.validator.tables.excluded_columns_missing",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
end
def validate_modified_columns
if (column_names = @modified_column_names - @existing_column_names).any?
@errors << I18n.t(
"schema.validator.tables.modified_columns_missing",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
if (column_names = @modified_column_names & @included_column_names).any?
@errors << I18n.t(
"schema.validator.tables.modified_columns_included",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
if (column_names = @modified_column_names & @excluded_column_names).any?
@errors << I18n.t(
"schema.validator.tables.modified_columns_excluded",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
if (column_names = @modified_column_names & @global.excluded_column_names).any?
@errors << I18n.t(
"schema.validator.tables.modified_columns_globally_excluded",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
end
def configured_column_names
if @included_column_names.any?
included_column_names = @included_column_names
modified_column_names = @modified_column_names
else
included_column_names = @existing_column_names - @excluded_column_names
modified_column_names = included_column_names & @modified_column_names
end
column_names = (included_column_names + modified_column_names).uniq & @existing_column_names
column_names - @global.excluded_column_names + @added_column_names
end
def validate_any_columns_configured
column_names = configured_column_names
if column_names.empty?
@errors << I18n.t("schema.validator.tables.no_columns_configured", table_name: @table_name)
end
end
end
end

73
migrations/lib/database/schema/validation/globally_configured_columns_validator.rb Normal file
View File

@ -0,0 +1,73 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class GloballyConfiguredColumnsValidator < BaseValidator
def initialize(config, errors, db)
super
@global = ::Migrations::Database::Schema::GlobalConfig.new(@schema_config)
end
def validate
all_column_names = calculate_all_column_names
validate_excluded_column_names(all_column_names)
validate_modified_columns(all_column_names)
end
private
def calculate_all_column_names
existing_table_names = @db.tables
configured_table_names =
@schema_config[:tables].map do |table_name, table_config|
table_config[:copy_of] || table_name.to_s
end
globally_excluded_table_names = @schema_config.dig(:global, :tables, :exclude) || []
excluded_table_names = @schema_config.dig(:global, :tables, :exclude) || []
all_table_names = existing_table_names - globally_excluded_table_names - excluded_table_names
all_table_names = all_table_names.uniq & configured_table_names
all_table_names.flat_map { |table_name| @db.columns(table_name).map(&:name) }.uniq
end
def validate_excluded_column_names(all_column_names)
globally_excluded_column_names = @global.excluded_column_names
excluded_missing_column_names = globally_excluded_column_names - all_column_names
if excluded_missing_column_names.any?
@errors << I18n.t(
"schema.validator.global.excluded_columns_missing",
column_names: sort_and_join(excluded_missing_column_names),
)
end
end
def validate_modified_columns(all_column_names)
globally_modified_columns = @global.modified_columns
excluded_missing_columns =
globally_modified_columns.reject do |column|
if column[:name]
all_column_names.include?(column[:name])
elsif column[:name_regex]
all_column_names.any? { |column_name| column[:name_regex]&.match?(column_name) }
else
false
end
end
if excluded_missing_columns.any?
excluded_missing_column_names =
excluded_missing_columns.map { |column| column[:name_regex_original] || column[:name] }
@errors << I18n.t(
"schema.validator.global.modified_columns_missing",
column_names: sort_and_join(excluded_missing_column_names),
)
end
end
end
end

20
migrations/lib/database/schema/validation/globally_excluded_tables_validator.rb Normal file
View File

@ -0,0 +1,20 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class GloballyExcludedTablesValidator < BaseValidator
def validate
excluded_table_names = @schema_config.dig(:global, :tables, :exclude)
return if excluded_table_names.blank?
existing_table_names = @db.tables
missing_table_names = excluded_table_names - existing_table_names
if missing_table_names.any?
@errors << I18n.t(
"schema.validator.global.excluded_tables_missing",
table_names: sort_and_join(missing_table_names),
)
end
end
end
end

33
migrations/lib/database/schema/validation/json_schema_validator.rb Normal file
View File

@ -0,0 +1,33 @@
# frozen_string_literal: true
require "json_schemer"
module Migrations::Database::Schema::Validation
class JsonSchemaValidator
def initialize(config, errors)
@config = config
@errors = errors
end
def validate
schema = load_json_schema
schemer = ::JSONSchemer.schema(schema)
response = schemer.validate(@config)
response.each { |r| @errors << transform_json_schema_errors(r.fetch("error")) }
end
private
def load_json_schema
schema_path = File.join(::Migrations.root_path, "config", "json_schemas", "db_schema.json")
JSON.load_file(schema_path)
end
def transform_json_schema_errors(error_message)
error_message.gsub!(/value at (`.+?`) matches `not` schema/) do
I18n.t("schema.validator.include_exclude_not_allowed", path: $1)
end
error_message
end
end
end

46
migrations/lib/database/schema/validation/output_config_validator.rb Normal file
View File

@ -0,0 +1,46 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class OutputConfigValidator < BaseValidator
def initialize(config, errors)
super(config, errors, nil)
@output_config = config[:output]
end
def validate
validate_schema_file_directory
validate_models_directory
validate_models_namespace
end
private
def validate_schema_file_directory
schema_file_path = File.dirname(@output_config[:schema_file])
schema_file_path = File.expand_path(schema_file_path, ::Migrations.root_path)
if !Dir.exist?(schema_file_path)
@errors << I18n.t("schema.validator.output.schema_file_directory_not_found")
end
end
def validate_models_directory
models_directory = File.expand_path(@output_config[:models_directory], ::Migrations.root_path)
if !Dir.exist?(models_directory)
@errors << I18n.t("schema.validator.output.models_directory_not_found")
end
end
def validate_models_namespace
existing_namespace =
begin
Object.const_get(@output_config[:models_namespace]).is_a?(Module)
rescue NameError
false
end
@errors << I18n.t("schema.validator.output.models_namespace_undefined") if !existing_namespace
end
end
end

28
migrations/lib/database/schema/validation/plugin_config_validator.rb Normal file
View File

@ -0,0 +1,28 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class PluginConfigValidator < BaseValidator
def initialize(config, errors)
super(config, errors, nil)
end
def validate
all_plugin_names = Discourse.plugins.map(&:name)
configured_plugin_names = @config[:plugins]
if (additional_plugins = all_plugin_names - configured_plugin_names).any?
@errors << I18n.t(
"schema.validator.plugins.additional_installed",
plugin_names: sort_and_join(additional_plugins),
)
end
if (missing_plugins = configured_plugin_names - all_plugin_names).any?
@errors << I18n.t(
"schema.validator.plugins.not_installed",
plugin_names: sort_and_join(missing_plugins),
)
end
end
end
end

18
migrations/lib/database/schema/validation/schema_config_validator.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class SchemaConfigValidator
def initialize(config, errors)
@config = config
@errors = errors
end
def validate
ActiveRecord::Base.with_connection do |db|
GloballyExcludedTablesValidator.new(@config, @errors, db).validate
GloballyConfiguredColumnsValidator.new(@config, @errors, db).validate
TablesValidator.new(@config, @errors, db).validate
end
end
end
end

70
migrations/lib/database/schema/validation/tables_validator.rb Normal file
View File

@ -0,0 +1,70 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class TablesValidator < BaseValidator
def initialize(config, errors, db)
super
@existing_table_names = @db.tables
@configured_tables = @schema_config[:tables]
@configured_table_names = @configured_tables.keys.map(&:to_s)
@excluded_table_names = @schema_config.dig(:global, :tables, :exclude) || []
end
def validate
validate_excluded_tables
validate_unconfigured_tables
validate_copied_tables
validate_columns
end
private
def validate_excluded_tables
table_names = @configured_table_names & @excluded_table_names
if table_names.any?
@errors << I18n.t(
"schema.validator.tables.excluded_tables_configured",
table_names: sort_and_join(table_names),
)
end
end
def validate_unconfigured_tables
table_names = @existing_table_names - @configured_table_names - @excluded_table_names
if table_names.any?
@errors << I18n.t(
"schema.validator.tables.not_configured",
table_names: sort_and_join(table_names),
)
end
end
def validate_copied_tables
@configured_tables.each do |_table_name, table_config|
next unless table_config[:copy_of]
if !@existing_table_names.include?(table_config[:copy_of])
@errors << I18n.t(
"schema.validator.tables.copy_table_not_found",
table_name: table_config[:copy_of],
)
end
end
end
def validate_columns
@configured_tables.each do |table_name, table_config|
validator = ColumnsValidator.new(@config, @errors, @db)
if table_config[:copy_of]
validator.validate(table_name.to_s, table_config[:copy_of])
else
validator.validate(table_name.to_s)
end
end
end
end
end

141
migrations/lib/importer/steps/users.rb Normal file
View File

@ -0,0 +1,141 @@
# frozen_string_literal: true
module Migrations::Importer::Steps
class Users < ::Migrations::Importer::CopyStep
INSERT_MAPPED_USERNAMES_SQL = <<~SQL
INSERT INTO mapped.usernames (discourse_user_id, original_username, discourse_username)
VALUES (?, ?, ?)
SQL
# requires_shared_data :usernames, :group_names
requires_mapping :user_ids_by_email, "SELECT LOWER(email) AS email, user_id FROM user_emails"
requires_mapping :user_ids_by_external_id,
"SELECT external_id, user_id FROM single_sign_on_records"
table_name :users
column_names %i[
id
username
username_lower
name
active
trust_level
group_locked_trust_level
manual_locked_trust_level
admin
moderator
date_of_birth
locale
ip_address
registration_ip_address
primary_group_id
flair_group_id
suspended_at
suspended_till
first_seen_at
last_seen_at
last_emailed_at
silenced_till
approved
approved_at
approved_by_id
views
created_at
updated_at
]
store_mapped_ids true
total_rows_query <<~SQL, MappingType::USERS
SELECT COUNT(*)
FROM users u
LEFT JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?
WHERE mu.original_id IS NULL
SQL
rows_query <<~SQL, MappingType::USERS
SELECT u.*,
us.suspended_at,
us.suspended_till,
JSON_GROUP_ARRAY(LOWER(ue.email)) AS emails
FROM users u
LEFT JOIN user_emails ue ON u.original_id = ue.user_id
LEFT JOIN user_suspensions us ON u.original_id = us.user_id AND us.suspended_at < DATETIME() AND
(us.suspended_till IS NULL OR us.suspended_till > DATETIME())
LEFT JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?
WHERE mu.original_id IS NULL
GROUP BY u.original_id
ORDER BY u.ROWID
SQL
def initialize(intermediate_db, discourse_db, shared_data)
super
@unique_name_finder = ::Migrations::Importer::UniqueNameFinder.new(@shared_data)
end
private
def transform_row(row)
if row[:emails].present?
JSON
.parse(row[:emails])
.each do |email|
if (existing_user_id = @user_ids_by_email[email])
row[:id] = existing_user_id
return nil
end
end
end
if row[:external_id].present? &&
(existing_user_id = @user_ids_by_external_id[row[:external_id]])
row[:id] = existing_user_id
return nil
end
row[:original_username] ||= row[:username]
row[:username] = @unique_name_finder.find_available_username(
row[:username],
allow_reserved_username: row[:admin] == 1,
)
row[:username_lower] = row[:username].downcase
row[:trust_level] ||= TrustLevel[1]
row[:active] = true if row[:active].nil?
row[:admin] = false if row[:admin].nil?
row[:moderator] = false if row[:moderator].nil?
row[:staged] = false if row[:staged].nil?
row[:last_emailed_at] ||= NOW
row[:suspended_till] ||= 200.years.from_now if row[:suspended_at].present?
date_of_birth = Migrations::Database.to_date(row[:date_of_birth])
if date_of_birth && date_of_birth.year != 1904
row[:date_of_birth] = Date.new(1904, date_of_birth.month, date_of_birth.day)
end
if SiteSetting.must_approve_users || !row[:approved].nil?
row[:approved] = true if row[:approved].nil?
row[:approved_at] = row[:approved] ? row[:approved_at] || NOW : nil
row[:approved_by_id] = row[:approved] ? row[:approved_by_id] || SYSTEM_USER_ID : nil
end
row[:views] ||= 0
super
end
def after_commit_of_inserted_rows(rows)
super
rows.each do |row|
if row[:id] && row[:username] && row[:username] != row[:original_username]
@intermediate_db.insert(
INSERT_MAPPED_USERNAMES_SQL,
[row[:id], row[:original_username], row[:username]],
)
end
end
end
end
end

307
migrations/scripts/generate_schema
View File

@ -1,307 +0,0 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
# Generate the converter's base intermediate database migration file from
# the core database state and YAML configuration in schema.yml
# Invoke from core root directory as `./migrations/scripts/generate_schema`
# It accepts an optional command line argument for the output file path which
# overrides the path configured in schema.yml
require_relative "../lib/migrations"
module Migrations
load_rails_environment
class SchemaGenerator
def initialize(opts = {})
config = load_config
@core_db_connection = ActiveRecord::Base.connection
@output_stream = StringIO.new
@indirectly_ignored_columns = Hash.new { |h, k| h[k] = [] }
@output_file_path = opts[:output_file_path] || config[:output_file_path]
@table_configs = config[:tables]
@column_configs = config[:columns]
@configured_table_names = @table_configs&.keys&.sort || []
@global_column_ignore_list = @column_configs&.fetch(:ignore) || []
end
def run
puts "Generating base converter migration file for Discourse #{Discourse::VERSION::STRING}"
generate_header
generate_tables
generate_indirectly_ignored_columns_log
generate_migration_file
validate_migration_file
puts "", "Done"
end
private
def load_config
path = File.expand_path("../config/intermediate_db.yml", __dir__)
YAML.load_file(path, symbolize_names: true)
end
def generate_header
return if @configured_table_names.empty?
@output_stream.puts <<~HEADER
/*
This file is auto-generated from the Discourse core database schema. Instead of editing it directly,
please update the `schema.yml` configuration file and re-run the `generate_schema` script to update it.
*/
HEADER
end
def generate_tables
puts "Generating tables..."
@configured_table_names.each do |name|
raise "Core table named '#{name}' not found" unless valid_table?(name)
generate_table(name)
end
end
def generate_indirectly_ignored_columns_log
return if @indirectly_ignored_columns.empty?
puts "Generating indirectly ignored column list..."
@output_stream.puts "\n\n/*"
@output_stream.puts <<~NOTE
Core table columns implicitly excluded from the generated schema above via the `include` configuration option
in `schema.yml`. This serves as an inventory of these columns, allowing new core additions to be tracked and,
if necessary, synchronized with the intermediate database schema.\n
NOTE
@indirectly_ignored_columns.each_with_index do |(table_name, columns), index|
next if virtual_table?(table_name) || columns.blank?
@output_stream.puts "" if index.positive?
@output_stream.puts "Table: #{table_name}"
@output_stream.puts "--------#{"-" * table_name.length}"
columns.each do |column|
@output_stream.puts " #{column.name} #{column.type} #{column.null}"
end
end
@output_stream.puts "*/"
end
def generate_migration_file
file_path = File.expand_path(@output_file_path, __dir__)
puts "Generating base migration file '#{file_path}'..."
File.open(file_path, "w") { |f| f << @output_stream.string.chomp }
end
def generate_column_definition(column)
definition = " #{column.name} #{type(column)}"
definition << " NOT NULL" unless column.null
definition
end
def generate_index(table_name, index)
@output_stream.print "CREATE "
@output_stream.print "UNIQUE " if index[:unique]
@output_stream.print "INDEX #{index[:name]} ON #{table_name} (#{index[:columns].join(", ")})"
@output_stream.print " #{index[:condition]}" if index[:condition].present?
@output_stream.puts ";"
end
def column_list_for(table_name)
ignore_columns = @table_configs.dig(table_name, :ignore) || []
include_columns = @table_configs.dig(table_name, :include) || []
include_columns.present? ? [:include, include_columns] : [:ignore, ignore_columns]
end
def generate_table(name)
puts "Generating #{name}..."
column_definitions = []
column_records = columns(name)
mode, column_list = column_list_for(name)
indexes = indexes(name)
configured_primary_key = primary_key(name)
primary_key, composite_key =
if configured_primary_key.present?
[configured_primary_key].flatten.each do |pk|
if column_records.map(&:name).exclude?(pk)
raise "Column named '#{pk}' does not exist in table '#{name}'"
end
end
[
configured_primary_key,
configured_primary_key.is_a?(Array) && configured_primary_key.length > 1,
]
else
virtual_table?(name) ? [] : [@core_db_connection.primary_key(name), false]
end
@output_stream.puts ""
@output_stream.puts "CREATE TABLE #{name}"
@output_stream.puts "("
if !composite_key && primary_key.present?
primary_key_column = column_records.find { |c| c.name == primary_key }
if (mode == :include && column_list.include?(primary_key_column.name)) ||
(mode == :ignore && column_list.exclude?(primary_key_column.name))
column_definitions << " #{primary_key_column.name} #{type(primary_key_column)} NOT NULL PRIMARY KEY"
end
end
column_records.each do |column|
next if @global_column_ignore_list.include?(column.name)
next if (mode == :ignore) && column_list.include?(column.name)
if !column.is_a?(CustomColumn) && (mode == :include) && column_list.exclude?(column.name)
@indirectly_ignored_columns[name] << column
next
end
next if !composite_key && (column.name == primary_key)
column_definitions << generate_column_definition(column)
end
format_columns!(column_definitions)
column_definitions << " PRIMARY KEY (#{primary_key.join(", ")})" if composite_key
@output_stream.puts column_definitions.join(",\n")
@output_stream.puts ");"
@output_stream.puts "" if indexes.present?
indexes.each { |index| generate_index(name, index) }
end
def validate_migration_file
db = Extralite::Database.new(":memory:")
if (sql = @output_stream.string).blank?
warn "No SQL generated, skipping validation".red
else
db.execute(sql)
end
ensure
db.close if db
end
def format_columns!(column_definitions)
column_definitions.map! do |c|
c.match(
/^\s*(?<name>\w+)\s(?<datatype>\w+)\s?(?<nullable>NOT NULL)?\s?(?<primary_key>PRIMARY KEY)?/,
).named_captures
end
max_name_length = column_definitions.map { |c| c["name"].length }.max
max_datatype_length = column_definitions.map { |c| c["datatype"].length }.max
column_definitions.sort_by! do |c|
[c["primary_key"] ? 0 : 1, c["nullable"] ? 0 : 1, c["name"]]
end
column_definitions.map! do |c|
" #{c["name"].ljust(max_name_length)} #{c["datatype"].ljust(max_datatype_length)} #{c["nullable"]} #{c["primary_key"]}".rstrip
end
end
class CustomColumn
attr_reader :name
def initialize(name, type, null)
@name = name
@raw_type = type
@raw_null = null
end
def type
@raw_type&.to_sym || :text
end
def null
@raw_null.nil? ? true : @raw_null
end
def merge!(other_column)
@raw_null = other_column.null if @raw_null.nil?
@raw_type ||= other_column.type
self
end
end
def columns(name)
extensions = column_extensions(name)
return extensions if virtual_table?(name)
default_columns = @core_db_connection.columns(name)
return default_columns if extensions.blank?
extended_columns =
default_columns.map do |default_column|
extension = extensions.find { |ext| ext.name == default_column.name }
if extension
extensions.delete(extension)
extension.merge!(default_column)
else
default_column
end
end
extended_columns + extensions
end
def column_extensions(name)
extensions = @table_configs.dig(name, :extend)
return [] if extensions.nil?
extensions.map { |column| CustomColumn.new(column[:name], column[:type], column[:is_null]) }
end
def type(column)
case column.type
when :string, :inet
"TEXT"
else
column.type.to_s.upcase
end
end
def valid_table?(name)
@core_db_connection.tables.include?(name.to_s) || virtual_table?(name)
end
def virtual_table?(name)
!!@table_configs.dig(name, :virtual)
end
def indexes(table_name)
@table_configs.dig(table_name, :indexes) || []
end
def primary_key(table_name)
@table_configs.dig(table_name, :primary_key)
end
end
end
::Migrations::SchemaGenerator.new(output_file_path: ARGV.first).run

481
migrations/scripts/schema.yml
View File

@ -1,481 +0,0 @@
## Configuration options for the base intermediate schema generator
##
## After modifying this file, regenerate the base intermediate schema
## by running the `generate_schema` script.
# Default relative path for generated base schema file.
# An absolute path can also be provided to the script as the first CLI argument.
# If the CLI argument is present, it takes precedence over the value specified here.
output_file_path: ../common/intermediate_db_schema/000_base_schema.sql
## Tables to include in the generated base intermediate schema.
##
## Available table options:
## virtual: Boolean. Enables the inclusion of a table in the schema solely based.
## on the provided configuration. A virtual table does not need to be available in the core schema.
## ignore: List of columns to ignore. Convenient if most of the table's column are needed.
## Usage is mutually exclusive with the `include` option. Only one should be used at a time.
## include: List of columns to include. Convenient if only a few columns are needed.
## Usage is mutually exclusive with the `include`` option. Only one should be used at a time.
## primary_key: Literal or list of columns to use as primary key.
## extend: List of objects describing columns to be added/extended.
## The following options are available for an "extend" object:
## name: Required. The name of the column being extended.
## is_null: Specifies if the column can be null.
## type: Column type. Defaults to TEXT.
## indexes: List of indexes to create. The following options are available for an "index" object:
## name: Index name.
## columns: List of column(s) to index.
tables:
schema_migrations:
virtual: true
primary_key: path
extend:
- name: path
is_null: false
- name: created_at
type: datetime
config:
virtual: true
primary_key: name
extend:
- name: name
is_null: false
- name: value
is_null: false
log_entries:
virtual: true
extend:
- name: created_at
type: datetime
is_null: false
- name: type
is_null: false
- name: message
is_null: false
- name: exception
- name: details
users:
ignore:
- seen_notification_id
- last_posted_at
- password_hash
- salt
- active
- last_emailed_at
- approved_by_id
- previous_visit_at
- suspended_at
- suspended_till
- views
- flag_level
- ip_address
- title
- uploaded_avatar_id
- locale
- primary_group_id
- first_seen_at
- silenced_till
- group_locked_trust_level
- manual_locked_trust_level
- secure_identifier
- flair_group_id
- last_seen_reviewable_id
- password_algorithm
- username_lower
extend:
- name: email
- name: created_at
is_null: true
- name: staged
is_null: true
- name: avatar_path
- name: avatar_url
- name: avatar_upload_id
- name: bio
- name: password
is_null: true
- name: trust_level
is_null: true
- name: suspension
- name: location
- name: website
- name: old_relative_url
- name: sso_record
- name: anonymized
type: boolean
- name: original_username
- name: timezone
- name: email_level
type: integer
- name: email_messages_level
type: integer
- name: email_digests
type: boolean
categories:
ignore:
- topic_id
- topic_count
- user_id
- topics_year
- topics_month
- topics_week
- auto_close_hours
- post_count
- latest_post_id
- latest_topic_id
- posts_year
- posts_month
- posts_week
- email_in
- email_in_allow_strangers
- topics_day
- posts_day
- allow_badges
- name_lower
- auto_close_based_on_last_post
- topic_template
- contains_messages
- sort_order
- sort_ascending
- uploaded_logo_id
- uploaded_background_id
- topic_featured_link_allowed
- all_topics_wiki
- show_subcategory_list
- num_featured_topics
- default_view
- subcategory_list_style
- default_top_period
- mailinglist_mirror
- minimum_required_tags
- navigate_to_first_post_after_read
- search_priority
- allow_global_tags
- reviewable_by_group_id
- read_only_banner
- default_list_filter
- allow_unlimited_owner_edits_on_first_post
- default_slow_mode_seconds
- uploaded_logo_dark_id
- uploaded_background_dark_id
extend:
- name: about_topic_title
- name: old_relative_url
- name: existing_id
type: integer
- name: permissions
type: json_text # JSON_TEXT ???
- name: logo_upload_id
- name: tag_group_ids
type: json_text # JSON_TEXT ???
topics:
ignore:
- last_posted_at
- posts_count
- last_post_user_id
- reply_count
- featured_user1_id
- featured_user2_id
- featured_user3_id
- featured_user4_id
- deleted_at
- highest_post_number
- like_count
- incoming_link_count
- moderator_posts_count
- bumped_at
- has_summary
- archetype
- notify_moderators_count
- spam_count
- score
- percent_rank
- slug
- deleted_by_id
- participant_count
- word_count
- excerpt
- fancy_title
- highest_staff_post_number
- featured_link
- reviewable_score
- image_upload_id
- slow_mode_seconds
- bannered_until
- external_id
extend:
- name: old_relative_url
- name: private_message
posts:
ignore:
- cooked
- reply_to_post_number
- reply_count
- quote_count
- deleted_at
- off_topic_count
- incoming_link_count
- bookmark_count
- score
- reads
- post_type
- sort_order
- last_editor_id
- hidden
- hidden_reason_id
- notify_moderators_count
- spam_count
- illegal_count
- inappropriate_count
- last_version_at
- user_deleted
- reply_to_user_id
- percent_rank
- notify_user_count
- like_score
- deleted_by_id
- edit_reason
- word_count
- version
- cook_method
- wiki
- baked_at
- baked_version
- hidden_at
- self_edits
- reply_quoted
- via_email
- raw_email
- public_version
- action_code
- locked_by_id
- image_upload_id
- outbound_message_id
- qa_vote_count # TODO: added from plugin, maybe skip these automatically for core schema?
extend:
- name: reply_to_post_id # NOTE: should this be text??
- name: original_raw
- name: upload_ids
type: json_text
- name: post_number
type: integer
- name: old_relative_url
- name: accepted_answer
type: boolean
- name: small_action
- name: whisper
type: boolean
- name: placeholders
type: json_text
indexes:
- name: posts_by_topic_post_number
columns: [topic_id, post_number]
uploads:
ignore:
- original_filename
- filesize
- width
- height
- url
- created_at
- sha1
- origin
- retain_hours
- extension
- thumbnail_width
- thumbnail_height
- etag
- secure
- access_control_post_id
- original_sha1
- animated
- verification_status
- security_last_changed_at
- security_last_changed_reason
- dominant_color
extend:
- name: filename
is_null: false
- name: relative_path
- name: type
- name: data
type: blob
groups:
include:
- id
- name
- full_name
- visibility_level
- members_visibility_level
- mentionable_level
- messageable_level
extend:
- name: description
group_members:
virtual: true
primary_key: [group_id, user_id]
extend:
- name: group_id
type: integer
- name: user_id
type: integer
- name: owner
type: boolean
likes:
virtual: true
primary_key: [user_id, post_id]
extend:
- name: post_id
type: integer
is_null: false
- name: user_id
type: integer
is_null: false
- name: created_at
type: datetime
is_null: false
# TODO: Pending default values & auto incrementing id column
user_fields:
ignore:
- created_at
- external_name
- external_type
extend:
- name: options
type: json_text
muted_users:
primary_key: [user_id, muted_user_id]
ignore:
- id
- created_at
# NOTE: Perhaps use core's user_field_options instead?
user_field_values:
virtual: true
extend:
- name: user_id
type: integer
is_null: false
- name: field_id
type: integer
is_null: false
- name: is_multiselect_field
type: boolean
is_null: false
- name: value
indexes:
- name: user_field_values_multiselect
columns: [user_id, field_id, value]
unique: true
condition: WHERE is_multiselect_field = TRUE
- name: user_field_values_not_multiselect
columns: [user_id, field_id]
unique: true
condition: WHERE is_multiselect_field = FALSE
tags:
include:
- id
- name
extend:
- name: tag_group_id
type: integer
tag_groups:
include:
- id
- name
topic_tags:
primary_key: [topic_id, tag_id]
ignore:
- id
- created_at
tag_users:
primary_key: [tag_id, user_id]
ignore:
- id
- created_at
badges:
ignore:
- grant_count
- allow_title
- icon
- listable
- target_posts
- enabled
- auto_revoke
- trigger
- show_posts
- system
- image
- badge_grouping_id
extend:
- name: bage_group
user_badges:
include:
- user_id
- badge_id
- granted_at
topic_users:
primary_key: [user_id, topic_id]
ignore:
- id
- posted
- cleared_pinned_at
- last_emailed_post_number
- liked
- bookmarked
- last_posted_at
permalink_normalizations:
virtual: true
primary_key: normalization
extend:
- name: normalization
is_null: false
site_settings:
include:
- name
- value
extend:
- name: action
category_custom_fields:
primary_key: [category_id, name]
ignore:
- id
- created_at
post_custom_fields:
primary_key: [post_id, name]
ignore:
- id
- created_at
polls: {}
poll_options:
ignore:
- digest
- html
- anonymous_votes
extend:
- name: poll_id
is_null: false
- name: text
is_null: false
- name: position
type: integer
- name: created_at
is_null: true
poll_votes:
primary_key: [poll_option_id, user_id]
ignore: [poll_id]
extend:
- name: created_at
is_null: true
- name: poll_option_id
is_null: false
- name: user_id
is_null: false
## Schema-wide column configuration options. These options apply to all tables.
## See table specific column configuration options above.
##
## Available Options:
## ignore: List of core/plugin table columns to ignore and exclude from intermediate schema.
columns:
ignore:
- updated_at

5
migrations/spec/lib/database/intermediate_db/upload_spec.rb Normal file
View File

@ -0,0 +1,5 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Database::IntermediateDB::Upload do
it_behaves_like "a database entity"
end

5
migrations/spec/lib/database/intermediate_db/user_suspension_spec.rb Normal file
View File

@ -0,0 +1,5 @@
# frozen_string_literal: true
RSpec.describe ::Migrations::Database::IntermediateDB::UserSuspension do
it_behaves_like "a database entity"
end