From 17ba19c7ae88c65e95fe6c381382f624c7e90329 Mon Sep 17 00:00:00 2001 From: Gerhard Schlager Date: Mon, 7 Apr 2025 17:12:18 +0200 Subject: [PATCH] REFACTOR: Code generator for migrations IntemerdiateDB * Splits the existing script into multiple classes * Adds command for generating IntermediateDB schema (`migrations/bin/cli schema generate`) * Changes the syntax of the IntermediateDB schema config * Adds validation for the schema config * It uses YAML schema aka JSON schema to validate the config file * It generates the SQL schema file and Ruby classes for storing data in the IntermediateDB --- .github/workflows/migration-tests.yml | 15 +- migrations/bin/cli | 3 + migrations/config/intermediate_db.yml | 335 ++++++++++-- migrations/config/json_schemas/db_schema.json | 258 ++++++++++ migrations/config/locales/migrations.en.yml | 32 ++ .../004-user_suspensions.sql | 9 + .../100-base-schema.sql | 112 +++- migrations/lib/cli/schema_sub_command.rb | 90 ++++ migrations/lib/common/date_helper.rb | 4 +- .../lib/database/intermediate_db/user.rb | 105 ++++ .../database/intermediate_db/user_email.rb | 31 ++ .../database/intermediate_db/user_option.rb | 199 ++++++++ .../intermediate_db/user_suspension.rb | 29 ++ migrations/lib/database/schema.rb | 175 +++++++ .../lib/database/schema/config_validator.rb | 46 ++ .../lib/database/schema/global_config.rb | 61 +++ migrations/lib/database/schema/loader.rb | 116 +++++ .../lib/database/schema/model_writer.rb | 139 +++++ .../lib/database/schema/table_writer.rb | 104 ++++ .../schema/validation/base_validator.rb | 18 + .../schema/validation/columns_validator.rb | 115 +++++ .../globally_configured_columns_validator.rb | 73 +++ .../globally_excluded_tables_validator.rb | 20 + .../validation/json_schema_validator.rb | 33 ++ .../validation/output_config_validator.rb | 46 ++ .../validation/plugin_config_validator.rb | 28 + .../validation/schema_config_validator.rb | 18 + .../schema/validation/tables_validator.rb | 70 +++ migrations/lib/importer/steps/users.rb | 141 +++++ migrations/scripts/generate_schema | 307 ----------- migrations/scripts/schema.yml | 481 ------------------ .../database/intermediate_db/upload_spec.rb | 5 + .../intermediate_db/user_suspension_spec.rb | 5 + 33 files changed, 2359 insertions(+), 864 deletions(-) create mode 100644 migrations/config/json_schemas/db_schema.json create mode 100644 migrations/db/intermediate_db_schema/004-user_suspensions.sql create mode 100644 migrations/lib/cli/schema_sub_command.rb create mode 100644 migrations/lib/database/intermediate_db/user.rb create mode 100644 migrations/lib/database/intermediate_db/user_email.rb create mode 100644 migrations/lib/database/intermediate_db/user_option.rb create mode 100644 migrations/lib/database/intermediate_db/user_suspension.rb create mode 100644 migrations/lib/database/schema.rb create mode 100644 migrations/lib/database/schema/config_validator.rb create mode 100644 migrations/lib/database/schema/global_config.rb create mode 100644 migrations/lib/database/schema/loader.rb create mode 100644 migrations/lib/database/schema/model_writer.rb create mode 100644 migrations/lib/database/schema/table_writer.rb create mode 100644 migrations/lib/database/schema/validation/base_validator.rb create mode 100644 migrations/lib/database/schema/validation/columns_validator.rb create mode 100644 migrations/lib/database/schema/validation/globally_configured_columns_validator.rb create mode 100644 migrations/lib/database/schema/validation/globally_excluded_tables_validator.rb create mode 100644 migrations/lib/database/schema/validation/json_schema_validator.rb create mode 100644 migrations/lib/database/schema/validation/output_config_validator.rb create mode 100644 migrations/lib/database/schema/validation/plugin_config_validator.rb create mode 100644 migrations/lib/database/schema/validation/schema_config_validator.rb create mode 100644 migrations/lib/database/schema/validation/tables_validator.rb create mode 100644 migrations/lib/importer/steps/users.rb delete mode 100755 migrations/scripts/generate_schema delete mode 100644 migrations/scripts/schema.yml create mode 100644 migrations/spec/lib/database/intermediate_db/upload_spec.rb create mode 100644 migrations/spec/lib/database/intermediate_db/user_suspension_spec.rb diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.yml index 7bcd4443133..30271cdb5ec 100644 --- a/.github/workflows/migration-tests.yml +++ b/.github/workflows/migration-tests.yml @@ -111,11 +111,16 @@ jobs: if: steps.app-cache.outputs.cache-hit != 'true' run: rm -rf tmp/app-cache/uploads && cp -r public/uploads tmp/app-cache/uploads - # - name: Check core database drift - # run: | - # mkdir /tmp/intermediate_db - # ./migrations/scripts/schema_generator /tmp/intermediate_db/base_migration.sql - # diff -u migrations/common/intermediate_db_schema/000_base_schema.sql /tmp/intermediate_db/base_migration.sql + - name: Validate IntermediateDB schema + run: | + migrations/bin/cli schema generate --db=intermediate_db + + if [ ! -z "$(git status --porcelain migrations/db/intermediate_db_schema/)" ]; then + echo "IntermediateDB schema is not up to date." + echo "---------------------------------------------" + git -c color.ui=always diff migrations/db/intermediate_db_schema/ + exit 1 + fi - name: RSpec run: bin/rspec --default-path migrations/spec diff --git a/migrations/bin/cli b/migrations/bin/cli index e0894181f94..5f664360334 100755 --- a/migrations/bin/cli +++ b/migrations/bin/cli @@ -37,6 +37,9 @@ module Migrations ::Migrations::CLI::UploadCommand.new(options).execute end + desc "schema [COMMAND]", "Manage database schema" + subcommand "schema", ::Migrations::CLI::SchemaSubCommand + def self.exit_on_failure? true end diff --git a/migrations/config/intermediate_db.yml b/migrations/config/intermediate_db.yml index 435b666cfb2..556cf848ac6 100644 --- a/migrations/config/intermediate_db.yml +++ b/migrations/config/intermediate_db.yml @@ -1,50 +1,291 @@ -## Configuration options for the base intermediate schema generator -## -## After modifying this file, regenerate the base intermediate schema -## by running the `generate_schema` script. +# $schema: ./json_schemas/db_schema.json -# Default relative path for generated base schema file. -# An absolute path can also be provided to the script as the first CLI argument. -# If the CLI argument is present, it takes precedence over the value specified here. -output_file_path: "../db/schema/100-base-schema.sql" +output: + schema_file: "db/intermediate_db_schema/100-base-schema.sql" + models_directory: "lib/database/intermediate_db" + models_namespace: Migrations::Database::IntermediateDB -## Tables to include in the generated base intermediate schema. -## -## Available table options: -## virtual: Boolean. Enables the inclusion of a table in the schema solely based. -## on the provided configuration. A virtual table does not need to be available in the core schema. -## ignore: List of columns to ignore. Convenient if most of the table's column are needed. -## Usage is mutually exclusive with the `include` option. Only one should be used at a time. -## include: List of columns to include. Convenient if only a few columns are needed. -## Usage is mutually exclusive with the `include`` option. Only one should be used at a time. -## primary_key: Literal or list of columns to use as primary key. -## extend: List of objects describing columns to be added/extended. -## The following options are available for an "extend" object: -## name: Required. The name of the column being extended. -## is_null: Specifies if the column can be null. -## type: Column type. Defaults to TEXT. -## indexes: List of indexes to create. The following options are available for an "index" object: -## name: Index name. -## columns: List of column(s) to index. -tables: - users: - ignore: - - flag_level - - last_emailed_at - - last_posted_at - - last_seen_reviewable_id - - password_algorithm - - password_hash - - salt - - secure_identifier - - seen_notification_id - - username_lower +schema: + tables: + user_emails: + columns: + include: + - "email" + - "primary" + - "user_id" + - "created_at" + primary_key_column_names: [ "email" ] + user_options: + primary_key_column_names: [ "user_id" ] + users: + columns: + add: + - name: "original_username" + datatype: text + exclude: + - "flag_level" + - "last_emailed_at" + - "last_posted_at" + - "last_seen_reviewable_id" + - "previous_visit_at" + - "required_fields_version" + - "secure_identifier" + - "seen_notification_id" + - "suspended_at" + - "suspended_till" + - "username_lower" -## Schema-wide column configuration options. These options apply to all tables. -## See table specific column configuration options above. -## -## Available Options: -## ignore: List of core/plugin table columns to ignore and exclude from intermediate schema. -columns: - ignore: - - updated_at + global: + columns: + modify: + - name: "id" + datatype: numeric + rename_to: "original_id" + - name_regex: ".*upload.*_id$" + datatype: text + - name_regex: ".*_id$" + datatype: numeric + exclude: + - "updated_at" + + tables: + exclude: + - "admin_notices" + - "allowed_pm_users" + - "anonymous_users" + - "api_key_scopes" + - "api_keys" + - "application_requests" + - "ar_internal_metadata" + - "associated_groups" + - "backup_draft_posts" + - "backup_draft_topics" + - "backup_metadata" + - "badge_groupings" + - "badge_types" + - "badges" + - "bookmarks" + - "categories" + - "categories_web_hooks" + - "category_custom_fields" + - "category_featured_topics" + - "category_form_templates" + - "category_groups" + - "category_moderation_groups" + - "category_required_tag_groups" + - "category_search_data" + - "category_settings" + - "category_tag_groups" + - "category_tag_stats" + - "category_tags" + - "category_users" + - "chat_channel_archives" + - "chat_channel_custom_fields" + - "chat_channels" + - "chat_drafts" + - "chat_mention_notifications" + - "chat_mentions" + - "chat_message_custom_fields" + - "chat_message_interactions" + - "chat_message_reactions" + - "chat_message_revisions" + - "chat_messages" + - "chat_thread_custom_fields" + - "chat_threads" + - "chat_webhook_events" + - "child_themes" + - "color_scheme_colors" + - "color_schemes" + - "custom_emojis" + - "developers" + - "direct_message_channels" + - "direct_message_users" + - "directory_columns" + - "directory_items" + - "discourse_automation_automations" + - "discourse_automation_fields" + - "discourse_automation_pending_automations" + - "discourse_automation_pending_pms" + - "discourse_automation_stats" + - "discourse_automation_user_global_notices" + - "dismissed_topic_users" + - "do_not_disturb_timings" + - "draft_sequences" + - "drafts" + - "email_change_requests" + - "email_logs" + - "email_tokens" + - "embeddable_host_tags" + - "embeddable_hosts" + - "external_upload_stubs" + - "flags" + - "form_templates" + - "given_daily_likes" + - "group_archived_messages" + - "group_associated_groups" + - "group_category_notification_defaults" + - "group_custom_fields" + - "group_histories" + - "group_mentions" + - "group_requests" + - "group_tag_notification_defaults" + - "group_users" + - "groups" + - "groups_web_hooks" + - "ignored_users" + - "imap_sync_logs" + - "incoming_chat_webhooks" + - "incoming_domains" + - "incoming_emails" + - "incoming_links" + - "incoming_referers" + - "invited_groups" + - "invited_users" + - "invites" + - "javascript_caches" + - "linked_topics" + - "message_bus" + - "moved_posts" + - "muted_users" + - "notifications" + - "oauth2_user_infos" + - "onceoff_logs" + - "optimized_images" + - "permalinks" + - "plugin_store_rows" + - "poll_options" + - "poll_votes" + - "polls" + - "post_action_types" + - "post_actions" + - "post_custom_fields" + - "post_details" + - "post_hotlinked_media" + - "post_replies" + - "post_reply_keys" + - "post_revisions" + - "post_search_data" + - "post_stats" + - "post_timings" + - "posts" + - "problem_check_trackers" + - "published_pages" + - "push_subscriptions" + - "quoted_posts" + - "redelivering_webhook_events" + - "remote_themes" + - "reviewable_claimed_topics" + - "reviewable_histories" + - "reviewable_scores" + - "reviewables" + - "scheduler_stats" + - "schema_migration_details" + - "schema_migrations" + - "screened_emails" + - "screened_ip_addresses" + - "screened_urls" + - "search_logs" + - "shared_drafts" + - "shelved_notifications" + - "sidebar_section_links" + - "sidebar_sections" + - "sidebar_urls" + - "single_sign_on_records" + - "site_settings" + - "sitemaps" + - "skipped_email_logs" + - "stylesheet_cache" + - "summary_sections" + - "tag_group_memberships" + - "tag_group_permissions" + - "tag_groups" + - "tag_search_data" + - "tag_users" + - "tags" + - "tags_web_hooks" + - "theme_color_schemes" + - "theme_fields" + - "theme_modifier_sets" + - "theme_settings" + - "theme_settings_migrations" + - "theme_svg_sprites" + - "theme_translation_overrides" + - "themes" + - "top_topics" + - "topic_allowed_groups" + - "topic_allowed_users" + - "topic_custom_fields" + - "topic_embeds" + - "topic_groups" + - "topic_hot_scores" + - "topic_invites" + - "topic_link_clicks" + - "topic_links" + - "topic_search_data" + - "topic_tags" + - "topic_thumbnails" + - "topic_timers" + - "topic_users" + - "topic_view_stats" + - "topic_views" + - "topics" + - "translation_overrides" + - "unsubscribe_keys" + - "upload_references" + - "uploads" + - "user_actions" + - "user_api_key_client_scopes" + - "user_api_key_clients" + - "user_api_key_scopes" + - "user_api_keys" + - "user_archived_messages" + - "user_associated_accounts" + - "user_associated_groups" + - "user_auth_token_logs" + - "user_auth_tokens" + - "user_avatars" + - "user_badges" + - "user_chat_channel_memberships" + - "user_chat_thread_memberships" + - "user_custom_fields" + - "user_exports" + - "user_field_options" + - "user_fields" + - "user_histories" + - "user_ip_address_histories" + - "user_notification_schedules" + - "user_open_ids" + - "user_passwords" + - "user_profile_views" + - "user_profiles" + - "user_required_fields_versions" + - "user_search_data" + - "user_second_factors" + - "user_security_keys" + - "user_stats" + - "user_statuses" + - "user_uploads" + - "user_visits" + - "user_warnings" + - "watched_word_groups" + - "watched_words" + - "web_crawler_requests" + - "web_hook_event_types" + - "web_hook_event_types_hooks" + - "web_hook_events" + - "web_hook_events_daily_aggregates" + - "web_hooks" + +plugins: + - "automation" + - "chat" + - "checklist" + - "discourse-details" + - "discourse-lazy-videos" + - "discourse-local-dates" + - "discourse-narrative-bot" + - "discourse-presence" + - "footnote" + - "poll" + - "spoiler-alert" + - "styleguide" diff --git a/migrations/config/json_schemas/db_schema.json b/migrations/config/json_schemas/db_schema.json new file mode 100644 index 00000000000..7fecc9b887b --- /dev/null +++ b/migrations/config/json_schemas/db_schema.json @@ -0,0 +1,258 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "datatypes": { + "type": "string", + "enum": [ + "blob", + "boolean", + "date", + "datetime", + "float", + "inet", + "integer", + "json", + "numeric", + "text" + ] + } + }, + "type": "object", + "required": [ + "output", + "schema", + "plugins" + ], + "properties": { + "output": { + "type": "object", + "additionalProperties": false, + "properties": { + "schema_file": { + "type": "string" + }, + "models_directory": { + "type": "string" + }, + "models_namespace": { + "type": "string" + } + }, + "required": [ + "schema_file", + "models_directory", + "models_namespace" + ] + }, + "schema": { + "type": "object", + "required": [ + "tables", + "global" + ], + "properties": { + "tables": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "columns": { + "type": "object", + "properties": { + "exclude": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "include": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "modify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "datatype": { + "$ref": "#/$defs/datatypes" + } + }, + "additionalProperties": false, + "required": [ + "name", + "datatype" + ] + } + }, + "add": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "datatype": { + "$ref": "#/$defs/datatypes" + }, + "nullable": { + "type": "boolean" + }, + "max_length": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "name", + "datatype" + ] + } + } + }, + "additionalProperties": false, + "not": { + "required": [ + "exclude", + "include" + ] + } + }, + "indexes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "columns": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "unique": { + "type": "boolean" + }, + "condition": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name", + "columns" + ] + } + }, + "primary_key_column_names": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "copy_of": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "global": { + "type": "object", + "properties": { + "columns": { + "properties": { + "modify": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "name_regex": { + "type": "string", + "format": "regex" + }, + "datatype": { + "$ref": "#/$defs/datatypes" + }, + "rename_to": { + "type": "string" + } + }, + "additionalProperties": false, + "anyOf": [ + { + "required": [ + "datatype" + ] + }, + { + "required": [ + "rename_to" + ] + } + ], + "oneOf": [ + { + "required": [ + "name" + ] + }, + { + "required": [ + "name_regex" + ] + } + ] + } + }, + "exclude": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + } + } + }, + "tables": { + "properties": { + "exclude": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + } + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "plugins": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + } + }, + "additionalProperties": false +} diff --git a/migrations/config/locales/migrations.en.yml b/migrations/config/locales/migrations.en.yml index c2c9168ba10..fa194f82c2b 100644 --- a/migrations/config/locales/migrations.en.yml +++ b/migrations/config/locales/migrations.en.yml @@ -19,3 +19,35 @@ en: default_step_title: "Converting %{type}" max_progress_calculation: "Calculating items took %{duration}" + schema: + validator: + include_exclude_not_allowed: "Cannot use `include` and `exclude` together at %{path}" + invalid_name_regex: "Invalid `name_regex`: %{message}" + + output: + schema_file_directory_not_found: "Schema file directory not found" + models_directory_not_found: "Models directory not found" + models_namespace_undefined: "Models namespace not defined" + + global: + excluded_tables_missing: "Missing globally excluded tables: %{table_names}" + excluded_columns_missing: "Missing globally excluded columns: %{column_names}" + modified_columns_missing: "Missing globally modified columns: %{column_names}" + + tables: + excluded_tables_configured: "Excluded tables configured: %{table_names}" + not_configured: "Tables not configured or excluded: %{table_names}" + copy_table_not_found: "Can't copy table '%{table_name}' because it doesn't exist" + + added_columns_exist: "Added columns already exist in '%{table_name}': %{column_names}" + included_columns_missing: "Included columns do not exist in '%{table_name}': %{column_names}" + excluded_columns_missing: "Excluded columns do not exist in '%{table_name}': %{column_names}" + modified_columns_missing: "Missing modified columns in '%{table_name}': %{column_names}" + modified_columns_included: "Modified columns included in '%{table_name}': %{column_names}" + modified_columns_excluded: "Modified columns excluded in '%{table_name}': %{column_names}" + modified_columns_globally_excluded: "Modified columns globally excluded in '%{table_name}': %{column_names}" + no_columns_configured: "No columns configured in '%{table_name}'" + + plugins: + not_installed: "Plugins not installed: %{plugin_names}" + additional_installed: "Unconfigured plugins installed: %{plugin_names}" diff --git a/migrations/db/intermediate_db_schema/004-user_suspensions.sql b/migrations/db/intermediate_db_schema/004-user_suspensions.sql new file mode 100644 index 00000000000..70bf4beb8ff --- /dev/null +++ b/migrations/db/intermediate_db_schema/004-user_suspensions.sql @@ -0,0 +1,9 @@ +CREATE TABLE user_suspensions +( + user_id NUMERIC NOT NULL, + suspended_at DATETIME NOT NULL, + suspended_till DATETIME, + suspended_by_id NUMERIC, + reason TEXT, + PRIMARY KEY (user_id, suspended_at) +); diff --git a/migrations/db/intermediate_db_schema/100-base-schema.sql b/migrations/db/intermediate_db_schema/100-base-schema.sql index bd86d41cb1c..01af108251a 100644 --- a/migrations/db/intermediate_db_schema/100-base-schema.sql +++ b/migrations/db/intermediate_db_schema/100-base-schema.sql @@ -1,37 +1,105 @@ -/* - This file is auto-generated from the Discourse core database schema. Instead of editing it directly, - please update the `schema.yml` configuration file and re-run the `generate_schema` script to update it. -*/ +-- This file is auto-generated from the IntermediateDB schema. To make changes, +-- update the "config/intermediate_db.yml" configuration file and then run +-- `bin/cli schema generate` to regenerate this file. + +CREATE TABLE user_emails +( + email TEXT NOT NULL PRIMARY KEY, + created_at DATETIME NOT NULL, + "primary" BOOLEAN, + user_id NUMERIC NOT NULL +); + +CREATE TABLE user_options +( + user_id NUMERIC NOT NULL PRIMARY KEY, + allow_private_messages BOOLEAN, + auto_track_topics_after_msecs INTEGER, + automatically_unpin_topics BOOLEAN, + bookmark_auto_delete_preference INTEGER, + chat_email_frequency INTEGER, + chat_enabled BOOLEAN, + chat_header_indicator_preference INTEGER, + chat_quick_reaction_type INTEGER, + chat_quick_reactions_custom TEXT, + chat_send_shortcut INTEGER, + chat_separate_sidebar_mode INTEGER, + chat_sound TEXT, + color_scheme_id NUMERIC, + dark_scheme_id NUMERIC, + default_calendar INTEGER, + digest_after_minutes INTEGER, + dismissed_channel_retention_reminder BOOLEAN, + dismissed_dm_retention_reminder BOOLEAN, + dynamic_favicon BOOLEAN, + email_digests BOOLEAN, + email_in_reply_to BOOLEAN, + email_level INTEGER, + email_messages_level INTEGER, + email_previous_replies INTEGER, + enable_allowed_pm_users BOOLEAN, + enable_defer BOOLEAN, + enable_experimental_sidebar BOOLEAN, + enable_quoting BOOLEAN, + enable_smart_lists BOOLEAN, + external_links_in_new_tab BOOLEAN, + hide_presence BOOLEAN, + hide_profile BOOLEAN, + hide_profile_and_presence BOOLEAN, + homepage_id NUMERIC, + ignore_channel_wide_mention BOOLEAN, + include_tl0_in_digests BOOLEAN, + last_redirected_to_top_at DATETIME, + like_notification_frequency INTEGER, + mailing_list_mode BOOLEAN, + mailing_list_mode_frequency INTEGER, + new_topic_duration_minutes INTEGER, + notification_level_when_replying INTEGER, + oldest_search_log_date DATETIME, + only_chat_push_notifications BOOLEAN, + seen_popups INTEGER, + show_thread_title_prompts BOOLEAN, + sidebar_link_to_filtered_list BOOLEAN, + sidebar_show_count_of_new_items BOOLEAN, + skip_new_user_tips BOOLEAN, + text_size_key INTEGER, + text_size_seq INTEGER, + theme_ids INTEGER, + theme_key_seq INTEGER, + timezone TEXT, + title_count_mode_key INTEGER, + topics_unread_when_closed BOOLEAN, + watched_precedence_over_muted BOOLEAN +); CREATE TABLE users ( - id INTEGER NOT NULL PRIMARY KEY, - active BOOLEAN NOT NULL, - admin BOOLEAN NOT NULL, - approved BOOLEAN NOT NULL, - created_at DATETIME NOT NULL, - staged BOOLEAN NOT NULL, - trust_level INTEGER NOT NULL, - username TEXT NOT NULL, - views INTEGER NOT NULL, + original_id NUMERIC NOT NULL PRIMARY KEY, + active BOOLEAN, + admin BOOLEAN, + approved BOOLEAN, approved_at DATETIME, - approved_by_id INTEGER, + approved_by_id NUMERIC, + created_at DATETIME NOT NULL, date_of_birth DATE, first_seen_at DATETIME, - flair_group_id INTEGER, + flair_group_id NUMERIC, group_locked_trust_level INTEGER, - ip_address TEXT, + ip_address INET_TEXT, last_seen_at DATETIME, locale TEXT, manual_locked_trust_level INTEGER, moderator BOOLEAN, name TEXT, - previous_visit_at DATETIME, - primary_group_id INTEGER, - registration_ip_address TEXT, + original_username TEXT, + primary_group_id NUMERIC, + registration_ip_address INET_TEXT, silenced_till DATETIME, - suspended_at DATETIME, - suspended_till DATETIME, + staged BOOLEAN, title TEXT, - uploaded_avatar_id TEXT + trust_level INTEGER NOT NULL, + uploaded_avatar_id TEXT, + username TEXT NOT NULL, + views INTEGER ); + diff --git a/migrations/lib/cli/schema_sub_command.rb b/migrations/lib/cli/schema_sub_command.rb new file mode 100644 index 00000000000..78d422db7d8 --- /dev/null +++ b/migrations/lib/cli/schema_sub_command.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Migrations::CLI + class SchemaSubCommand < Thor + Schema = ::Migrations::Database::Schema + + desc "generate", "Generates the database schema" + method_option :db, type: :string, default: "intermediate_db", desc: "Name of the database" + def generate + db = options[:db] + config = load_config_file(db) + + puts "Generating schema for #{db.bold}..." + ::Migrations.load_rails_environment(quiet: true) + + validate_config(config) + + loader = Schema::Loader.new(config[:schema]) + schema = loader.load_schema + header = file_header(db) + + write_db_schema(config, header, schema) + write_db_models(config, header, schema) + + puts "Done" + end + + private + + def validate_config(config) + validator = Schema::ConfigValidator.new + validator.validate(config) + + if validator.has_errors? + validator.errors.each { |error| print_error(error) } + exit(2) + end + end + + def write_db_schema(config, header, schema) + schema_file_path = File.expand_path(config.dig(:output, :schema_file), ::Migrations.root_path) + + File.open(schema_file_path, "w") do |schema_file| + writer = Schema::TableWriter.new(schema_file) + writer.output_file_header(header) + + schema.each { |table| writer.output_table(table) } + end + end + + def write_db_models(config, header, schema) + writer = Schema::ModelWriter.new(config.dig(:output, :models_namespace), header) + models_path = File.expand_path(config.dig(:output, :models_directory), ::Migrations.root_path) + + schema.each do |table| + model_file_path = File.join(models_path, Schema::ModelWriter.filename_for(table)) + File.open(model_file_path, "w") { |model_file| writer.output_table(table, model_file) } + end + + Schema::ModelWriter.format_files(models_path) + end + + def relative_config_path(db) + File.join("config", "#{db}.yml") + end + + def file_header(db) + <<~HEADER + This file is auto-generated from the IntermediateDB schema. To make changes, + update the "#{relative_config_path(db)}" configuration file and then run + `bin/cli schema generate` to regenerate this file. + HEADER + end + + def load_config_file(db) + config_path = File.join(::Migrations.root_path, relative_config_path(db)) + + if !File.exist?(config_path) + print_error("Configuration file for #{db} wasn't found at '#{config_path}'") + exit 1 + end + + YAML.load_file(config_path, symbolize_names: true) + end + + def print_error(message) + $stderr.puts "ERROR: ".red + message + end + end +end diff --git a/migrations/lib/common/date_helper.rb b/migrations/lib/common/date_helper.rb index 8858c35f2e2..e8a445de82b 100644 --- a/migrations/lib/common/date_helper.rb +++ b/migrations/lib/common/date_helper.rb @@ -9,9 +9,9 @@ module Migrations end def self.track_time - start_time = Time.now + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) yield - Time.now - start_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time end end end diff --git a/migrations/lib/database/intermediate_db/user.rb b/migrations/lib/database/intermediate_db/user.rb new file mode 100644 index 00000000000..01a1c079d32 --- /dev/null +++ b/migrations/lib/database/intermediate_db/user.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +# This file is auto-generated from the IntermediateDB schema. To make changes, +# update the "config/intermediate_db.yml" configuration file and then run +# `bin/cli schema generate` to regenerate this file. + +module Migrations::Database::IntermediateDB + module User + SQL = <<~SQL + INSERT INTO users ( + original_id, + active, + admin, + approved, + approved_at, + approved_by_id, + created_at, + date_of_birth, + first_seen_at, + flair_group_id, + group_locked_trust_level, + ip_address, + last_seen_at, + locale, + manual_locked_trust_level, + moderator, + name, + original_username, + primary_group_id, + registration_ip_address, + silenced_till, + staged, + title, + trust_level, + uploaded_avatar_id, + username, + views + ) + VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + SQL + + def self.create( + original_id:, + active: nil, + admin: nil, + approved: nil, + approved_at: nil, + approved_by_id: nil, + created_at:, + date_of_birth: nil, + first_seen_at: nil, + flair_group_id: nil, + group_locked_trust_level: nil, + ip_address: nil, + last_seen_at: nil, + locale: nil, + manual_locked_trust_level: nil, + moderator: nil, + name: nil, + original_username: nil, + primary_group_id: nil, + registration_ip_address: nil, + silenced_till: nil, + staged: nil, + title: nil, + trust_level:, + uploaded_avatar_id: nil, + username:, + views: nil + ) + ::Migrations::Database::IntermediateDB.insert( + SQL, + original_id, + ::Migrations::Database.format_boolean(active), + ::Migrations::Database.format_boolean(admin), + ::Migrations::Database.format_boolean(approved), + ::Migrations::Database.format_datetime(approved_at), + approved_by_id, + ::Migrations::Database.format_datetime(created_at), + ::Migrations::Database.format_date(date_of_birth), + ::Migrations::Database.format_datetime(first_seen_at), + flair_group_id, + group_locked_trust_level, + ::Migrations::Database.format_ip_address(ip_address), + ::Migrations::Database.format_datetime(last_seen_at), + locale, + manual_locked_trust_level, + ::Migrations::Database.format_boolean(moderator), + name, + original_username, + primary_group_id, + ::Migrations::Database.format_ip_address(registration_ip_address), + ::Migrations::Database.format_datetime(silenced_till), + ::Migrations::Database.format_boolean(staged), + title, + trust_level, + uploaded_avatar_id, + username, + views, + ) + end + end +end diff --git a/migrations/lib/database/intermediate_db/user_email.rb b/migrations/lib/database/intermediate_db/user_email.rb new file mode 100644 index 00000000000..e8ec92938d8 --- /dev/null +++ b/migrations/lib/database/intermediate_db/user_email.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +# This file is auto-generated from the IntermediateDB schema. To make changes, +# update the "config/intermediate_db.yml" configuration file and then run +# `bin/cli schema generate` to regenerate this file. + +module Migrations::Database::IntermediateDB + module UserEmail + SQL = <<~SQL + INSERT INTO user_emails ( + email, + created_at, + "primary", + user_id + ) + VALUES ( + ?, ?, ?, ? + ) + SQL + + def self.create(email:, created_at:, primary: nil, user_id:) + ::Migrations::Database::IntermediateDB.insert( + SQL, + email, + ::Migrations::Database.format_datetime(created_at), + ::Migrations::Database.format_boolean(primary), + user_id, + ) + end + end +end diff --git a/migrations/lib/database/intermediate_db/user_option.rb b/migrations/lib/database/intermediate_db/user_option.rb new file mode 100644 index 00000000000..6ef7f0de428 --- /dev/null +++ b/migrations/lib/database/intermediate_db/user_option.rb @@ -0,0 +1,199 @@ +# frozen_string_literal: true + +# This file is auto-generated from the IntermediateDB schema. To make changes, +# update the "config/intermediate_db.yml" configuration file and then run +# `bin/cli schema generate` to regenerate this file. + +module Migrations::Database::IntermediateDB + module UserOption + SQL = <<~SQL + INSERT INTO user_options ( + user_id, + allow_private_messages, + auto_track_topics_after_msecs, + automatically_unpin_topics, + bookmark_auto_delete_preference, + chat_email_frequency, + chat_enabled, + chat_header_indicator_preference, + chat_quick_reaction_type, + chat_quick_reactions_custom, + chat_send_shortcut, + chat_separate_sidebar_mode, + chat_sound, + color_scheme_id, + dark_scheme_id, + default_calendar, + digest_after_minutes, + dismissed_channel_retention_reminder, + dismissed_dm_retention_reminder, + dynamic_favicon, + email_digests, + email_in_reply_to, + email_level, + email_messages_level, + email_previous_replies, + enable_allowed_pm_users, + enable_defer, + enable_experimental_sidebar, + enable_quoting, + enable_smart_lists, + external_links_in_new_tab, + hide_presence, + hide_profile, + hide_profile_and_presence, + homepage_id, + ignore_channel_wide_mention, + include_tl0_in_digests, + last_redirected_to_top_at, + like_notification_frequency, + mailing_list_mode, + mailing_list_mode_frequency, + new_topic_duration_minutes, + notification_level_when_replying, + oldest_search_log_date, + only_chat_push_notifications, + seen_popups, + show_thread_title_prompts, + sidebar_link_to_filtered_list, + sidebar_show_count_of_new_items, + skip_new_user_tips, + text_size_key, + text_size_seq, + theme_ids, + theme_key_seq, + timezone, + title_count_mode_key, + topics_unread_when_closed, + watched_precedence_over_muted + ) + VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + SQL + + def self.create( + user_id:, + allow_private_messages: nil, + auto_track_topics_after_msecs: nil, + automatically_unpin_topics: nil, + bookmark_auto_delete_preference: nil, + chat_email_frequency: nil, + chat_enabled: nil, + chat_header_indicator_preference: nil, + chat_quick_reaction_type: nil, + chat_quick_reactions_custom: nil, + chat_send_shortcut: nil, + chat_separate_sidebar_mode: nil, + chat_sound: nil, + color_scheme_id: nil, + dark_scheme_id: nil, + default_calendar: nil, + digest_after_minutes: nil, + dismissed_channel_retention_reminder: nil, + dismissed_dm_retention_reminder: nil, + dynamic_favicon: nil, + email_digests: nil, + email_in_reply_to: nil, + email_level: nil, + email_messages_level: nil, + email_previous_replies: nil, + enable_allowed_pm_users: nil, + enable_defer: nil, + enable_experimental_sidebar: nil, + enable_quoting: nil, + enable_smart_lists: nil, + external_links_in_new_tab: nil, + hide_presence: nil, + hide_profile: nil, + hide_profile_and_presence: nil, + homepage_id: nil, + ignore_channel_wide_mention: nil, + include_tl0_in_digests: nil, + last_redirected_to_top_at: nil, + like_notification_frequency: nil, + mailing_list_mode: nil, + mailing_list_mode_frequency: nil, + new_topic_duration_minutes: nil, + notification_level_when_replying: nil, + oldest_search_log_date: nil, + only_chat_push_notifications: nil, + seen_popups: nil, + show_thread_title_prompts: nil, + sidebar_link_to_filtered_list: nil, + sidebar_show_count_of_new_items: nil, + skip_new_user_tips: nil, + text_size_key: nil, + text_size_seq: nil, + theme_ids: nil, + theme_key_seq: nil, + timezone: nil, + title_count_mode_key: nil, + topics_unread_when_closed: nil, + watched_precedence_over_muted: nil + ) + ::Migrations::Database::IntermediateDB.insert( + SQL, + user_id, + ::Migrations::Database.format_boolean(allow_private_messages), + auto_track_topics_after_msecs, + ::Migrations::Database.format_boolean(automatically_unpin_topics), + bookmark_auto_delete_preference, + chat_email_frequency, + ::Migrations::Database.format_boolean(chat_enabled), + chat_header_indicator_preference, + chat_quick_reaction_type, + chat_quick_reactions_custom, + chat_send_shortcut, + chat_separate_sidebar_mode, + chat_sound, + color_scheme_id, + dark_scheme_id, + default_calendar, + digest_after_minutes, + ::Migrations::Database.format_boolean(dismissed_channel_retention_reminder), + ::Migrations::Database.format_boolean(dismissed_dm_retention_reminder), + ::Migrations::Database.format_boolean(dynamic_favicon), + ::Migrations::Database.format_boolean(email_digests), + ::Migrations::Database.format_boolean(email_in_reply_to), + email_level, + email_messages_level, + email_previous_replies, + ::Migrations::Database.format_boolean(enable_allowed_pm_users), + ::Migrations::Database.format_boolean(enable_defer), + ::Migrations::Database.format_boolean(enable_experimental_sidebar), + ::Migrations::Database.format_boolean(enable_quoting), + ::Migrations::Database.format_boolean(enable_smart_lists), + ::Migrations::Database.format_boolean(external_links_in_new_tab), + ::Migrations::Database.format_boolean(hide_presence), + ::Migrations::Database.format_boolean(hide_profile), + ::Migrations::Database.format_boolean(hide_profile_and_presence), + homepage_id, + ::Migrations::Database.format_boolean(ignore_channel_wide_mention), + ::Migrations::Database.format_boolean(include_tl0_in_digests), + ::Migrations::Database.format_datetime(last_redirected_to_top_at), + like_notification_frequency, + ::Migrations::Database.format_boolean(mailing_list_mode), + mailing_list_mode_frequency, + new_topic_duration_minutes, + notification_level_when_replying, + ::Migrations::Database.format_datetime(oldest_search_log_date), + ::Migrations::Database.format_boolean(only_chat_push_notifications), + seen_popups, + ::Migrations::Database.format_boolean(show_thread_title_prompts), + ::Migrations::Database.format_boolean(sidebar_link_to_filtered_list), + ::Migrations::Database.format_boolean(sidebar_show_count_of_new_items), + ::Migrations::Database.format_boolean(skip_new_user_tips), + text_size_key, + text_size_seq, + theme_ids, + theme_key_seq, + timezone, + title_count_mode_key, + ::Migrations::Database.format_boolean(topics_unread_when_closed), + ::Migrations::Database.format_boolean(watched_precedence_over_muted), + ) + end + end +end diff --git a/migrations/lib/database/intermediate_db/user_suspension.rb b/migrations/lib/database/intermediate_db/user_suspension.rb new file mode 100644 index 00000000000..6836e8b7da5 --- /dev/null +++ b/migrations/lib/database/intermediate_db/user_suspension.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Migrations::Database::IntermediateDB + module UserSuspension + SQL = <<~SQL + INSERT INTO user_suspensions ( + user_id, + suspended_at, + suspended_till, + suspended_by_id, + reason + ) + VALUES ( + ?, ?, ?, ?, ? + ) + SQL + + def self.create(user_id:, suspended_at:, suspended_till: nil, suspended_by_id: nil, reason: nil) + ::Migrations::Database::IntermediateDB.insert( + SQL, + user_id, + ::Migrations::Database.format_datetime(suspended_at), + ::Migrations::Database.format_datetime(suspended_till), + suspended_by_id, + reason, + ) + end + end +end diff --git a/migrations/lib/database/schema.rb b/migrations/lib/database/schema.rb new file mode 100644 index 00000000000..42d9434f80e --- /dev/null +++ b/migrations/lib/database/schema.rb @@ -0,0 +1,175 @@ +# frozen_string_literal: true + +module Migrations::Database + module Schema + Table = + Data.define(:name, :columns, :indexes, :primary_key_column_names) do + def sorted_columns + columns.sort_by { |c| [c.is_primary_key ? 0 : 1, c.name] } + end + end + Column = Data.define(:name, :datatype, :nullable, :max_length, :is_primary_key) + Index = Data.define(:name, :column_names, :unique, :condition) + + class ConfigError < StandardError + end + + SQLITE_KEYWORDS = %w[ + abort + action + add + after + all + alter + always + analyze + and + as + asc + attach + autoincrement + before + begin + between + by + cascade + case + cast + check + collate + column + commit + conflict + constraint + create + cross + current + current_date + current_time + current_timestamp + database + default + deferrable + deferred + delete + desc + detach + distinct + do + drop + each + else + end + escape + except + exclude + exclusive + exists + explain + fail + filter + first + following + for + foreign + from + full + generated + glob + group + groups + having + if + ignore + immediate + in + index + indexed + initially + inner + insert + instead + intersect + into + is + isnull + join + key + last + left + like + limit + match + materialized + natural + no + not + nothing + notnull + null + nulls + of + offset + on + or + order + others + outer + over + partition + plan + pragma + preceding + primary + query + raise + range + recursive + references + regexp + reindex + release + rename + replace + restrict + returning + right + rollback + row + rows + savepoint + select + set + table + temp + temporary + then + ties + to + transaction + trigger + unbounded + union + unique + update + using + vacuum + values + view + virtual + when + where + window + with + without + ] + + def self.escape_identifier(identifier) + if SQLITE_KEYWORDS.include?(identifier) + %Q("#{identifier}") + else + identifier + end + end + end +end diff --git a/migrations/lib/database/schema/config_validator.rb b/migrations/lib/database/schema/config_validator.rb new file mode 100644 index 00000000000..17ffaf436aa --- /dev/null +++ b/migrations/lib/database/schema/config_validator.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema + class ConfigValidator + attr_reader :errors + + def initialize + @errors = [] + end + + def validate(config) + @errors.clear + + validate_with_json_schema(config) + return self if has_errors? + + validate_output_config(config) + validate_schema_config(config) + validate_plugins(config) + + self + end + + def has_errors? + @errors.any? + end + + private + + def validate_with_json_schema(config) + Validation::JsonSchemaValidator.new(config, @errors).validate + end + + def validate_output_config(config) + Validation::OutputConfigValidator.new(config, @errors).validate + end + + def validate_schema_config(config) + Validation::SchemaConfigValidator.new(config, @errors).validate + end + + def validate_plugins(config) + Validation::PluginConfigValidator.new(config, @errors).validate + end + end +end diff --git a/migrations/lib/database/schema/global_config.rb b/migrations/lib/database/schema/global_config.rb new file mode 100644 index 00000000000..a1d2f1c8d5d --- /dev/null +++ b/migrations/lib/database/schema/global_config.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema + class GlobalConfig + attr_reader :excluded_column_names, :modified_columns + + def initialize(schema_config) + @schema_config = schema_config + @excluded_table_names = load_globally_excluded_table_names.freeze + @excluded_column_names = load_globally_excluded_column_names.freeze + @modified_columns = load_globally_modified_columns.freeze + end + + def excluded_table_name?(table_name) + @excluded_table_names.include?(table_name) + end + + def modified_name(column_name) + if (modified_column = find_modified_column(column_name)) + modified_column[:rename_to] + end + end + + def modified_datatype(column_name) + if (modified_column = find_modified_column(column_name)) + modified_column[:datatype] + end + end + + private + + def find_modified_column(column_name) + @modified_columns.find { |column| column[:name] == column_name } || + @modified_columns.find { |column| column[:name_regex]&.match?(column_name) } + end + + def load_globally_excluded_table_names + table_names = @schema_config.dig(:global, :tables, :exclude) + table_names.presence&.to_set || Set.new + end + + def load_globally_excluded_column_names + column_names = @schema_config.dig(:global, :columns, :exclude) + column_names.presence || [] + end + + def load_globally_modified_columns + modified_columns = @schema_config.dig(:global, :columns, :modify) + return {} if modified_columns.blank? + + modified_columns.map do |column| + if column[:name_regex] + column[:name_regex_original] = column[:name_regex] + column[:name_regex] = Regexp.new(column[:name_regex]) + end + column[:datatype] = column[:datatype]&.to_sym + column + end + end + end +end diff --git a/migrations/lib/database/schema/loader.rb b/migrations/lib/database/schema/loader.rb new file mode 100644 index 00000000000..765f6734d04 --- /dev/null +++ b/migrations/lib/database/schema/loader.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema + class Loader + def initialize(schema_config) + @schema_config = schema_config + @global = GlobalConfig.new(@schema_config) + end + + def load_schema + @db = ActiveRecord::Base.lease_connection + + schema = [] + existing_table_names = @db.tables.to_set + + @schema_config[:tables].sort.each do |table_name, config| + table_name = table_name.to_s + + if config[:copy_of].present? + table_alias = table_name + table_name = config[:copy_of] + else + next if @global.excluded_table_name?(table_name) + end + + if existing_table_names.include?(table_name) + schema << table(table_name, config, table_alias) + end + end + + @db = nil + ActiveRecord::Base.release_connection + + schema + end + + private + + def table(table_name, config, table_alias = nil) + primary_key_column_names = + config[:primary_key_column_names].presence || @db.primary_keys(table_name) + + columns = + filtered_columns_of(table_name, config).map do |column| + Column.new( + name: name_for(column), + datatype: datatype_for(column), + nullable: column.null || column.default, + max_length: column.type == :text ? column.limit : nil, + is_primary_key: primary_key_column_names.include?(column.name), + ) + end + added_columns(config, primary_key_column_names) + + Table.new(table_alias || table_name, columns, indexes(config), primary_key_column_names) + end + + def filtered_columns_of(table_name, config) + columns_by_name = @db.columns(table_name).index_by(&:name) + columns_by_name.except!(*@global.excluded_column_names) + + if (included_columns = config.dig(:columns, :include)) + columns_by_name.slice!(*included_columns) + elsif (excluded_columns = config.dig(:columns, :exclude)) + columns_by_name.except!(*excluded_columns) + end + + columns_by_name.values + end + + def added_columns(config, primary_key_column_names) + columns = config.dig(:columns, :add) || [] + columns.map do |column| + datatype = column[:datatype].to_sym + Column.new( + name: column[:name], + datatype:, + nullable: column.fetch(:nullable, true), + max_length: datatype == :text ? column[:max_length] : nil, + is_primary_key: primary_key_column_names.include?(column[:name]), + ) + end + end + + def name_for(column) + @global.modified_name(column.name) || column.name + end + + def datatype_for(column) + datatype = @global.modified_datatype(column.name) || column.type + + case datatype + when :binary + :blob + when :string, :enum, :uuid + :text + when :jsonb + :json + when :boolean, :date, :datetime, :float, :inet, :integer, :numeric, :json, :text + datatype + else + raise "Unknown datatype: #{datatype}" + end + end + + def indexes(config) + config[:indexes]&.map do |index| + Index.new( + name: index[:name], + column_names: Array.wrap(index[:columns]), + unique: index.fetch(:unique, false), + condition: index[:condition], + ) + end + end + end +end diff --git a/migrations/lib/database/schema/model_writer.rb b/migrations/lib/database/schema/model_writer.rb new file mode 100644 index 00000000000..422b85a3816 --- /dev/null +++ b/migrations/lib/database/schema/model_writer.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +require "rake" +require "syntax_tree/rake_tasks" + +module Migrations::Database::Schema + class ModelWriter + def initialize(namespace, header) + @namespace = namespace + @header = header.gsub(/^/, "# ") + end + + def self.filename_for(table) + "#{table.name.singularize}.rb" + end + + def self.format_files(path) + glob_pattern = File.join(path, "**/*.rb") + + system( + "bundle", + "exec", + "stree", + "write", + glob_pattern, + exception: true, + out: File::NULL, + err: File::NULL, + ) + rescue StandardError + raise "Failed to run `bundle exec stree write '#{glob_pattern}'`" + end + + def output_table(table, output_stream) + columns = table.sorted_columns + + output_stream.puts "# frozen_string_literal: true" + output_stream.puts + output_stream.puts @header + output_stream.puts + output_stream.puts "module #{@namespace}" + output_stream.puts " module #{to_singular_classname(table.name)}" + output_stream.puts " SQL = <<~SQL" + output_stream.puts " INSERT INTO #{escape_identifier(table.name)} (" + output_stream.puts column_names(columns) + output_stream.puts " )" + output_stream.puts " VALUES (" + output_stream.puts value_placeholders(columns) + output_stream.puts " )" + output_stream.puts " SQL" + output_stream.puts + output_stream.puts " def self.create(" + output_stream.puts method_parameters(columns) + output_stream.puts " )" + output_stream.puts " ::Migrations::Database::IntermediateDB.insert(" + output_stream.puts " SQL," + output_stream.puts insertion_arguments(columns) + output_stream.puts " )" + output_stream.puts " end" + output_stream.puts " end" + output_stream.puts "end" + end + + private + + def to_singular_classname(snake_case_string) + snake_case_string.singularize.camelize + end + + def column_names(columns) + columns.map { |c| " #{escape_identifier(c.name)}" }.join(",\n") + end + + def value_placeholders(columns) + indentation = " " + max_length = 100 - indentation.length + placeholder = "?, " + placeholder_count = columns.size + + current_length = 0 + placeholders = indentation.dup + + (1..placeholder_count).each do |index| + placeholder = "?" if index == placeholder_count + + if current_length + placeholder.length > max_length + placeholders.rstrip! + placeholders << "\n" << indentation + current_length = 0 + end + + placeholders << placeholder + current_length += placeholder.length + end + + placeholders + end + + def method_parameters(columns) + columns + .map do |c| + default_value = !c.is_primary_key && c.nullable ? " nil" : "" + " #{c.name}:#{default_value}" + end + .join(",\n") + end + + def insertion_arguments(columns) + columns + .map do |c| + argument = + case c.datatype + when :datetime + "::Migrations::Database.format_datetime(#{c.name})" + when :date + "::Migrations::Database.format_date(#{c.name})" + when :boolean + "::Migrations::Database.format_boolean(#{c.name})" + when :inet + "::Migrations::Database.format_ip_address(#{c.name})" + when :blob + "::Migrations::Database.to_blob(#{c.name})" + when :json + "::Migrations::Database.to_json(#{c.name})" + when :float, :integer, :numeric, :text + c.name + else + raise "Unknown dataype: #{type}" + end + " #{argument}," + end + .join("\n") + end + + def escape_identifier(identifier) + ::Migrations::Database::Schema.escape_identifier(identifier) + end + end +end diff --git a/migrations/lib/database/schema/table_writer.rb b/migrations/lib/database/schema/table_writer.rb new file mode 100644 index 00000000000..7e8e9ff7c24 --- /dev/null +++ b/migrations/lib/database/schema/table_writer.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema + class TableWriter + def initialize(output_stream) + @output = output_stream + end + + def output_file_header(header) + @output.puts header.gsub(/^/, "-- ") + @output.puts + end + + def output_table(table) + output_create_table_statement(table) + output_columns(table) + output_primary_key(table) + output_indexes(table) + @output.puts "" + end + + private + + def output_create_table_statement(table) + @output.puts "CREATE TABLE #{escape_identifier(table.name)}" + @output.puts "(" + end + + def output_columns(table) + column_definitions = create_column_definitions(table) + @output.puts column_definitions.join(",\n") + end + + def output_primary_key(table) + if table.primary_key_column_names.size > 1 + pk_definition = + table.primary_key_column_names.map { |name| escape_identifier(name) }.join(", ") + @output.puts " PRIMARY KEY (#{pk_definition})" + end + @output.puts ");" + end + + def create_column_definitions(table) + columns = table.sorted_columns + has_composite_primary_key = table.primary_key_column_names.size > 1 + + max_column_name_length = columns.map { |c| escape_identifier(c.name).length }.max + max_datatype_length = columns.map { |c| convert_datatype(c.datatype).length }.max + + columns.map do |c| + definition = [ + escape_identifier(c.name).ljust(max_column_name_length), + convert_datatype(c.datatype).ljust(max_datatype_length), + ] + + if c.is_primary_key && !has_composite_primary_key + definition << "NOT NULL" if c.datatype != :integer + definition << "PRIMARY KEY" + else + definition << "NOT NULL" unless c.nullable + end + + definition = definition.join(" ") + definition.strip! + + " #{definition}" + end + end + + def convert_datatype(type) + case type + when :blob, :boolean, :date, :datetime, :float, :integer, :numeric, :text + type.to_s.upcase + when :inet + "INET_TEXT" + when :json + "JSON_TEXT" + else + raise "Unknown datatype: #{type}" + end + end + + def escape_identifier(identifier) + ::Migrations::Database::Schema.escape_identifier(identifier) + end + + def output_indexes(table) + return unless table.indexes + + table.indexes.each do |index| + index_name = escape_identifier(index.name) + table_name = escape_identifier(table.name) + column_names = index.column_names.map { |name| escape_identifier(name) } + + @output.puts "" + @output.print "CREATE " + @output.print "UNIQUE " if index.unique + @output.print "INDEX #{index_name} ON #{table_name} (#{column_names.join(", ")})" + @output.print " #{index.condition}" if index.condition.present? + @output.puts ";" + end + end + end +end diff --git a/migrations/lib/database/schema/validation/base_validator.rb b/migrations/lib/database/schema/validation/base_validator.rb new file mode 100644 index 00000000000..4635646904d --- /dev/null +++ b/migrations/lib/database/schema/validation/base_validator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema::Validation + class BaseValidator + def initialize(config, errors, db) + @config = config + @schema_config ||= @config[:schema] + @errors = errors + @db = db + end + + private + + def sort_and_join(values) + values.sort.join(", ") + end + end +end diff --git a/migrations/lib/database/schema/validation/columns_validator.rb b/migrations/lib/database/schema/validation/columns_validator.rb new file mode 100644 index 00000000000..544799bb475 --- /dev/null +++ b/migrations/lib/database/schema/validation/columns_validator.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema::Validation + class ColumnsValidator < BaseValidator + def validate(table_name, copy_of_table_name = nil) + @table_name = table_name + columns = @schema_config[:tables][table_name.to_sym][:columns] || {} + + @existing_column_names = find_existing_column_names(table_name, copy_of_table_name) + @added_column_names = columns[:add]&.map { |column| column[:name] } || [] + @included_column_names = columns[:include] || [] + @excluded_column_names = columns[:exclude] || [] + @modified_column_names = columns[:modify]&.map { |column| column[:name] } || [] + @global = ::Migrations::Database::Schema::GlobalConfig.new(@schema_config) + + validated_added_columns + validate_included_columns + validate_excluded_columns + validate_modified_columns + validate_any_columns_configured + end + + private + + def find_existing_column_names(table_name, copy_of_table_name) + table_name = copy_of_table_name if copy_of_table_name.present? + @db.columns(table_name).map { |c| c.name } + end + + def validated_added_columns + if (column_names = @existing_column_names & @added_column_names).any? + @errors << I18n.t( + "schema.validator.tables.added_columns_exist", + table_name: @table_name, + column_names: sort_and_join(column_names), + ) + end + end + + def validate_included_columns + if (column_names = @included_column_names - @existing_column_names).any? + @errors << I18n.t( + "schema.validator.tables.included_columns_missing", + table_name: @table_name, + column_names: sort_and_join(column_names), + ) + end + end + + def validate_excluded_columns + if (column_names = @excluded_column_names - @existing_column_names).any? + @errors << I18n.t( + "schema.validator.tables.excluded_columns_missing", + table_name: @table_name, + column_names: sort_and_join(column_names), + ) + end + end + + def validate_modified_columns + if (column_names = @modified_column_names - @existing_column_names).any? + @errors << I18n.t( + "schema.validator.tables.modified_columns_missing", + table_name: @table_name, + column_names: sort_and_join(column_names), + ) + end + + if (column_names = @modified_column_names & @included_column_names).any? + @errors << I18n.t( + "schema.validator.tables.modified_columns_included", + table_name: @table_name, + column_names: sort_and_join(column_names), + ) + end + + if (column_names = @modified_column_names & @excluded_column_names).any? + @errors << I18n.t( + "schema.validator.tables.modified_columns_excluded", + table_name: @table_name, + column_names: sort_and_join(column_names), + ) + end + + if (column_names = @modified_column_names & @global.excluded_column_names).any? + @errors << I18n.t( + "schema.validator.tables.modified_columns_globally_excluded", + table_name: @table_name, + column_names: sort_and_join(column_names), + ) + end + end + + def configured_column_names + if @included_column_names.any? + included_column_names = @included_column_names + modified_column_names = @modified_column_names + else + included_column_names = @existing_column_names - @excluded_column_names + modified_column_names = included_column_names & @modified_column_names + end + + column_names = (included_column_names + modified_column_names).uniq & @existing_column_names + column_names - @global.excluded_column_names + @added_column_names + end + + def validate_any_columns_configured + column_names = configured_column_names + + if column_names.empty? + @errors << I18n.t("schema.validator.tables.no_columns_configured", table_name: @table_name) + end + end + end +end diff --git a/migrations/lib/database/schema/validation/globally_configured_columns_validator.rb b/migrations/lib/database/schema/validation/globally_configured_columns_validator.rb new file mode 100644 index 00000000000..b4120e6dd29 --- /dev/null +++ b/migrations/lib/database/schema/validation/globally_configured_columns_validator.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema::Validation + class GloballyConfiguredColumnsValidator < BaseValidator + def initialize(config, errors, db) + super + @global = ::Migrations::Database::Schema::GlobalConfig.new(@schema_config) + end + + def validate + all_column_names = calculate_all_column_names + + validate_excluded_column_names(all_column_names) + validate_modified_columns(all_column_names) + end + + private + + def calculate_all_column_names + existing_table_names = @db.tables + + configured_table_names = + @schema_config[:tables].map do |table_name, table_config| + table_config[:copy_of] || table_name.to_s + end + + globally_excluded_table_names = @schema_config.dig(:global, :tables, :exclude) || [] + excluded_table_names = @schema_config.dig(:global, :tables, :exclude) || [] + + all_table_names = existing_table_names - globally_excluded_table_names - excluded_table_names + all_table_names = all_table_names.uniq & configured_table_names + + all_table_names.flat_map { |table_name| @db.columns(table_name).map(&:name) }.uniq + end + + def validate_excluded_column_names(all_column_names) + globally_excluded_column_names = @global.excluded_column_names + excluded_missing_column_names = globally_excluded_column_names - all_column_names + + if excluded_missing_column_names.any? + @errors << I18n.t( + "schema.validator.global.excluded_columns_missing", + column_names: sort_and_join(excluded_missing_column_names), + ) + end + end + + def validate_modified_columns(all_column_names) + globally_modified_columns = @global.modified_columns + + excluded_missing_columns = + globally_modified_columns.reject do |column| + if column[:name] + all_column_names.include?(column[:name]) + elsif column[:name_regex] + all_column_names.any? { |column_name| column[:name_regex]&.match?(column_name) } + else + false + end + end + + if excluded_missing_columns.any? + excluded_missing_column_names = + excluded_missing_columns.map { |column| column[:name_regex_original] || column[:name] } + + @errors << I18n.t( + "schema.validator.global.modified_columns_missing", + column_names: sort_and_join(excluded_missing_column_names), + ) + end + end + end +end diff --git a/migrations/lib/database/schema/validation/globally_excluded_tables_validator.rb b/migrations/lib/database/schema/validation/globally_excluded_tables_validator.rb new file mode 100644 index 00000000000..36459049867 --- /dev/null +++ b/migrations/lib/database/schema/validation/globally_excluded_tables_validator.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema::Validation + class GloballyExcludedTablesValidator < BaseValidator + def validate + excluded_table_names = @schema_config.dig(:global, :tables, :exclude) + return if excluded_table_names.blank? + + existing_table_names = @db.tables + missing_table_names = excluded_table_names - existing_table_names + + if missing_table_names.any? + @errors << I18n.t( + "schema.validator.global.excluded_tables_missing", + table_names: sort_and_join(missing_table_names), + ) + end + end + end +end diff --git a/migrations/lib/database/schema/validation/json_schema_validator.rb b/migrations/lib/database/schema/validation/json_schema_validator.rb new file mode 100644 index 00000000000..58fd1bbae20 --- /dev/null +++ b/migrations/lib/database/schema/validation/json_schema_validator.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require "json_schemer" + +module Migrations::Database::Schema::Validation + class JsonSchemaValidator + def initialize(config, errors) + @config = config + @errors = errors + end + + def validate + schema = load_json_schema + schemer = ::JSONSchemer.schema(schema) + response = schemer.validate(@config) + response.each { |r| @errors << transform_json_schema_errors(r.fetch("error")) } + end + + private + + def load_json_schema + schema_path = File.join(::Migrations.root_path, "config", "json_schemas", "db_schema.json") + JSON.load_file(schema_path) + end + + def transform_json_schema_errors(error_message) + error_message.gsub!(/value at (`.+?`) matches `not` schema/) do + I18n.t("schema.validator.include_exclude_not_allowed", path: $1) + end + error_message + end + end +end diff --git a/migrations/lib/database/schema/validation/output_config_validator.rb b/migrations/lib/database/schema/validation/output_config_validator.rb new file mode 100644 index 00000000000..36213a3b8b7 --- /dev/null +++ b/migrations/lib/database/schema/validation/output_config_validator.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema::Validation + class OutputConfigValidator < BaseValidator + def initialize(config, errors) + super(config, errors, nil) + @output_config = config[:output] + end + + def validate + validate_schema_file_directory + validate_models_directory + validate_models_namespace + end + + private + + def validate_schema_file_directory + schema_file_path = File.dirname(@output_config[:schema_file]) + schema_file_path = File.expand_path(schema_file_path, ::Migrations.root_path) + + if !Dir.exist?(schema_file_path) + @errors << I18n.t("schema.validator.output.schema_file_directory_not_found") + end + end + + def validate_models_directory + models_directory = File.expand_path(@output_config[:models_directory], ::Migrations.root_path) + + if !Dir.exist?(models_directory) + @errors << I18n.t("schema.validator.output.models_directory_not_found") + end + end + + def validate_models_namespace + existing_namespace = + begin + Object.const_get(@output_config[:models_namespace]).is_a?(Module) + rescue NameError + false + end + + @errors << I18n.t("schema.validator.output.models_namespace_undefined") if !existing_namespace + end + end +end diff --git a/migrations/lib/database/schema/validation/plugin_config_validator.rb b/migrations/lib/database/schema/validation/plugin_config_validator.rb new file mode 100644 index 00000000000..ffaf87eea2b --- /dev/null +++ b/migrations/lib/database/schema/validation/plugin_config_validator.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema::Validation + class PluginConfigValidator < BaseValidator + def initialize(config, errors) + super(config, errors, nil) + end + + def validate + all_plugin_names = Discourse.plugins.map(&:name) + configured_plugin_names = @config[:plugins] + + if (additional_plugins = all_plugin_names - configured_plugin_names).any? + @errors << I18n.t( + "schema.validator.plugins.additional_installed", + plugin_names: sort_and_join(additional_plugins), + ) + end + + if (missing_plugins = configured_plugin_names - all_plugin_names).any? + @errors << I18n.t( + "schema.validator.plugins.not_installed", + plugin_names: sort_and_join(missing_plugins), + ) + end + end + end +end diff --git a/migrations/lib/database/schema/validation/schema_config_validator.rb b/migrations/lib/database/schema/validation/schema_config_validator.rb new file mode 100644 index 00000000000..4d252d41cc3 --- /dev/null +++ b/migrations/lib/database/schema/validation/schema_config_validator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema::Validation + class SchemaConfigValidator + def initialize(config, errors) + @config = config + @errors = errors + end + + def validate + ActiveRecord::Base.with_connection do |db| + GloballyExcludedTablesValidator.new(@config, @errors, db).validate + GloballyConfiguredColumnsValidator.new(@config, @errors, db).validate + TablesValidator.new(@config, @errors, db).validate + end + end + end +end diff --git a/migrations/lib/database/schema/validation/tables_validator.rb b/migrations/lib/database/schema/validation/tables_validator.rb new file mode 100644 index 00000000000..0b99c5ba356 --- /dev/null +++ b/migrations/lib/database/schema/validation/tables_validator.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Migrations::Database::Schema::Validation + class TablesValidator < BaseValidator + def initialize(config, errors, db) + super + + @existing_table_names = @db.tables + @configured_tables = @schema_config[:tables] + @configured_table_names = @configured_tables.keys.map(&:to_s) + @excluded_table_names = @schema_config.dig(:global, :tables, :exclude) || [] + end + + def validate + validate_excluded_tables + validate_unconfigured_tables + validate_copied_tables + validate_columns + end + + private + + def validate_excluded_tables + table_names = @configured_table_names & @excluded_table_names + + if table_names.any? + @errors << I18n.t( + "schema.validator.tables.excluded_tables_configured", + table_names: sort_and_join(table_names), + ) + end + end + + def validate_unconfigured_tables + table_names = @existing_table_names - @configured_table_names - @excluded_table_names + + if table_names.any? + @errors << I18n.t( + "schema.validator.tables.not_configured", + table_names: sort_and_join(table_names), + ) + end + end + + def validate_copied_tables + @configured_tables.each do |_table_name, table_config| + next unless table_config[:copy_of] + + if !@existing_table_names.include?(table_config[:copy_of]) + @errors << I18n.t( + "schema.validator.tables.copy_table_not_found", + table_name: table_config[:copy_of], + ) + end + end + end + + def validate_columns + @configured_tables.each do |table_name, table_config| + validator = ColumnsValidator.new(@config, @errors, @db) + + if table_config[:copy_of] + validator.validate(table_name.to_s, table_config[:copy_of]) + else + validator.validate(table_name.to_s) + end + end + end + end +end diff --git a/migrations/lib/importer/steps/users.rb b/migrations/lib/importer/steps/users.rb new file mode 100644 index 00000000000..61170f859e9 --- /dev/null +++ b/migrations/lib/importer/steps/users.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +module Migrations::Importer::Steps + class Users < ::Migrations::Importer::CopyStep + INSERT_MAPPED_USERNAMES_SQL = <<~SQL + INSERT INTO mapped.usernames (discourse_user_id, original_username, discourse_username) + VALUES (?, ?, ?) + SQL + + # requires_shared_data :usernames, :group_names + requires_mapping :user_ids_by_email, "SELECT LOWER(email) AS email, user_id FROM user_emails" + requires_mapping :user_ids_by_external_id, + "SELECT external_id, user_id FROM single_sign_on_records" + + table_name :users + column_names %i[ + id + username + username_lower + name + active + trust_level + group_locked_trust_level + manual_locked_trust_level + admin + moderator + date_of_birth + locale + ip_address + registration_ip_address + primary_group_id + flair_group_id + suspended_at + suspended_till + first_seen_at + last_seen_at + last_emailed_at + silenced_till + approved + approved_at + approved_by_id + views + created_at + updated_at + ] + + store_mapped_ids true + + total_rows_query <<~SQL, MappingType::USERS + SELECT COUNT(*) + FROM users u + LEFT JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ? + WHERE mu.original_id IS NULL + SQL + + rows_query <<~SQL, MappingType::USERS + SELECT u.*, + us.suspended_at, + us.suspended_till, + JSON_GROUP_ARRAY(LOWER(ue.email)) AS emails + FROM users u + LEFT JOIN user_emails ue ON u.original_id = ue.user_id + LEFT JOIN user_suspensions us ON u.original_id = us.user_id AND us.suspended_at < DATETIME() AND + (us.suspended_till IS NULL OR us.suspended_till > DATETIME()) + LEFT JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ? + WHERE mu.original_id IS NULL + GROUP BY u.original_id + ORDER BY u.ROWID + SQL + + def initialize(intermediate_db, discourse_db, shared_data) + super + @unique_name_finder = ::Migrations::Importer::UniqueNameFinder.new(@shared_data) + end + + private + + def transform_row(row) + if row[:emails].present? + JSON + .parse(row[:emails]) + .each do |email| + if (existing_user_id = @user_ids_by_email[email]) + row[:id] = existing_user_id + return nil + end + end + end + + if row[:external_id].present? && + (existing_user_id = @user_ids_by_external_id[row[:external_id]]) + row[:id] = existing_user_id + return nil + end + + row[:original_username] ||= row[:username] + row[:username] = @unique_name_finder.find_available_username( + row[:username], + allow_reserved_username: row[:admin] == 1, + ) + row[:username_lower] = row[:username].downcase + + row[:trust_level] ||= TrustLevel[1] + row[:active] = true if row[:active].nil? + row[:admin] = false if row[:admin].nil? + row[:moderator] = false if row[:moderator].nil? + row[:staged] = false if row[:staged].nil? + + row[:last_emailed_at] ||= NOW + row[:suspended_till] ||= 200.years.from_now if row[:suspended_at].present? + + date_of_birth = Migrations::Database.to_date(row[:date_of_birth]) + if date_of_birth && date_of_birth.year != 1904 + row[:date_of_birth] = Date.new(1904, date_of_birth.month, date_of_birth.day) + end + + if SiteSetting.must_approve_users || !row[:approved].nil? + row[:approved] = true if row[:approved].nil? + row[:approved_at] = row[:approved] ? row[:approved_at] || NOW : nil + row[:approved_by_id] = row[:approved] ? row[:approved_by_id] || SYSTEM_USER_ID : nil + end + + row[:views] ||= 0 + + super + end + + def after_commit_of_inserted_rows(rows) + super + + rows.each do |row| + if row[:id] && row[:username] && row[:username] != row[:original_username] + @intermediate_db.insert( + INSERT_MAPPED_USERNAMES_SQL, + [row[:id], row[:original_username], row[:username]], + ) + end + end + end + end +end diff --git a/migrations/scripts/generate_schema b/migrations/scripts/generate_schema deleted file mode 100755 index 13b98a40c7a..00000000000 --- a/migrations/scripts/generate_schema +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# Generate the converter's base intermediate database migration file from -# the core database state and YAML configuration in schema.yml -# Invoke from core root directory as `./migrations/scripts/generate_schema` -# It accepts an optional command line argument for the output file path which -# overrides the path configured in schema.yml - -require_relative "../lib/migrations" - -module Migrations - load_rails_environment - - class SchemaGenerator - def initialize(opts = {}) - config = load_config - - @core_db_connection = ActiveRecord::Base.connection - @output_stream = StringIO.new - @indirectly_ignored_columns = Hash.new { |h, k| h[k] = [] } - - @output_file_path = opts[:output_file_path] || config[:output_file_path] - - @table_configs = config[:tables] - @column_configs = config[:columns] - - @configured_table_names = @table_configs&.keys&.sort || [] - @global_column_ignore_list = @column_configs&.fetch(:ignore) || [] - end - - def run - puts "Generating base converter migration file for Discourse #{Discourse::VERSION::STRING}" - - generate_header - generate_tables - generate_indirectly_ignored_columns_log - generate_migration_file - validate_migration_file - - puts "", "Done" - end - - private - - def load_config - path = File.expand_path("../config/intermediate_db.yml", __dir__) - YAML.load_file(path, symbolize_names: true) - end - - def generate_header - return if @configured_table_names.empty? - - @output_stream.puts <<~HEADER - /* - This file is auto-generated from the Discourse core database schema. Instead of editing it directly, - please update the `schema.yml` configuration file and re-run the `generate_schema` script to update it. - */ - HEADER - end - - def generate_tables - puts "Generating tables..." - - @configured_table_names.each do |name| - raise "Core table named '#{name}' not found" unless valid_table?(name) - - generate_table(name) - end - end - - def generate_indirectly_ignored_columns_log - return if @indirectly_ignored_columns.empty? - - puts "Generating indirectly ignored column list..." - - @output_stream.puts "\n\n/*" - - @output_stream.puts <<~NOTE - Core table columns implicitly excluded from the generated schema above via the `include` configuration option - in `schema.yml`. This serves as an inventory of these columns, allowing new core additions to be tracked and, - if necessary, synchronized with the intermediate database schema.\n - NOTE - - @indirectly_ignored_columns.each_with_index do |(table_name, columns), index| - next if virtual_table?(table_name) || columns.blank? - - @output_stream.puts "" if index.positive? - @output_stream.puts "Table: #{table_name}" - @output_stream.puts "--------#{"-" * table_name.length}" - - columns.each do |column| - @output_stream.puts " #{column.name} #{column.type} #{column.null}" - end - end - - @output_stream.puts "*/" - end - - def generate_migration_file - file_path = File.expand_path(@output_file_path, __dir__) - - puts "Generating base migration file '#{file_path}'..." - - File.open(file_path, "w") { |f| f << @output_stream.string.chomp } - end - - def generate_column_definition(column) - definition = " #{column.name} #{type(column)}" - definition << " NOT NULL" unless column.null - - definition - end - - def generate_index(table_name, index) - @output_stream.print "CREATE " - @output_stream.print "UNIQUE " if index[:unique] - @output_stream.print "INDEX #{index[:name]} ON #{table_name} (#{index[:columns].join(", ")})" - @output_stream.print " #{index[:condition]}" if index[:condition].present? - @output_stream.puts ";" - end - - def column_list_for(table_name) - ignore_columns = @table_configs.dig(table_name, :ignore) || [] - include_columns = @table_configs.dig(table_name, :include) || [] - - include_columns.present? ? [:include, include_columns] : [:ignore, ignore_columns] - end - - def generate_table(name) - puts "Generating #{name}..." - - column_definitions = [] - column_records = columns(name) - mode, column_list = column_list_for(name) - indexes = indexes(name) - configured_primary_key = primary_key(name) - - primary_key, composite_key = - if configured_primary_key.present? - [configured_primary_key].flatten.each do |pk| - if column_records.map(&:name).exclude?(pk) - raise "Column named '#{pk}' does not exist in table '#{name}'" - end - end - - [ - configured_primary_key, - configured_primary_key.is_a?(Array) && configured_primary_key.length > 1, - ] - else - virtual_table?(name) ? [] : [@core_db_connection.primary_key(name), false] - end - - @output_stream.puts "" - @output_stream.puts "CREATE TABLE #{name}" - @output_stream.puts "(" - - if !composite_key && primary_key.present? - primary_key_column = column_records.find { |c| c.name == primary_key } - - if (mode == :include && column_list.include?(primary_key_column.name)) || - (mode == :ignore && column_list.exclude?(primary_key_column.name)) - column_definitions << " #{primary_key_column.name} #{type(primary_key_column)} NOT NULL PRIMARY KEY" - end - end - - column_records.each do |column| - next if @global_column_ignore_list.include?(column.name) - next if (mode == :ignore) && column_list.include?(column.name) - if !column.is_a?(CustomColumn) && (mode == :include) && column_list.exclude?(column.name) - @indirectly_ignored_columns[name] << column - - next - end - next if !composite_key && (column.name == primary_key) - - column_definitions << generate_column_definition(column) - end - - format_columns!(column_definitions) - - column_definitions << " PRIMARY KEY (#{primary_key.join(", ")})" if composite_key - - @output_stream.puts column_definitions.join(",\n") - @output_stream.puts ");" - @output_stream.puts "" if indexes.present? - - indexes.each { |index| generate_index(name, index) } - end - - def validate_migration_file - db = Extralite::Database.new(":memory:") - - if (sql = @output_stream.string).blank? - warn "No SQL generated, skipping validation".red - else - db.execute(sql) - end - ensure - db.close if db - end - - def format_columns!(column_definitions) - column_definitions.map! do |c| - c.match( - /^\s*(?\w+)\s(?\w+)\s?(?NOT NULL)?\s?(?PRIMARY KEY)?/, - ).named_captures - end - - max_name_length = column_definitions.map { |c| c["name"].length }.max - max_datatype_length = column_definitions.map { |c| c["datatype"].length }.max - - column_definitions.sort_by! do |c| - [c["primary_key"] ? 0 : 1, c["nullable"] ? 0 : 1, c["name"]] - end - column_definitions.map! do |c| - " #{c["name"].ljust(max_name_length)} #{c["datatype"].ljust(max_datatype_length)} #{c["nullable"]} #{c["primary_key"]}".rstrip - end - end - - class CustomColumn - attr_reader :name - - def initialize(name, type, null) - @name = name - @raw_type = type - @raw_null = null - end - - def type - @raw_type&.to_sym || :text - end - - def null - @raw_null.nil? ? true : @raw_null - end - - def merge!(other_column) - @raw_null = other_column.null if @raw_null.nil? - @raw_type ||= other_column.type - - self - end - end - - def columns(name) - extensions = column_extensions(name) - - return extensions if virtual_table?(name) - - default_columns = @core_db_connection.columns(name) - - return default_columns if extensions.blank? - - extended_columns = - default_columns.map do |default_column| - extension = extensions.find { |ext| ext.name == default_column.name } - - if extension - extensions.delete(extension) - - extension.merge!(default_column) - else - default_column - end - end - - extended_columns + extensions - end - - def column_extensions(name) - extensions = @table_configs.dig(name, :extend) - - return [] if extensions.nil? - - extensions.map { |column| CustomColumn.new(column[:name], column[:type], column[:is_null]) } - end - - def type(column) - case column.type - when :string, :inet - "TEXT" - else - column.type.to_s.upcase - end - end - - def valid_table?(name) - @core_db_connection.tables.include?(name.to_s) || virtual_table?(name) - end - - def virtual_table?(name) - !!@table_configs.dig(name, :virtual) - end - - def indexes(table_name) - @table_configs.dig(table_name, :indexes) || [] - end - - def primary_key(table_name) - @table_configs.dig(table_name, :primary_key) - end - end -end - -::Migrations::SchemaGenerator.new(output_file_path: ARGV.first).run diff --git a/migrations/scripts/schema.yml b/migrations/scripts/schema.yml deleted file mode 100644 index 1cd4ca8a1b9..00000000000 --- a/migrations/scripts/schema.yml +++ /dev/null @@ -1,481 +0,0 @@ -## Configuration options for the base intermediate schema generator -## -## After modifying this file, regenerate the base intermediate schema -## by running the `generate_schema` script. - -# Default relative path for generated base schema file. -# An absolute path can also be provided to the script as the first CLI argument. -# If the CLI argument is present, it takes precedence over the value specified here. -output_file_path: ../common/intermediate_db_schema/000_base_schema.sql - -## Tables to include in the generated base intermediate schema. -## -## Available table options: -## virtual: Boolean. Enables the inclusion of a table in the schema solely based. -## on the provided configuration. A virtual table does not need to be available in the core schema. -## ignore: List of columns to ignore. Convenient if most of the table's column are needed. -## Usage is mutually exclusive with the `include` option. Only one should be used at a time. -## include: List of columns to include. Convenient if only a few columns are needed. -## Usage is mutually exclusive with the `include`` option. Only one should be used at a time. -## primary_key: Literal or list of columns to use as primary key. -## extend: List of objects describing columns to be added/extended. -## The following options are available for an "extend" object: -## name: Required. The name of the column being extended. -## is_null: Specifies if the column can be null. -## type: Column type. Defaults to TEXT. -## indexes: List of indexes to create. The following options are available for an "index" object: -## name: Index name. -## columns: List of column(s) to index. -tables: - schema_migrations: - virtual: true - primary_key: path - extend: - - name: path - is_null: false - - name: created_at - type: datetime - config: - virtual: true - primary_key: name - extend: - - name: name - is_null: false - - name: value - is_null: false - log_entries: - virtual: true - extend: - - name: created_at - type: datetime - is_null: false - - name: type - is_null: false - - name: message - is_null: false - - name: exception - - name: details - users: - ignore: - - seen_notification_id - - last_posted_at - - password_hash - - salt - - active - - last_emailed_at - - approved_by_id - - previous_visit_at - - suspended_at - - suspended_till - - views - - flag_level - - ip_address - - title - - uploaded_avatar_id - - locale - - primary_group_id - - first_seen_at - - silenced_till - - group_locked_trust_level - - manual_locked_trust_level - - secure_identifier - - flair_group_id - - last_seen_reviewable_id - - password_algorithm - - username_lower - extend: - - name: email - - name: created_at - is_null: true - - name: staged - is_null: true - - name: avatar_path - - name: avatar_url - - name: avatar_upload_id - - name: bio - - name: password - is_null: true - - name: trust_level - is_null: true - - name: suspension - - name: location - - name: website - - name: old_relative_url - - name: sso_record - - name: anonymized - type: boolean - - name: original_username - - name: timezone - - name: email_level - type: integer - - name: email_messages_level - type: integer - - name: email_digests - type: boolean - categories: - ignore: - - topic_id - - topic_count - - user_id - - topics_year - - topics_month - - topics_week - - auto_close_hours - - post_count - - latest_post_id - - latest_topic_id - - posts_year - - posts_month - - posts_week - - email_in - - email_in_allow_strangers - - topics_day - - posts_day - - allow_badges - - name_lower - - auto_close_based_on_last_post - - topic_template - - contains_messages - - sort_order - - sort_ascending - - uploaded_logo_id - - uploaded_background_id - - topic_featured_link_allowed - - all_topics_wiki - - show_subcategory_list - - num_featured_topics - - default_view - - subcategory_list_style - - default_top_period - - mailinglist_mirror - - minimum_required_tags - - navigate_to_first_post_after_read - - search_priority - - allow_global_tags - - reviewable_by_group_id - - read_only_banner - - default_list_filter - - allow_unlimited_owner_edits_on_first_post - - default_slow_mode_seconds - - uploaded_logo_dark_id - - uploaded_background_dark_id - extend: - - name: about_topic_title - - name: old_relative_url - - name: existing_id - type: integer - - name: permissions - type: json_text # JSON_TEXT ??? - - name: logo_upload_id - - name: tag_group_ids - type: json_text # JSON_TEXT ??? - topics: - ignore: - - last_posted_at - - posts_count - - last_post_user_id - - reply_count - - featured_user1_id - - featured_user2_id - - featured_user3_id - - featured_user4_id - - deleted_at - - highest_post_number - - like_count - - incoming_link_count - - moderator_posts_count - - bumped_at - - has_summary - - archetype - - notify_moderators_count - - spam_count - - score - - percent_rank - - slug - - deleted_by_id - - participant_count - - word_count - - excerpt - - fancy_title - - highest_staff_post_number - - featured_link - - reviewable_score - - image_upload_id - - slow_mode_seconds - - bannered_until - - external_id - extend: - - name: old_relative_url - - name: private_message - posts: - ignore: - - cooked - - reply_to_post_number - - reply_count - - quote_count - - deleted_at - - off_topic_count - - incoming_link_count - - bookmark_count - - score - - reads - - post_type - - sort_order - - last_editor_id - - hidden - - hidden_reason_id - - notify_moderators_count - - spam_count - - illegal_count - - inappropriate_count - - last_version_at - - user_deleted - - reply_to_user_id - - percent_rank - - notify_user_count - - like_score - - deleted_by_id - - edit_reason - - word_count - - version - - cook_method - - wiki - - baked_at - - baked_version - - hidden_at - - self_edits - - reply_quoted - - via_email - - raw_email - - public_version - - action_code - - locked_by_id - - image_upload_id - - outbound_message_id - - qa_vote_count # TODO: added from plugin, maybe skip these automatically for core schema? - extend: - - name: reply_to_post_id # NOTE: should this be text?? - - name: original_raw - - name: upload_ids - type: json_text - - name: post_number - type: integer - - name: old_relative_url - - name: accepted_answer - type: boolean - - name: small_action - - name: whisper - type: boolean - - name: placeholders - type: json_text - indexes: - - name: posts_by_topic_post_number - columns: [topic_id, post_number] - uploads: - ignore: - - original_filename - - filesize - - width - - height - - url - - created_at - - sha1 - - origin - - retain_hours - - extension - - thumbnail_width - - thumbnail_height - - etag - - secure - - access_control_post_id - - original_sha1 - - animated - - verification_status - - security_last_changed_at - - security_last_changed_reason - - dominant_color - extend: - - name: filename - is_null: false - - name: relative_path - - name: type - - name: data - type: blob - groups: - include: - - id - - name - - full_name - - visibility_level - - members_visibility_level - - mentionable_level - - messageable_level - extend: - - name: description - group_members: - virtual: true - primary_key: [group_id, user_id] - extend: - - name: group_id - type: integer - - name: user_id - type: integer - - name: owner - type: boolean - likes: - virtual: true - primary_key: [user_id, post_id] - extend: - - name: post_id - type: integer - is_null: false - - name: user_id - type: integer - is_null: false - - name: created_at - type: datetime - is_null: false - # TODO: Pending default values & auto incrementing id column - user_fields: - ignore: - - created_at - - external_name - - external_type - extend: - - name: options - type: json_text - muted_users: - primary_key: [user_id, muted_user_id] - ignore: - - id - - created_at - # NOTE: Perhaps use core's user_field_options instead? - user_field_values: - virtual: true - extend: - - name: user_id - type: integer - is_null: false - - name: field_id - type: integer - is_null: false - - name: is_multiselect_field - type: boolean - is_null: false - - name: value - indexes: - - name: user_field_values_multiselect - columns: [user_id, field_id, value] - unique: true - condition: WHERE is_multiselect_field = TRUE - - name: user_field_values_not_multiselect - columns: [user_id, field_id] - unique: true - condition: WHERE is_multiselect_field = FALSE - tags: - include: - - id - - name - extend: - - name: tag_group_id - type: integer - tag_groups: - include: - - id - - name - topic_tags: - primary_key: [topic_id, tag_id] - ignore: - - id - - created_at - tag_users: - primary_key: [tag_id, user_id] - ignore: - - id - - created_at - badges: - ignore: - - grant_count - - allow_title - - icon - - listable - - target_posts - - enabled - - auto_revoke - - trigger - - show_posts - - system - - image - - badge_grouping_id - extend: - - name: bage_group - user_badges: - include: - - user_id - - badge_id - - granted_at - topic_users: - primary_key: [user_id, topic_id] - ignore: - - id - - posted - - cleared_pinned_at - - last_emailed_post_number - - liked - - bookmarked - - last_posted_at - permalink_normalizations: - virtual: true - primary_key: normalization - extend: - - name: normalization - is_null: false - site_settings: - include: - - name - - value - extend: - - name: action - category_custom_fields: - primary_key: [category_id, name] - ignore: - - id - - created_at - post_custom_fields: - primary_key: [post_id, name] - ignore: - - id - - created_at - polls: {} - poll_options: - ignore: - - digest - - html - - anonymous_votes - extend: - - name: poll_id - is_null: false - - name: text - is_null: false - - name: position - type: integer - - name: created_at - is_null: true - poll_votes: - primary_key: [poll_option_id, user_id] - ignore: [poll_id] - extend: - - name: created_at - is_null: true - - name: poll_option_id - is_null: false - - name: user_id - is_null: false -## Schema-wide column configuration options. These options apply to all tables. -## See table specific column configuration options above. -## -## Available Options: -## ignore: List of core/plugin table columns to ignore and exclude from intermediate schema. -columns: - ignore: - - updated_at \ No newline at end of file diff --git a/migrations/spec/lib/database/intermediate_db/upload_spec.rb b/migrations/spec/lib/database/intermediate_db/upload_spec.rb new file mode 100644 index 00000000000..518ae204cd7 --- /dev/null +++ b/migrations/spec/lib/database/intermediate_db/upload_spec.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Database::IntermediateDB::Upload do + it_behaves_like "a database entity" +end diff --git a/migrations/spec/lib/database/intermediate_db/user_suspension_spec.rb b/migrations/spec/lib/database/intermediate_db/user_suspension_spec.rb new file mode 100644 index 00000000000..8c00d3c486a --- /dev/null +++ b/migrations/spec/lib/database/intermediate_db/user_suspension_spec.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +RSpec.describe ::Migrations::Database::IntermediateDB::UserSuspension do + it_behaves_like "a database entity" +end