github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-05-29 11:48:08 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

REFACTOR: Code generator for migrations IntemerdiateDB

Browse Source 
* Splits the existing script into multiple classes
* Adds command for generating IntermediateDB schema (`migrations/bin/cli schema generate`)
* Changes the syntax of the IntermediateDB schema config
* Adds validation for the schema config
* It uses YAML schema aka JSON schema to validate the config file
* It generates the SQL schema file and Ruby classes for storing data in the IntermediateDB
This commit is contained in:
Gerhard Schlager
2025-04-07 17:12:18 +02:00
committed by Gerhard Schlager
parent 71a90dcba2
commit 17ba19c7ae
33 changed files with 2359 additions and 864 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
migrations/lib/database/intermediate_db/user.rb Normal file
View File

@ -0,0 +1,105 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
# update the "config/intermediate_db.yml" configuration file and then run
# `bin/cli schema generate` to regenerate this file.
module Migrations::Database::IntermediateDB
module User
SQL = <<~SQL
INSERT INTO users (
original_id,
active,
admin,
approved,
approved_at,
approved_by_id,
created_at,
date_of_birth,
first_seen_at,
flair_group_id,
group_locked_trust_level,
ip_address,
last_seen_at,
locale,
manual_locked_trust_level,
moderator,
name,
original_username,
primary_group_id,
registration_ip_address,
silenced_till,
staged,
title,
trust_level,
uploaded_avatar_id,
username,
views
)
VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
SQL
def self.create(
original_id:,
active: nil,
admin: nil,
approved: nil,
approved_at: nil,
approved_by_id: nil,
created_at:,
date_of_birth: nil,
first_seen_at: nil,
flair_group_id: nil,
group_locked_trust_level: nil,
ip_address: nil,
last_seen_at: nil,
locale: nil,
manual_locked_trust_level: nil,
moderator: nil,
name: nil,
original_username: nil,
primary_group_id: nil,
registration_ip_address: nil,
silenced_till: nil,
staged: nil,
title: nil,
trust_level:,
uploaded_avatar_id: nil,
username:,
views: nil
)
::Migrations::Database::IntermediateDB.insert(
SQL,
original_id,
::Migrations::Database.format_boolean(active),
::Migrations::Database.format_boolean(admin),
::Migrations::Database.format_boolean(approved),
::Migrations::Database.format_datetime(approved_at),
approved_by_id,
::Migrations::Database.format_datetime(created_at),
::Migrations::Database.format_date(date_of_birth),
::Migrations::Database.format_datetime(first_seen_at),
flair_group_id,
group_locked_trust_level,
::Migrations::Database.format_ip_address(ip_address),
::Migrations::Database.format_datetime(last_seen_at),
locale,
manual_locked_trust_level,
::Migrations::Database.format_boolean(moderator),
name,
original_username,
primary_group_id,
::Migrations::Database.format_ip_address(registration_ip_address),
::Migrations::Database.format_datetime(silenced_till),
::Migrations::Database.format_boolean(staged),
title,
trust_level,
uploaded_avatar_id,
username,
views,
)
end
end
end

31
migrations/lib/database/intermediate_db/user_email.rb Normal file
View File

@ -0,0 +1,31 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
# update the "config/intermediate_db.yml" configuration file and then run
# `bin/cli schema generate` to regenerate this file.
module Migrations::Database::IntermediateDB
module UserEmail
SQL = <<~SQL
INSERT INTO user_emails (
email,
created_at,
"primary",
user_id
)
VALUES (
?, ?, ?, ?
)
SQL
def self.create(email:, created_at:, primary: nil, user_id:)
::Migrations::Database::IntermediateDB.insert(
SQL,
email,
::Migrations::Database.format_datetime(created_at),
::Migrations::Database.format_boolean(primary),
user_id,
)
end
end
end

199
migrations/lib/database/intermediate_db/user_option.rb Normal file
View File

@ -0,0 +1,199 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
# update the "config/intermediate_db.yml" configuration file and then run
# `bin/cli schema generate` to regenerate this file.
module Migrations::Database::IntermediateDB
module UserOption
SQL = <<~SQL
INSERT INTO user_options (
user_id,
allow_private_messages,
auto_track_topics_after_msecs,
automatically_unpin_topics,
bookmark_auto_delete_preference,
chat_email_frequency,
chat_enabled,
chat_header_indicator_preference,
chat_quick_reaction_type,
chat_quick_reactions_custom,
chat_send_shortcut,
chat_separate_sidebar_mode,
chat_sound,
color_scheme_id,
dark_scheme_id,
default_calendar,
digest_after_minutes,
dismissed_channel_retention_reminder,
dismissed_dm_retention_reminder,
dynamic_favicon,
email_digests,
email_in_reply_to,
email_level,
email_messages_level,
email_previous_replies,
enable_allowed_pm_users,
enable_defer,
enable_experimental_sidebar,
enable_quoting,
enable_smart_lists,
external_links_in_new_tab,
hide_presence,
hide_profile,
hide_profile_and_presence,
homepage_id,
ignore_channel_wide_mention,
include_tl0_in_digests,
last_redirected_to_top_at,
like_notification_frequency,
mailing_list_mode,
mailing_list_mode_frequency,
new_topic_duration_minutes,
notification_level_when_replying,
oldest_search_log_date,
only_chat_push_notifications,
seen_popups,
show_thread_title_prompts,
sidebar_link_to_filtered_list,
sidebar_show_count_of_new_items,
skip_new_user_tips,
text_size_key,
text_size_seq,
theme_ids,
theme_key_seq,
timezone,
title_count_mode_key,
topics_unread_when_closed,
watched_precedence_over_muted
)
VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
SQL
def self.create(
user_id:,
allow_private_messages: nil,
auto_track_topics_after_msecs: nil,
automatically_unpin_topics: nil,
bookmark_auto_delete_preference: nil,
chat_email_frequency: nil,
chat_enabled: nil,
chat_header_indicator_preference: nil,
chat_quick_reaction_type: nil,
chat_quick_reactions_custom: nil,
chat_send_shortcut: nil,
chat_separate_sidebar_mode: nil,
chat_sound: nil,
color_scheme_id: nil,
dark_scheme_id: nil,
default_calendar: nil,
digest_after_minutes: nil,
dismissed_channel_retention_reminder: nil,
dismissed_dm_retention_reminder: nil,
dynamic_favicon: nil,
email_digests: nil,
email_in_reply_to: nil,
email_level: nil,
email_messages_level: nil,
email_previous_replies: nil,
enable_allowed_pm_users: nil,
enable_defer: nil,
enable_experimental_sidebar: nil,
enable_quoting: nil,
enable_smart_lists: nil,
external_links_in_new_tab: nil,
hide_presence: nil,
hide_profile: nil,
hide_profile_and_presence: nil,
homepage_id: nil,
ignore_channel_wide_mention: nil,
include_tl0_in_digests: nil,
last_redirected_to_top_at: nil,
like_notification_frequency: nil,
mailing_list_mode: nil,
mailing_list_mode_frequency: nil,
new_topic_duration_minutes: nil,
notification_level_when_replying: nil,
oldest_search_log_date: nil,
only_chat_push_notifications: nil,
seen_popups: nil,
show_thread_title_prompts: nil,
sidebar_link_to_filtered_list: nil,
sidebar_show_count_of_new_items: nil,
skip_new_user_tips: nil,
text_size_key: nil,
text_size_seq: nil,
theme_ids: nil,
theme_key_seq: nil,
timezone: nil,
title_count_mode_key: nil,
topics_unread_when_closed: nil,
watched_precedence_over_muted: nil
)
::Migrations::Database::IntermediateDB.insert(
SQL,
user_id,
::Migrations::Database.format_boolean(allow_private_messages),
auto_track_topics_after_msecs,
::Migrations::Database.format_boolean(automatically_unpin_topics),
bookmark_auto_delete_preference,
chat_email_frequency,
::Migrations::Database.format_boolean(chat_enabled),
chat_header_indicator_preference,
chat_quick_reaction_type,
chat_quick_reactions_custom,
chat_send_shortcut,
chat_separate_sidebar_mode,
chat_sound,
color_scheme_id,
dark_scheme_id,
default_calendar,
digest_after_minutes,
::Migrations::Database.format_boolean(dismissed_channel_retention_reminder),
::Migrations::Database.format_boolean(dismissed_dm_retention_reminder),
::Migrations::Database.format_boolean(dynamic_favicon),
::Migrations::Database.format_boolean(email_digests),
::Migrations::Database.format_boolean(email_in_reply_to),
email_level,
email_messages_level,
email_previous_replies,
::Migrations::Database.format_boolean(enable_allowed_pm_users),
::Migrations::Database.format_boolean(enable_defer),
::Migrations::Database.format_boolean(enable_experimental_sidebar),
::Migrations::Database.format_boolean(enable_quoting),
::Migrations::Database.format_boolean(enable_smart_lists),
::Migrations::Database.format_boolean(external_links_in_new_tab),
::Migrations::Database.format_boolean(hide_presence),
::Migrations::Database.format_boolean(hide_profile),
::Migrations::Database.format_boolean(hide_profile_and_presence),
homepage_id,
::Migrations::Database.format_boolean(ignore_channel_wide_mention),
::Migrations::Database.format_boolean(include_tl0_in_digests),
::Migrations::Database.format_datetime(last_redirected_to_top_at),
like_notification_frequency,
::Migrations::Database.format_boolean(mailing_list_mode),
mailing_list_mode_frequency,
new_topic_duration_minutes,
notification_level_when_replying,
::Migrations::Database.format_datetime(oldest_search_log_date),
::Migrations::Database.format_boolean(only_chat_push_notifications),
seen_popups,
::Migrations::Database.format_boolean(show_thread_title_prompts),
::Migrations::Database.format_boolean(sidebar_link_to_filtered_list),
::Migrations::Database.format_boolean(sidebar_show_count_of_new_items),
::Migrations::Database.format_boolean(skip_new_user_tips),
text_size_key,
text_size_seq,
theme_ids,
theme_key_seq,
timezone,
title_count_mode_key,
::Migrations::Database.format_boolean(topics_unread_when_closed),
::Migrations::Database.format_boolean(watched_precedence_over_muted),
)
end
end
end

29
migrations/lib/database/intermediate_db/user_suspension.rb Normal file
View File

@ -0,0 +1,29 @@
# frozen_string_literal: true
module Migrations::Database::IntermediateDB
module UserSuspension
SQL = <<~SQL
INSERT INTO user_suspensions (
user_id,
suspended_at,
suspended_till,
suspended_by_id,
reason
)
VALUES (
?, ?, ?, ?, ?
)
SQL
def self.create(user_id:, suspended_at:, suspended_till: nil, suspended_by_id: nil, reason: nil)
::Migrations::Database::IntermediateDB.insert(
SQL,
user_id,
::Migrations::Database.format_datetime(suspended_at),
::Migrations::Database.format_datetime(suspended_till),
suspended_by_id,
reason,
)
end
end
end

175
migrations/lib/database/schema.rb Normal file
View File

@ -0,0 +1,175 @@
# frozen_string_literal: true
module Migrations::Database
module Schema
Table =
Data.define(:name, :columns, :indexes, :primary_key_column_names) do
def sorted_columns
columns.sort_by { |c| [c.is_primary_key ? 0 : 1, c.name] }
end
end
Column = Data.define(:name, :datatype, :nullable, :max_length, :is_primary_key)
Index = Data.define(:name, :column_names, :unique, :condition)
class ConfigError < StandardError
end
SQLITE_KEYWORDS = %w[
abort
action
add
after
all
alter
always
analyze
and
as
asc
attach
autoincrement
before
begin
between
by
cascade
case
cast
check
collate
column
commit
conflict
constraint
create
cross
current
current_date
current_time
current_timestamp
database
default
deferrable
deferred
delete
desc
detach
distinct
do
drop
each
else
end
escape
except
exclude
exclusive
exists
explain
fail
filter
first
following
for
foreign
from
full
generated
glob
group
groups
having
if
ignore
immediate
in
index
indexed
initially
inner
insert
instead
intersect
into
is
isnull
join
key
last
left
like
limit
match
materialized
natural
no
not
nothing
notnull
null
nulls
of
offset
on
or
order
others
outer
over
partition
plan
pragma
preceding
primary
query
raise
range
recursive
references
regexp
reindex
release
rename
replace
restrict
returning
right
rollback
row
rows
savepoint
select
set
table
temp
temporary
then
ties
to
transaction
trigger
unbounded
union
unique
update
using
vacuum
values
view
virtual
when
where
window
with
without
]
def self.escape_identifier(identifier)
if SQLITE_KEYWORDS.include?(identifier)
%Q("#{identifier}")
else
identifier
end
end
end
end

46
migrations/lib/database/schema/config_validator.rb Normal file
View File

@ -0,0 +1,46 @@
# frozen_string_literal: true
module Migrations::Database::Schema
class ConfigValidator
attr_reader :errors
def initialize
@errors = []
end
def validate(config)
@errors.clear
validate_with_json_schema(config)
return self if has_errors?
validate_output_config(config)
validate_schema_config(config)
validate_plugins(config)
self
end
def has_errors?
@errors.any?
end
private
def validate_with_json_schema(config)
Validation::JsonSchemaValidator.new(config, @errors).validate
end
def validate_output_config(config)
Validation::OutputConfigValidator.new(config, @errors).validate
end
def validate_schema_config(config)
Validation::SchemaConfigValidator.new(config, @errors).validate
end
def validate_plugins(config)
Validation::PluginConfigValidator.new(config, @errors).validate
end
end
end

61
migrations/lib/database/schema/global_config.rb Normal file
View File

@ -0,0 +1,61 @@
# frozen_string_literal: true
module Migrations::Database::Schema
class GlobalConfig
attr_reader :excluded_column_names, :modified_columns
def initialize(schema_config)
@schema_config = schema_config
@excluded_table_names = load_globally_excluded_table_names.freeze
@excluded_column_names = load_globally_excluded_column_names.freeze
@modified_columns = load_globally_modified_columns.freeze
end
def excluded_table_name?(table_name)
@excluded_table_names.include?(table_name)
end
def modified_name(column_name)
if (modified_column = find_modified_column(column_name))
modified_column[:rename_to]
end
end
def modified_datatype(column_name)
if (modified_column = find_modified_column(column_name))
modified_column[:datatype]
end
end
private
def find_modified_column(column_name)
@modified_columns.find { |column| column[:name] == column_name } ||
@modified_columns.find { |column| column[:name_regex]&.match?(column_name) }
end
def load_globally_excluded_table_names
table_names = @schema_config.dig(:global, :tables, :exclude)
table_names.presence&.to_set || Set.new
end
def load_globally_excluded_column_names
column_names = @schema_config.dig(:global, :columns, :exclude)
column_names.presence || []
end
def load_globally_modified_columns
modified_columns = @schema_config.dig(:global, :columns, :modify)
return {} if modified_columns.blank?
modified_columns.map do |column|
if column[:name_regex]
column[:name_regex_original] = column[:name_regex]
column[:name_regex] = Regexp.new(column[:name_regex])
end
column[:datatype] = column[:datatype]&.to_sym
column
end
end
end
end

116
migrations/lib/database/schema/loader.rb Normal file
View File

@ -0,0 +1,116 @@
# frozen_string_literal: true
module Migrations::Database::Schema
class Loader
def initialize(schema_config)
@schema_config = schema_config
@global = GlobalConfig.new(@schema_config)
end
def load_schema
@db = ActiveRecord::Base.lease_connection
schema = []
existing_table_names = @db.tables.to_set
@schema_config[:tables].sort.each do |table_name, config|
table_name = table_name.to_s
if config[:copy_of].present?
table_alias = table_name
table_name = config[:copy_of]
else
next if @global.excluded_table_name?(table_name)
end
if existing_table_names.include?(table_name)
schema << table(table_name, config, table_alias)
end
end
@db = nil
ActiveRecord::Base.release_connection
schema
end
private
def table(table_name, config, table_alias = nil)
primary_key_column_names =
config[:primary_key_column_names].presence || @db.primary_keys(table_name)
columns =
filtered_columns_of(table_name, config).map do |column|
Column.new(
name: name_for(column),
datatype: datatype_for(column),
nullable: column.null || column.default,
max_length: column.type == :text ? column.limit : nil,
is_primary_key: primary_key_column_names.include?(column.name),
)
end + added_columns(config, primary_key_column_names)
Table.new(table_alias || table_name, columns, indexes(config), primary_key_column_names)
end
def filtered_columns_of(table_name, config)
columns_by_name = @db.columns(table_name).index_by(&:name)
columns_by_name.except!(*@global.excluded_column_names)
if (included_columns = config.dig(:columns, :include))
columns_by_name.slice!(*included_columns)
elsif (excluded_columns = config.dig(:columns, :exclude))
columns_by_name.except!(*excluded_columns)
end
columns_by_name.values
end
def added_columns(config, primary_key_column_names)
columns = config.dig(:columns, :add) || []
columns.map do |column|
datatype = column[:datatype].to_sym
Column.new(
name: column[:name],
datatype:,
nullable: column.fetch(:nullable, true),
max_length: datatype == :text ? column[:max_length] : nil,
is_primary_key: primary_key_column_names.include?(column[:name]),
)
end
end
def name_for(column)
@global.modified_name(column.name) || column.name
end
def datatype_for(column)
datatype = @global.modified_datatype(column.name) || column.type
case datatype
when :binary
:blob
when :string, :enum, :uuid
:text
when :jsonb
:json
when :boolean, :date, :datetime, :float, :inet, :integer, :numeric, :json, :text
datatype
else
raise "Unknown datatype: #{datatype}"
end
end
def indexes(config)
config[:indexes]&.map do |index|
Index.new(
name: index[:name],
column_names: Array.wrap(index[:columns]),
unique: index.fetch(:unique, false),
condition: index[:condition],
)
end
end
end
end

139
migrations/lib/database/schema/model_writer.rb Normal file
View File

@ -0,0 +1,139 @@
# frozen_string_literal: true
require "rake"
require "syntax_tree/rake_tasks"
module Migrations::Database::Schema
class ModelWriter
def initialize(namespace, header)
@namespace = namespace
@header = header.gsub(/^/, "# ")
end
def self.filename_for(table)
"#{table.name.singularize}.rb"
end
def self.format_files(path)
glob_pattern = File.join(path, "**/*.rb")
system(
"bundle",
"exec",
"stree",
"write",
glob_pattern,
exception: true,
out: File::NULL,
err: File::NULL,
)
rescue StandardError
raise "Failed to run `bundle exec stree write '#{glob_pattern}'`"
end
def output_table(table, output_stream)
columns = table.sorted_columns
output_stream.puts "# frozen_string_literal: true"
output_stream.puts
output_stream.puts @header
output_stream.puts
output_stream.puts "module #{@namespace}"
output_stream.puts " module #{to_singular_classname(table.name)}"
output_stream.puts " SQL = <<~SQL"
output_stream.puts " INSERT INTO #{escape_identifier(table.name)} ("
output_stream.puts column_names(columns)
output_stream.puts " )"
output_stream.puts " VALUES ("
output_stream.puts value_placeholders(columns)
output_stream.puts " )"
output_stream.puts " SQL"
output_stream.puts
output_stream.puts " def self.create("
output_stream.puts method_parameters(columns)
output_stream.puts " )"
output_stream.puts " ::Migrations::Database::IntermediateDB.insert("
output_stream.puts " SQL,"
output_stream.puts insertion_arguments(columns)
output_stream.puts " )"
output_stream.puts " end"
output_stream.puts " end"
output_stream.puts "end"
end
private
def to_singular_classname(snake_case_string)
snake_case_string.singularize.camelize
end
def column_names(columns)
columns.map { |c| " #{escape_identifier(c.name)}" }.join(",\n")
end
def value_placeholders(columns)
indentation = " "
max_length = 100 - indentation.length
placeholder = "?, "
placeholder_count = columns.size
current_length = 0
placeholders = indentation.dup
(1..placeholder_count).each do |index|
placeholder = "?" if index == placeholder_count
if current_length + placeholder.length > max_length
placeholders.rstrip!
placeholders << "\n" << indentation
current_length = 0
end
placeholders << placeholder
current_length += placeholder.length
end
placeholders
end
def method_parameters(columns)
columns
.map do |c|
default_value = !c.is_primary_key && c.nullable ? " nil" : ""
" #{c.name}:#{default_value}"
end
.join(",\n")
end
def insertion_arguments(columns)
columns
.map do |c|
argument =
case c.datatype
when :datetime
"::Migrations::Database.format_datetime(#{c.name})"
when :date
"::Migrations::Database.format_date(#{c.name})"
when :boolean
"::Migrations::Database.format_boolean(#{c.name})"
when :inet
"::Migrations::Database.format_ip_address(#{c.name})"
when :blob
"::Migrations::Database.to_blob(#{c.name})"
when :json
"::Migrations::Database.to_json(#{c.name})"
when :float, :integer, :numeric, :text
c.name
else
raise "Unknown dataype: #{type}"
end
" #{argument},"
end
.join("\n")
end
def escape_identifier(identifier)
::Migrations::Database::Schema.escape_identifier(identifier)
end
end
end

104
migrations/lib/database/schema/table_writer.rb Normal file
View File

@ -0,0 +1,104 @@
# frozen_string_literal: true
module Migrations::Database::Schema
class TableWriter
def initialize(output_stream)
@output = output_stream
end
def output_file_header(header)
@output.puts header.gsub(/^/, "-- ")
@output.puts
end
def output_table(table)
output_create_table_statement(table)
output_columns(table)
output_primary_key(table)
output_indexes(table)
@output.puts ""
end
private
def output_create_table_statement(table)
@output.puts "CREATE TABLE #{escape_identifier(table.name)}"
@output.puts "("
end
def output_columns(table)
column_definitions = create_column_definitions(table)
@output.puts column_definitions.join(",\n")
end
def output_primary_key(table)
if table.primary_key_column_names.size > 1
pk_definition =
table.primary_key_column_names.map { |name| escape_identifier(name) }.join(", ")
@output.puts " PRIMARY KEY (#{pk_definition})"
end
@output.puts ");"
end
def create_column_definitions(table)
columns = table.sorted_columns
has_composite_primary_key = table.primary_key_column_names.size > 1
max_column_name_length = columns.map { |c| escape_identifier(c.name).length }.max
max_datatype_length = columns.map { |c| convert_datatype(c.datatype).length }.max
columns.map do |c|
definition = [
escape_identifier(c.name).ljust(max_column_name_length),
convert_datatype(c.datatype).ljust(max_datatype_length),
]
if c.is_primary_key && !has_composite_primary_key
definition << "NOT NULL" if c.datatype != :integer
definition << "PRIMARY KEY"
else
definition << "NOT NULL" unless c.nullable
end
definition = definition.join(" ")
definition.strip!
" #{definition}"
end
end
def convert_datatype(type)
case type
when :blob, :boolean, :date, :datetime, :float, :integer, :numeric, :text
type.to_s.upcase
when :inet
"INET_TEXT"
when :json
"JSON_TEXT"
else
raise "Unknown datatype: #{type}"
end
end
def escape_identifier(identifier)
::Migrations::Database::Schema.escape_identifier(identifier)
end
def output_indexes(table)
return unless table.indexes
table.indexes.each do |index|
index_name = escape_identifier(index.name)
table_name = escape_identifier(table.name)
column_names = index.column_names.map { |name| escape_identifier(name) }
@output.puts ""
@output.print "CREATE "
@output.print "UNIQUE " if index.unique
@output.print "INDEX #{index_name} ON #{table_name} (#{column_names.join(", ")})"
@output.print " #{index.condition}" if index.condition.present?
@output.puts ";"
end
end
end
end

18
migrations/lib/database/schema/validation/base_validator.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class BaseValidator
def initialize(config, errors, db)
@config = config
@schema_config ||= @config[:schema]
@errors = errors
@db = db
end
private
def sort_and_join(values)
values.sort.join(", ")
end
end
end

115
migrations/lib/database/schema/validation/columns_validator.rb Normal file
View File

@ -0,0 +1,115 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class ColumnsValidator < BaseValidator
def validate(table_name, copy_of_table_name = nil)
@table_name = table_name
columns = @schema_config[:tables][table_name.to_sym][:columns] || {}
@existing_column_names = find_existing_column_names(table_name, copy_of_table_name)
@added_column_names = columns[:add]&.map { |column| column[:name] } || []
@included_column_names = columns[:include] || []
@excluded_column_names = columns[:exclude] || []
@modified_column_names = columns[:modify]&.map { |column| column[:name] } || []
@global = ::Migrations::Database::Schema::GlobalConfig.new(@schema_config)
validated_added_columns
validate_included_columns
validate_excluded_columns
validate_modified_columns
validate_any_columns_configured
end
private
def find_existing_column_names(table_name, copy_of_table_name)
table_name = copy_of_table_name if copy_of_table_name.present?
@db.columns(table_name).map { |c| c.name }
end
def validated_added_columns
if (column_names = @existing_column_names & @added_column_names).any?
@errors << I18n.t(
"schema.validator.tables.added_columns_exist",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
end
def validate_included_columns
if (column_names = @included_column_names - @existing_column_names).any?
@errors << I18n.t(
"schema.validator.tables.included_columns_missing",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
end
def validate_excluded_columns
if (column_names = @excluded_column_names - @existing_column_names).any?
@errors << I18n.t(
"schema.validator.tables.excluded_columns_missing",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
end
def validate_modified_columns
if (column_names = @modified_column_names - @existing_column_names).any?
@errors << I18n.t(
"schema.validator.tables.modified_columns_missing",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
if (column_names = @modified_column_names & @included_column_names).any?
@errors << I18n.t(
"schema.validator.tables.modified_columns_included",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
if (column_names = @modified_column_names & @excluded_column_names).any?
@errors << I18n.t(
"schema.validator.tables.modified_columns_excluded",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
if (column_names = @modified_column_names & @global.excluded_column_names).any?
@errors << I18n.t(
"schema.validator.tables.modified_columns_globally_excluded",
table_name: @table_name,
column_names: sort_and_join(column_names),
)
end
end
def configured_column_names
if @included_column_names.any?
included_column_names = @included_column_names
modified_column_names = @modified_column_names
else
included_column_names = @existing_column_names - @excluded_column_names
modified_column_names = included_column_names & @modified_column_names
end
column_names = (included_column_names + modified_column_names).uniq & @existing_column_names
column_names - @global.excluded_column_names + @added_column_names
end
def validate_any_columns_configured
column_names = configured_column_names
if column_names.empty?
@errors << I18n.t("schema.validator.tables.no_columns_configured", table_name: @table_name)
end
end
end
end

73
migrations/lib/database/schema/validation/globally_configured_columns_validator.rb Normal file
View File

@ -0,0 +1,73 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class GloballyConfiguredColumnsValidator < BaseValidator
def initialize(config, errors, db)
super
@global = ::Migrations::Database::Schema::GlobalConfig.new(@schema_config)
end
def validate
all_column_names = calculate_all_column_names
validate_excluded_column_names(all_column_names)
validate_modified_columns(all_column_names)
end
private
def calculate_all_column_names
existing_table_names = @db.tables
configured_table_names =
@schema_config[:tables].map do |table_name, table_config|
table_config[:copy_of] || table_name.to_s
end
globally_excluded_table_names = @schema_config.dig(:global, :tables, :exclude) || []
excluded_table_names = @schema_config.dig(:global, :tables, :exclude) || []
all_table_names = existing_table_names - globally_excluded_table_names - excluded_table_names
all_table_names = all_table_names.uniq & configured_table_names
all_table_names.flat_map { |table_name| @db.columns(table_name).map(&:name) }.uniq
end
def validate_excluded_column_names(all_column_names)
globally_excluded_column_names = @global.excluded_column_names
excluded_missing_column_names = globally_excluded_column_names - all_column_names
if excluded_missing_column_names.any?
@errors << I18n.t(
"schema.validator.global.excluded_columns_missing",
column_names: sort_and_join(excluded_missing_column_names),
)
end
end
def validate_modified_columns(all_column_names)
globally_modified_columns = @global.modified_columns
excluded_missing_columns =
globally_modified_columns.reject do |column|
if column[:name]
all_column_names.include?(column[:name])
elsif column[:name_regex]
all_column_names.any? { |column_name| column[:name_regex]&.match?(column_name) }
else
false
end
end
if excluded_missing_columns.any?
excluded_missing_column_names =
excluded_missing_columns.map { |column| column[:name_regex_original] || column[:name] }
@errors << I18n.t(
"schema.validator.global.modified_columns_missing",
column_names: sort_and_join(excluded_missing_column_names),
)
end
end
end
end

20
migrations/lib/database/schema/validation/globally_excluded_tables_validator.rb Normal file
View File

@ -0,0 +1,20 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class GloballyExcludedTablesValidator < BaseValidator
def validate
excluded_table_names = @schema_config.dig(:global, :tables, :exclude)
return if excluded_table_names.blank?
existing_table_names = @db.tables
missing_table_names = excluded_table_names - existing_table_names
if missing_table_names.any?
@errors << I18n.t(
"schema.validator.global.excluded_tables_missing",
table_names: sort_and_join(missing_table_names),
)
end
end
end
end

33
migrations/lib/database/schema/validation/json_schema_validator.rb Normal file
View File

@ -0,0 +1,33 @@
# frozen_string_literal: true
require "json_schemer"
module Migrations::Database::Schema::Validation
class JsonSchemaValidator
def initialize(config, errors)
@config = config
@errors = errors
end
def validate
schema = load_json_schema
schemer = ::JSONSchemer.schema(schema)
response = schemer.validate(@config)
response.each { |r| @errors << transform_json_schema_errors(r.fetch("error")) }
end
private
def load_json_schema
schema_path = File.join(::Migrations.root_path, "config", "json_schemas", "db_schema.json")
JSON.load_file(schema_path)
end
def transform_json_schema_errors(error_message)
error_message.gsub!(/value at (`.+?`) matches `not` schema/) do
I18n.t("schema.validator.include_exclude_not_allowed", path: $1)
end
error_message
end
end
end

46
migrations/lib/database/schema/validation/output_config_validator.rb Normal file
View File

@ -0,0 +1,46 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class OutputConfigValidator < BaseValidator
def initialize(config, errors)
super(config, errors, nil)
@output_config = config[:output]
end
def validate
validate_schema_file_directory
validate_models_directory
validate_models_namespace
end
private
def validate_schema_file_directory
schema_file_path = File.dirname(@output_config[:schema_file])
schema_file_path = File.expand_path(schema_file_path, ::Migrations.root_path)
if !Dir.exist?(schema_file_path)
@errors << I18n.t("schema.validator.output.schema_file_directory_not_found")
end
end
def validate_models_directory
models_directory = File.expand_path(@output_config[:models_directory], ::Migrations.root_path)
if !Dir.exist?(models_directory)
@errors << I18n.t("schema.validator.output.models_directory_not_found")
end
end
def validate_models_namespace
existing_namespace =
begin
Object.const_get(@output_config[:models_namespace]).is_a?(Module)
rescue NameError
false
end
@errors << I18n.t("schema.validator.output.models_namespace_undefined") if !existing_namespace
end
end
end

28
migrations/lib/database/schema/validation/plugin_config_validator.rb Normal file
View File

@ -0,0 +1,28 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class PluginConfigValidator < BaseValidator
def initialize(config, errors)
super(config, errors, nil)
end
def validate
all_plugin_names = Discourse.plugins.map(&:name)
configured_plugin_names = @config[:plugins]
if (additional_plugins = all_plugin_names - configured_plugin_names).any?
@errors << I18n.t(
"schema.validator.plugins.additional_installed",
plugin_names: sort_and_join(additional_plugins),
)
end
if (missing_plugins = configured_plugin_names - all_plugin_names).any?
@errors << I18n.t(
"schema.validator.plugins.not_installed",
plugin_names: sort_and_join(missing_plugins),
)
end
end
end
end

18
migrations/lib/database/schema/validation/schema_config_validator.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class SchemaConfigValidator
def initialize(config, errors)
@config = config
@errors = errors
end
def validate
ActiveRecord::Base.with_connection do |db|
GloballyExcludedTablesValidator.new(@config, @errors, db).validate
GloballyConfiguredColumnsValidator.new(@config, @errors, db).validate
TablesValidator.new(@config, @errors, db).validate
end
end
end
end

70
migrations/lib/database/schema/validation/tables_validator.rb Normal file
View File

@ -0,0 +1,70 @@
# frozen_string_literal: true
module Migrations::Database::Schema::Validation
class TablesValidator < BaseValidator
def initialize(config, errors, db)
super
@existing_table_names = @db.tables
@configured_tables = @schema_config[:tables]
@configured_table_names = @configured_tables.keys.map(&:to_s)
@excluded_table_names = @schema_config.dig(:global, :tables, :exclude) || []
end
def validate
validate_excluded_tables
validate_unconfigured_tables
validate_copied_tables
validate_columns
end
private
def validate_excluded_tables
table_names = @configured_table_names & @excluded_table_names
if table_names.any?
@errors << I18n.t(
"schema.validator.tables.excluded_tables_configured",
table_names: sort_and_join(table_names),
)
end
end
def validate_unconfigured_tables
table_names = @existing_table_names - @configured_table_names - @excluded_table_names
if table_names.any?
@errors << I18n.t(
"schema.validator.tables.not_configured",
table_names: sort_and_join(table_names),
)
end
end
def validate_copied_tables
@configured_tables.each do |_table_name, table_config|
next unless table_config[:copy_of]
if !@existing_table_names.include?(table_config[:copy_of])
@errors << I18n.t(
"schema.validator.tables.copy_table_not_found",
table_name: table_config[:copy_of],
)
end
end
end
def validate_columns
@configured_tables.each do |table_name, table_config|
validator = ColumnsValidator.new(@config, @errors, @db)
if table_config[:copy_of]
validator.validate(table_name.to_s, table_config[:copy_of])
else
validator.validate(table_name.to_s)
end
end
end
end
end