mirror of
https://github.com/discourse/discourse.git
synced 2025-05-29 01:31:35 +08:00
DEV: Refactor migrations-tooling
* Updates GitHub Action for migrations * Rubocop: Always `EnforcedShorthandSyntax` for hashes in the `migrations` directory * Automatically load all available converter steps * Enable YJIT at runtime, if available * Progressbar shows skipped records and other small improvements
This commit is contained in:

committed by
Gerhard Schlager

parent
7b5839ec44
commit
71a90dcba2
39
.github/workflows/migration-tests.yml
vendored
39
.github/workflows/migration-tests.yml
vendored
@ -23,21 +23,16 @@ permissions:
|
|||||||
jobs:
|
jobs:
|
||||||
tests:
|
tests:
|
||||||
if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'
|
if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'
|
||||||
name: Tests with Ruby ${{ matrix.ruby }}
|
name: Tests
|
||||||
runs-on: ${{ (github.repository != 'discourse/discourse' && 'ubuntu-latest') || 'debian-12' }}
|
runs-on: ${{ (github.repository_owner == 'discourse' && 'debian-12') || 'ubuntu-latest' }}
|
||||||
container: discourse/discourse_test:slim
|
container: discourse/discourse_test:release
|
||||||
timeout-minutes: 20
|
timeout-minutes: 20
|
||||||
|
|
||||||
env:
|
env:
|
||||||
RAILS_ENV: test
|
RAILS_ENV: test
|
||||||
PGUSER: discourse
|
PGUSER: discourse
|
||||||
PGPASSWORD: discourse
|
PGPASSWORD: discourse
|
||||||
|
LOAD_PLUGINS: 1
|
||||||
strategy:
|
|
||||||
fail-fast: false
|
|
||||||
|
|
||||||
matrix:
|
|
||||||
ruby: ["3.3"]
|
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Set working directory owner
|
- name: Set working directory owner
|
||||||
@ -62,23 +57,9 @@ jobs:
|
|||||||
sudo -E -u postgres script/start_test_db.rb
|
sudo -E -u postgres script/start_test_db.rb
|
||||||
sudo -u postgres psql -c "CREATE ROLE $PGUSER LOGIN SUPERUSER PASSWORD '$PGPASSWORD';"
|
sudo -u postgres psql -c "CREATE ROLE $PGUSER LOGIN SUPERUSER PASSWORD '$PGPASSWORD';"
|
||||||
|
|
||||||
- name: Container envs
|
- name: Symlink vendor/bundle from image
|
||||||
id: container-envs
|
|
||||||
run: |
|
run: |
|
||||||
echo "ruby_version=$RUBY_VERSION" >> $GITHUB_OUTPUT
|
ln -s /var/www/discourse/vendor/bundle vendor/bundle
|
||||||
echo "debian_release=$DEBIAN_RELEASE" >> $GITHUB_OUTPUT
|
|
||||||
shell: bash
|
|
||||||
|
|
||||||
- name: Bundler cache
|
|
||||||
uses: actions/cache@v4
|
|
||||||
with:
|
|
||||||
path: vendor/bundle
|
|
||||||
key: >-
|
|
||||||
${{ runner.os }}-
|
|
||||||
${{ steps.container-envs.outputs.ruby_version }}-
|
|
||||||
${{ steps.container-envs.outputs.debian_release }}-
|
|
||||||
${{ hashFiles('**/Gemfile.lock') }}-
|
|
||||||
migrations-tooling
|
|
||||||
|
|
||||||
- name: Setup gems
|
- name: Setup gems
|
||||||
run: |
|
run: |
|
||||||
@ -88,11 +69,14 @@ jobs:
|
|||||||
bundle config --local without development
|
bundle config --local without development
|
||||||
bundle config --local with migrations
|
bundle config --local with migrations
|
||||||
bundle install --jobs $(($(nproc) - 1))
|
bundle install --jobs $(($(nproc) - 1))
|
||||||
bundle clean
|
|
||||||
|
|
||||||
- name: pnpm install
|
- name: pnpm install
|
||||||
run: pnpm install --frozen-lockfile
|
run: pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Get CPU cores
|
||||||
|
id: cpu-info
|
||||||
|
run: echo "cpu-cores=$(nproc)" >> $GITHUB_OUTPUT
|
||||||
|
|
||||||
- name: Fetch app state cache
|
- name: Fetch app state cache
|
||||||
uses: actions/cache@v4
|
uses: actions/cache@v4
|
||||||
id: app-cache
|
id: app-cache
|
||||||
@ -100,7 +84,8 @@ jobs:
|
|||||||
path: tmp/app-cache
|
path: tmp/app-cache
|
||||||
key: >-
|
key: >-
|
||||||
${{ runner.os }}-
|
${{ runner.os }}-
|
||||||
${{ hashFiles('.github/workflows/tests.yml') }}-
|
${{ steps.cpu-info.outputs.cpu-cores }}-
|
||||||
|
${{ hashFiles('.github/workflows/migration-tests.yml') }}-
|
||||||
${{ hashFiles('db/**/*', 'plugins/**/db/**/*') }}-
|
${{ hashFiles('db/**/*', 'plugins/**/db/**/*') }}-
|
||||||
${{ hashFiles('config/environments/test.rb') }}
|
${{ hashFiles('config/environments/test.rb') }}
|
||||||
|
|
||||||
|
4
migrations/.rubocop.yml
Normal file
4
migrations/.rubocop.yml
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
inherit_from: ../.rubocop.yml
|
||||||
|
|
||||||
|
Style/HashSyntax:
|
||||||
|
EnforcedShorthandSyntax: always
|
@ -1,7 +1,7 @@
|
|||||||
#!/usr/bin/env ruby
|
#!/usr/bin/env ruby
|
||||||
# frozen_string_literal: true
|
# frozen_string_literal: true
|
||||||
|
|
||||||
require_relative "../lib/migrations"
|
require_relative "../migrations"
|
||||||
|
|
||||||
require "colored2"
|
require "colored2"
|
||||||
require "thor"
|
require "thor"
|
||||||
@ -44,6 +44,12 @@ module Migrations
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
if defined?(RubyVM::YJIT)
|
||||||
|
RubyVM::YJIT.enable
|
||||||
|
else
|
||||||
|
warn "WARNING: Performance degraded: RubyVM::YJIT is not available".yellow
|
||||||
|
end
|
||||||
|
|
||||||
# rubocop:disable Discourse/NoChdir
|
# rubocop:disable Discourse/NoChdir
|
||||||
Dir.chdir(File.expand_path("../..", __dir__)) { ::Migrations::CLI::Application.start }
|
Dir.chdir(File.expand_path("../..", __dir__)) { ::Migrations::CLI::Application.start }
|
||||||
# rubocop:enable Discourse/NoChdir
|
# rubocop:enable Discourse/NoChdir
|
||||||
|
@ -1,5 +1,8 @@
|
|||||||
en:
|
en:
|
||||||
progressbar:
|
progressbar:
|
||||||
|
skips:
|
||||||
|
one: "%{count} skip"
|
||||||
|
other: "%{count} skips"
|
||||||
warnings:
|
warnings:
|
||||||
one: "%{count} warning"
|
one: "%{count} warning"
|
||||||
other: "%{count} warnings"
|
other: "%{count} warnings"
|
||||||
|
@ -2,21 +2,16 @@
|
|||||||
|
|
||||||
module Migrations
|
module Migrations
|
||||||
module DateHelper
|
module DateHelper
|
||||||
# based on code from https://gist.github.com/emmahsax/af285a4b71d8506a1625a3e591dc993b
|
def self.human_readable_time(seconds)
|
||||||
def self.human_readable_time(secs)
|
hours, remainder = seconds.divmod(3600)
|
||||||
return "< 1 second" if secs < 1
|
minutes, seconds = remainder.divmod(60)
|
||||||
|
format("%02d:%02d:%02d", hours, minutes, seconds)
|
||||||
[[60, :seconds], [60, :minutes], [24, :hours], [Float::INFINITY, :days]].map do |count, name|
|
|
||||||
next if secs <= 0
|
|
||||||
|
|
||||||
secs, number = secs.divmod(count)
|
|
||||||
unless number.to_i == 0
|
|
||||||
"#{number.to_i} #{number == 1 ? name.to_s.delete_suffix("s") : name}"
|
|
||||||
end
|
end
|
||||||
end
|
|
||||||
.compact
|
def self.track_time
|
||||||
.reverse
|
start_time = Time.now
|
||||||
.join(", ")
|
yield
|
||||||
|
Time.now - start_time
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
# frozen_string_literal: true
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require "colored2"
|
||||||
require "ruby-progressbar"
|
require "ruby-progressbar"
|
||||||
|
|
||||||
module Migrations
|
module Migrations
|
||||||
@ -7,9 +8,10 @@ module Migrations
|
|||||||
def initialize(max_progress: nil)
|
def initialize(max_progress: nil)
|
||||||
@max_progress = max_progress
|
@max_progress = max_progress
|
||||||
|
|
||||||
|
@skip_count = 0
|
||||||
@warning_count = 0
|
@warning_count = 0
|
||||||
@error_count = 0
|
@error_count = 0
|
||||||
@extra_information = ""
|
@extra_information = +""
|
||||||
|
|
||||||
@base_format = nil
|
@base_format = nil
|
||||||
@progressbar = nil
|
@progressbar = nil
|
||||||
@ -18,60 +20,58 @@ module Migrations
|
|||||||
def run
|
def run
|
||||||
raise "ProgressBar already started" if @progressbar
|
raise "ProgressBar already started" if @progressbar
|
||||||
|
|
||||||
format = setup_progressbar
|
format = calculate_format
|
||||||
|
setup_progressbar
|
||||||
|
|
||||||
yield self
|
yield self
|
||||||
|
|
||||||
finalize_progressbar(format)
|
finalize_progressbar(format)
|
||||||
|
|
||||||
nil
|
nil
|
||||||
end
|
end
|
||||||
|
|
||||||
def update(progress, warning_count, error_count)
|
def update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)
|
||||||
extra_information_changed = false
|
updated = false
|
||||||
|
|
||||||
|
if skip_count > 0
|
||||||
|
@skip_count += skip_count
|
||||||
|
updated = true
|
||||||
|
end
|
||||||
|
|
||||||
if warning_count > 0
|
if warning_count > 0
|
||||||
@warning_count += warning_count
|
@warning_count += warning_count
|
||||||
extra_information_changed = true
|
updated = true
|
||||||
end
|
end
|
||||||
|
|
||||||
if error_count > 0
|
if error_count > 0
|
||||||
@error_count += error_count
|
@error_count += error_count
|
||||||
extra_information_changed = true
|
updated = true
|
||||||
end
|
end
|
||||||
|
|
||||||
if extra_information_changed
|
update_format if updated
|
||||||
@extra_information = +""
|
|
||||||
|
|
||||||
if @warning_count > 0
|
if increment_by == 1
|
||||||
@extra_information << " | " <<
|
|
||||||
I18n.t("progressbar.warnings", count: @warning_count).yellow
|
|
||||||
end
|
|
||||||
|
|
||||||
if @error_count > 0
|
|
||||||
@extra_information << " | " << I18n.t("progressbar.errors", count: @error_count).red
|
|
||||||
end
|
|
||||||
|
|
||||||
@progressbar.format = "#{@base_format}#{@extra_information}"
|
|
||||||
end
|
|
||||||
|
|
||||||
if progress == 1
|
|
||||||
@progressbar.increment
|
@progressbar.increment
|
||||||
else
|
else
|
||||||
@progressbar.progress += progress
|
@progressbar.progress += increment_by
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def setup_progressbar
|
def calculate_format
|
||||||
format =
|
|
||||||
if @max_progress
|
if @max_progress
|
||||||
I18n.t("progressbar.processed.progress_with_max", current: "%c", max: "%C")
|
format = I18n.t("progressbar.processed.progress_with_max", current: "%c", max: "%C")
|
||||||
|
@base_format = " %a | %E | #{format}"
|
||||||
else
|
else
|
||||||
I18n.t("progressbar.processed.progress", current: "%c")
|
format = I18n.t("progressbar.processed.progress", current: "%c")
|
||||||
|
@base_format = " %a | #{format}"
|
||||||
end
|
end
|
||||||
|
|
||||||
@base_format = @max_progress ? " %a |%E | #{format}" : " %a | #{format}"
|
format
|
||||||
|
end
|
||||||
|
|
||||||
|
def setup_progressbar
|
||||||
@progressbar =
|
@progressbar =
|
||||||
::ProgressBar.create(
|
::ProgressBar.create(
|
||||||
total: @max_progress,
|
total: @max_progress,
|
||||||
@ -83,14 +83,23 @@ module Migrations
|
|||||||
format: @base_format,
|
format: @base_format,
|
||||||
throttle_rate: 0.5,
|
throttle_rate: 0.5,
|
||||||
)
|
)
|
||||||
|
end
|
||||||
|
|
||||||
format
|
def update_format
|
||||||
|
@extra_information.clear
|
||||||
|
|
||||||
|
messages = []
|
||||||
|
messages << I18n.t("progressbar.skips", count: @skip_count).cyan if @skip_count > 0
|
||||||
|
messages << I18n.t("progressbar.warnings", count: @warning_count).yellow if @warning_count > 0
|
||||||
|
messages << I18n.t("progressbar.errors", count: @error_count).red if @error_count > 0
|
||||||
|
|
||||||
|
@extra_information << " | #{messages.join(" | ")}" unless messages.empty?
|
||||||
|
@progressbar.format = "#{@base_format}#{@extra_information}"
|
||||||
end
|
end
|
||||||
|
|
||||||
def finalize_progressbar(format)
|
def finalize_progressbar(format)
|
||||||
print "\033[K" # delete the output of progressbar, because it doesn't overwrite longer lines
|
print "\033[K" # delete the output of progressbar, because it doesn't overwrite longer lines
|
||||||
final_format = @max_progress ? " %a | #{format}" : " %a | #{format}"
|
@progressbar.format = " %a | #{format}#{@extra_information}"
|
||||||
@progressbar.format = "#{final_format}#{@extra_information}"
|
|
||||||
@progressbar.finish
|
@progressbar.finish
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -30,7 +30,14 @@ module Migrations::Converters::Base
|
|||||||
end
|
end
|
||||||
|
|
||||||
def steps
|
def steps
|
||||||
raise NotImplementedError
|
step_class = ::Migrations::Converters::Base::Step
|
||||||
|
current_module = self.class.name.deconstantize.constantize
|
||||||
|
|
||||||
|
current_module
|
||||||
|
.constants
|
||||||
|
.map { |c| current_module.const_get(c) }
|
||||||
|
.select { |klass| klass.is_a?(Class) && klass < step_class }
|
||||||
|
.sort_by(&:to_s)
|
||||||
end
|
end
|
||||||
|
|
||||||
def before_step_execution(step)
|
def before_step_execution(step)
|
||||||
@ -70,7 +77,7 @@ module Migrations::Converters::Base
|
|||||||
end
|
end
|
||||||
|
|
||||||
def create_step(step_class)
|
def create_step(step_class)
|
||||||
default_args = { settings: settings }
|
default_args = { settings: }
|
||||||
|
|
||||||
args = default_args.merge(step_args(step_class))
|
args = default_args.merge(step_args(step_class))
|
||||||
step_class.new(StepTracker.new, args)
|
step_class.new(StepTracker.new, args)
|
||||||
|
@ -39,7 +39,11 @@ module Migrations::Converters::Base
|
|||||||
with_progressbar do |progressbar|
|
with_progressbar do |progressbar|
|
||||||
@step.items.each do |item|
|
@step.items.each do |item|
|
||||||
stats = job.run(item)
|
stats = job.run(item)
|
||||||
progressbar.update(stats.progress, stats.warning_count, stats.error_count)
|
progressbar.update(
|
||||||
|
increment_by: stats.progress,
|
||||||
|
warning_count: stats.warning_count,
|
||||||
|
error_count: stats.error_count,
|
||||||
|
)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
@ -90,7 +94,11 @@ module Migrations::Converters::Base
|
|||||||
::Migrations::Database::IntermediateDB.insert(sql, *parameters)
|
::Migrations::Database::IntermediateDB.insert(sql, *parameters)
|
||||||
end
|
end
|
||||||
|
|
||||||
progressbar.update(stats.progress, stats.warning_count, stats.error_count)
|
progressbar.update(
|
||||||
|
increment_by: stats.progress,
|
||||||
|
warning_count: stats.warning_count,
|
||||||
|
error_count: stats.error_count,
|
||||||
|
)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -36,12 +36,7 @@ module Migrations::Converters::Base
|
|||||||
private
|
private
|
||||||
|
|
||||||
def log(type, message, exception: nil, details: nil)
|
def log(type, message, exception: nil, details: nil)
|
||||||
::Migrations::Database::IntermediateDB::LogEntry.create!(
|
::Migrations::Database::IntermediateDB::LogEntry.create(type:, message:, exception:, details:)
|
||||||
type:,
|
|
||||||
message:,
|
|
||||||
exception:,
|
|
||||||
details:,
|
|
||||||
)
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -6,7 +6,7 @@ module Migrations::Converters::Example
|
|||||||
|
|
||||||
def execute
|
def execute
|
||||||
super
|
super
|
||||||
IntermediateDB::LogEntry.create!(type: "info", message: "This is a test")
|
IntermediateDB::LogEntry.create(type: "info", message: "This is a test")
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -14,7 +14,7 @@ module Migrations::Converters::Example
|
|||||||
step.log_warning("Test", details: item) if item.in?([3, 7, 9])
|
step.log_warning("Test", details: item) if item.in?([3, 7, 9])
|
||||||
step.log_error("Test", details: item) if item.in?([6, 10])
|
step.log_error("Test", details: item) if item.in?([6, 10])
|
||||||
|
|
||||||
IntermediateDB::LogEntry.create!(type: "info", message: "Step2 - #{item}")
|
IntermediateDB::LogEntry.create(type: "info", message: "Step2 - #{item}")
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -17,7 +17,7 @@ module Migrations::Converters::Example
|
|||||||
|
|
||||||
step.log_warning("Test", details: item) if item[:counter] > 10 && item[:counter] < 20
|
step.log_warning("Test", details: item) if item[:counter] > 10 && item[:counter] < 20
|
||||||
|
|
||||||
IntermediateDB::LogEntry.create!(type: "info", message: "Step3 - #{item[:counter]}")
|
IntermediateDB::LogEntry.create(type: "info", message: "Step3 - #{item[:counter]}")
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -59,5 +59,17 @@ module Migrations
|
|||||||
return nil if value.nil?
|
return nil if value.nil?
|
||||||
::Oj.dump(value, mode: :compat)
|
::Oj.dump(value, mode: :compat)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def self.to_date(text)
|
||||||
|
text.present? ? Date.parse(text) : nil
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.to_datetime(text)
|
||||||
|
text.present? ? DateTime.parse(text) : nil
|
||||||
|
end
|
||||||
|
|
||||||
|
def self.to_boolean(value)
|
||||||
|
value == 1
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -8,6 +8,7 @@ module Migrations::Database
|
|||||||
PREPARED_STATEMENT_CACHE_SIZE = 5
|
PREPARED_STATEMENT_CACHE_SIZE = 5
|
||||||
|
|
||||||
def self.open_database(path:)
|
def self.open_database(path:)
|
||||||
|
path = File.expand_path(path, ::Migrations.root_path)
|
||||||
FileUtils.mkdir_p(File.dirname(path))
|
FileUtils.mkdir_p(File.dirname(path))
|
||||||
|
|
||||||
db = ::Extralite::Database.new(path)
|
db = ::Extralite::Database.new(path)
|
||||||
@ -25,7 +26,7 @@ module Migrations::Database
|
|||||||
attr_reader :db, :path
|
attr_reader :db, :path
|
||||||
|
|
||||||
def initialize(path:, transaction_batch_size: TRANSACTION_BATCH_SIZE)
|
def initialize(path:, transaction_batch_size: TRANSACTION_BATCH_SIZE)
|
||||||
@path = path
|
@path = File.expand_path(path, ::Migrations.root_path)
|
||||||
@transaction_batch_size = transaction_batch_size
|
@transaction_batch_size = transaction_batch_size
|
||||||
@db = self.class.open_database(path:)
|
@db = self.class.open_database(path:)
|
||||||
@statement_counter = 0
|
@statement_counter = 0
|
||||||
@ -54,24 +55,34 @@ module Migrations::Database
|
|||||||
|
|
||||||
if (@statement_counter += 1) >= @transaction_batch_size
|
if (@statement_counter += 1) >= @transaction_batch_size
|
||||||
commit_transaction
|
commit_transaction
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def query(sql, *parameters, &block)
|
||||||
|
@db.query(sql, *parameters, &block)
|
||||||
|
end
|
||||||
|
|
||||||
|
def count(sql, *parameters)
|
||||||
|
@db.query_single_splat(sql, *parameters)
|
||||||
|
end
|
||||||
|
|
||||||
|
def execute(sql, *parameters)
|
||||||
|
@db.execute(sql, *parameters)
|
||||||
|
end
|
||||||
|
|
||||||
|
def begin_transaction
|
||||||
|
@db.execute("BEGIN DEFERRED TRANSACTION") unless @db.transaction_active?
|
||||||
|
end
|
||||||
|
|
||||||
|
def commit_transaction
|
||||||
|
if @db.transaction_active?
|
||||||
|
@db.execute("COMMIT")
|
||||||
@statement_counter = 0
|
@statement_counter = 0
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
def begin_transaction
|
|
||||||
return if @db.transaction_active?
|
|
||||||
|
|
||||||
@db.execute("BEGIN DEFERRED TRANSACTION")
|
|
||||||
end
|
|
||||||
|
|
||||||
def commit_transaction
|
|
||||||
return unless @db.transaction_active?
|
|
||||||
|
|
||||||
@db.execute("COMMIT")
|
|
||||||
end
|
|
||||||
|
|
||||||
def close_connection(keep_path:)
|
def close_connection(keep_path:)
|
||||||
return if @db.nil?
|
return if @db.nil?
|
||||||
|
|
||||||
|
@ -11,7 +11,7 @@ module Migrations::Database::IntermediateDB
|
|||||||
VALUES (?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?)
|
||||||
SQL
|
SQL
|
||||||
|
|
||||||
def self.create!(created_at: Time.now, type:, message:, exception: nil, details: nil)
|
def self.create(created_at: Time.now, type:, message:, exception: nil, details: nil)
|
||||||
::Migrations::Database::IntermediateDB.insert(
|
::Migrations::Database::IntermediateDB.insert(
|
||||||
SQL,
|
SQL,
|
||||||
::Migrations::Database.format_datetime(created_at),
|
::Migrations::Database.format_datetime(created_at),
|
||||||
|
@ -17,8 +17,7 @@ module Migrations::Database::IntermediateDB
|
|||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
SQL
|
SQL
|
||||||
|
|
||||||
class << self
|
def self.create_for_file(
|
||||||
def create_for_file!(
|
|
||||||
path:,
|
path:,
|
||||||
filename: nil,
|
filename: nil,
|
||||||
type: nil,
|
type: nil,
|
||||||
@ -26,7 +25,7 @@ module Migrations::Database::IntermediateDB
|
|||||||
origin: nil,
|
origin: nil,
|
||||||
user_id: nil
|
user_id: nil
|
||||||
)
|
)
|
||||||
create!(
|
create(
|
||||||
id: ::Migrations::ID.hash(path),
|
id: ::Migrations::ID.hash(path),
|
||||||
filename: filename || File.basename(path),
|
filename: filename || File.basename(path),
|
||||||
path:,
|
path:,
|
||||||
@ -37,8 +36,8 @@ module Migrations::Database::IntermediateDB
|
|||||||
)
|
)
|
||||||
end
|
end
|
||||||
|
|
||||||
def create_for_url!(url:, filename:, type: nil, description: nil, origin: nil, user_id: nil)
|
def self.create_for_url(url:, filename:, type: nil, description: nil, origin: nil, user_id: nil)
|
||||||
create!(
|
create(
|
||||||
id: ::Migrations::ID.hash(url),
|
id: ::Migrations::ID.hash(url),
|
||||||
filename:,
|
filename:,
|
||||||
url:,
|
url:,
|
||||||
@ -49,8 +48,15 @@ module Migrations::Database::IntermediateDB
|
|||||||
)
|
)
|
||||||
end
|
end
|
||||||
|
|
||||||
def create_for_data!(data:, filename:, type: nil, description: nil, origin: nil, user_id: nil)
|
def self.create_for_data(
|
||||||
create!(
|
data:,
|
||||||
|
filename:,
|
||||||
|
type: nil,
|
||||||
|
description: nil,
|
||||||
|
origin: nil,
|
||||||
|
user_id: nil
|
||||||
|
)
|
||||||
|
create(
|
||||||
id: ::Migrations::ID.hash(data),
|
id: ::Migrations::ID.hash(data),
|
||||||
filename:,
|
filename:,
|
||||||
data: ::Migrations::Database.to_blob(data),
|
data: ::Migrations::Database.to_blob(data),
|
||||||
@ -61,9 +67,7 @@ module Migrations::Database::IntermediateDB
|
|||||||
)
|
)
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
def self.create(
|
||||||
|
|
||||||
def create!(
|
|
||||||
id:,
|
id:,
|
||||||
filename:,
|
filename:,
|
||||||
path: nil,
|
path: nil,
|
||||||
@ -88,5 +92,4 @@ module Migrations::Database::IntermediateDB
|
|||||||
)
|
)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
module Migrations::Database
|
module Migrations::Database
|
||||||
class Migrator
|
class Migrator
|
||||||
def initialize(db_path)
|
def initialize(db_path)
|
||||||
@db_path = db_path
|
@db_path = File.expand_path(db_path, ::Migrations.root_path)
|
||||||
@db = nil
|
@db = nil
|
||||||
end
|
end
|
||||||
|
|
||||||
@ -72,7 +72,7 @@ module Migrations::Database
|
|||||||
|
|
||||||
@db.transaction do
|
@db.transaction do
|
||||||
@db.execute(sql)
|
@db.execute(sql)
|
||||||
@db.execute(<<~SQL, path: relative_path, sql_hash: sql_hash)
|
@db.execute(<<~SQL, path: relative_path, sql_hash:)
|
||||||
INSERT INTO schema_migrations (path, created_at, sql_hash)
|
INSERT INTO schema_migrations (path, created_at, sql_hash)
|
||||||
VALUES (:path, datetime('now'), :sql_hash)
|
VALUES (:path, datetime('now'), :sql_hash)
|
||||||
SQL
|
SQL
|
||||||
|
@ -71,7 +71,7 @@ module Migrations::Uploader
|
|||||||
end
|
end
|
||||||
|
|
||||||
def update_status_queue(row, upload, status)
|
def update_status_queue(row, upload, status)
|
||||||
status_queue << { id: row[:id], upload_id: upload[:id], status: status }
|
status_queue << { id: row[:id], upload_id: upload[:id], status: }
|
||||||
end
|
end
|
||||||
|
|
||||||
def log_status
|
def log_status
|
||||||
|
@ -7,21 +7,21 @@ require "active_support"
|
|||||||
require "active_support/core_ext"
|
require "active_support/core_ext"
|
||||||
require "zeitwerk"
|
require "zeitwerk"
|
||||||
|
|
||||||
require_relative "converters"
|
require_relative "lib/converters"
|
||||||
|
|
||||||
module Migrations
|
module Migrations
|
||||||
class NoSettingsFound < StandardError
|
class NoSettingsFound < StandardError
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.root_path
|
def self.root_path
|
||||||
@root_path ||= File.expand_path("..", __dir__)
|
@root_path ||= __dir__
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.load_rails_environment(quiet: false)
|
def self.load_rails_environment(quiet: false)
|
||||||
message = "Loading Rails environment ..."
|
message = "Loading Rails environment..."
|
||||||
print message if !quiet
|
print message if !quiet
|
||||||
|
|
||||||
rails_root = File.expand_path("../..", __dir__)
|
rails_root = File.expand_path("..", __dir__)
|
||||||
# rubocop:disable Discourse/NoChdir
|
# rubocop:disable Discourse/NoChdir
|
||||||
Dir.chdir(rails_root) do
|
Dir.chdir(rails_root) do
|
||||||
begin
|
begin
|
||||||
@ -48,7 +48,9 @@ module Migrations
|
|||||||
{
|
{
|
||||||
"cli" => "CLI",
|
"cli" => "CLI",
|
||||||
"id" => "ID",
|
"id" => "ID",
|
||||||
|
"discourse_db" => "DiscourseDB",
|
||||||
"intermediate_db" => "IntermediateDB",
|
"intermediate_db" => "IntermediateDB",
|
||||||
|
"mappings_db" => "MappingsDB",
|
||||||
"uploads_db" => "UploadsDB",
|
"uploads_db" => "UploadsDB",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -64,7 +66,7 @@ module Migrations
|
|||||||
|
|
||||||
Dir[File.join(converter_path, "**", "*")].each do |subdirectory|
|
Dir[File.join(converter_path, "**", "*")].each do |subdirectory|
|
||||||
next unless File.directory?(subdirectory)
|
next unless File.directory?(subdirectory)
|
||||||
loader.push_dir(subdirectory, namespace: namespace)
|
loader.push_dir(subdirectory, namespace:)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
@ -20,7 +20,7 @@ User = Data.define(:id, :name, :email, :created_at)
|
|||||||
USER_HASH =
|
USER_HASH =
|
||||||
begin
|
begin
|
||||||
name = SecureRandom.hex(10)
|
name = SecureRandom.hex(10)
|
||||||
{ id: 1, name: name, email: "#{name}@example.com", created_at: Time.now.utc.iso8601 }
|
{ id: 1, name:, email: "#{name}@example.com", created_at: Time.now.utc.iso8601 }
|
||||||
end
|
end
|
||||||
|
|
||||||
USER_DATA =
|
USER_DATA =
|
||||||
|
@ -41,7 +41,7 @@ end
|
|||||||
def create_users(row_count)
|
def create_users(row_count)
|
||||||
row_count.times.map do |id|
|
row_count.times.map do |id|
|
||||||
name = SecureRandom.hex(10)
|
name = SecureRandom.hex(10)
|
||||||
{ id: id, name: name, email: "#{name}@example.com", created_at: Time.now.utc.iso8601 }
|
{ id:, name:, email: "#{name}@example.com", created_at: Time.now.utc.iso8601 }
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
RSpec.describe ::Migrations::Converters::Base::StepTracker do
|
RSpec.describe ::Migrations::Converters::Base::StepTracker do
|
||||||
subject(:tracker) { described_class.new }
|
subject(:tracker) { described_class.new }
|
||||||
|
|
||||||
before { allow(::Migrations::Database::IntermediateDB::LogEntry).to receive(:create!) }
|
before { allow(::Migrations::Database::IntermediateDB::LogEntry).to receive(:create) }
|
||||||
|
|
||||||
describe "#initialize" do
|
describe "#initialize" do
|
||||||
it "starts at the correct values" do
|
it "starts at the correct values" do
|
||||||
@ -75,7 +75,7 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker do
|
|||||||
it "logs an info message" do
|
it "logs an info message" do
|
||||||
tracker.log_info("Info message")
|
tracker.log_info("Info message")
|
||||||
|
|
||||||
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
|
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(
|
||||||
type: ::Migrations::Database::IntermediateDB::LogEntry::INFO,
|
type: ::Migrations::Database::IntermediateDB::LogEntry::INFO,
|
||||||
message: "Info message",
|
message: "Info message",
|
||||||
exception: nil,
|
exception: nil,
|
||||||
@ -86,7 +86,7 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker do
|
|||||||
it "logs an info message with details" do
|
it "logs an info message with details" do
|
||||||
tracker.log_info("Info message", details: { key: "value" })
|
tracker.log_info("Info message", details: { key: "value" })
|
||||||
|
|
||||||
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
|
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(
|
||||||
type: ::Migrations::Database::IntermediateDB::LogEntry::INFO,
|
type: ::Migrations::Database::IntermediateDB::LogEntry::INFO,
|
||||||
message: "Info message",
|
message: "Info message",
|
||||||
exception: nil,
|
exception: nil,
|
||||||
@ -103,7 +103,7 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker do
|
|||||||
tracker.stats.warning_count
|
tracker.stats.warning_count
|
||||||
}.by(1)
|
}.by(1)
|
||||||
|
|
||||||
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
|
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(
|
||||||
type: ::Migrations::Database::IntermediateDB::LogEntry::WARNING,
|
type: ::Migrations::Database::IntermediateDB::LogEntry::WARNING,
|
||||||
message: "Warning message",
|
message: "Warning message",
|
||||||
exception: nil,
|
exception: nil,
|
||||||
@ -115,13 +115,13 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker do
|
|||||||
exception = StandardError.new("Warning exception")
|
exception = StandardError.new("Warning exception")
|
||||||
|
|
||||||
expect {
|
expect {
|
||||||
tracker.log_warning("Warning message", exception: exception, details: { key: "value" })
|
tracker.log_warning("Warning message", exception:, details: { key: "value" })
|
||||||
}.to change { tracker.stats.warning_count }.by(1)
|
}.to change { tracker.stats.warning_count }.by(1)
|
||||||
|
|
||||||
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
|
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(
|
||||||
type: ::Migrations::Database::IntermediateDB::LogEntry::WARNING,
|
type: ::Migrations::Database::IntermediateDB::LogEntry::WARNING,
|
||||||
message: "Warning message",
|
message: "Warning message",
|
||||||
exception: exception,
|
exception:,
|
||||||
details: {
|
details: {
|
||||||
key: "value",
|
key: "value",
|
||||||
},
|
},
|
||||||
@ -133,7 +133,7 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker do
|
|||||||
it "logs an error message and increments error_count" do
|
it "logs an error message and increments error_count" do
|
||||||
expect { tracker.log_error("Error message") }.to change { tracker.stats.error_count }.by(1)
|
expect { tracker.log_error("Error message") }.to change { tracker.stats.error_count }.by(1)
|
||||||
|
|
||||||
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
|
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(
|
||||||
type: ::Migrations::Database::IntermediateDB::LogEntry::ERROR,
|
type: ::Migrations::Database::IntermediateDB::LogEntry::ERROR,
|
||||||
message: "Error message",
|
message: "Error message",
|
||||||
exception: nil,
|
exception: nil,
|
||||||
@ -145,13 +145,13 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker do
|
|||||||
exception = StandardError.new("Error exception")
|
exception = StandardError.new("Error exception")
|
||||||
|
|
||||||
expect {
|
expect {
|
||||||
tracker.log_error("Error message", exception: exception, details: { key: "value" })
|
tracker.log_error("Error message", exception:, details: { key: "value" })
|
||||||
}.to change { tracker.stats.error_count }.by(1)
|
}.to change { tracker.stats.error_count }.by(1)
|
||||||
|
|
||||||
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create!).with(
|
expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(
|
||||||
type: ::Migrations::Database::IntermediateDB::LogEntry::ERROR,
|
type: ::Migrations::Database::IntermediateDB::LogEntry::ERROR,
|
||||||
message: "Error message",
|
message: "Error message",
|
||||||
exception: exception,
|
exception:,
|
||||||
details: {
|
details: {
|
||||||
key: "value",
|
key: "value",
|
||||||
},
|
},
|
||||||
|
@ -94,4 +94,25 @@ RSpec.describe ::Migrations::Database::IntermediateDB do
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
it "checks that manually created entities are tested" do
|
||||||
|
Dir[
|
||||||
|
File.join(::Migrations.root_path, "lib", "database", "intermediate_db", "*.rb")
|
||||||
|
].each do |path|
|
||||||
|
next if File.read(path).include?("auto-generated")
|
||||||
|
|
||||||
|
spec_path =
|
||||||
|
File.join(
|
||||||
|
::Migrations.root_path,
|
||||||
|
"spec",
|
||||||
|
"lib",
|
||||||
|
"database",
|
||||||
|
"intermediate_db",
|
||||||
|
"#{File.basename(path, ".rb")}_spec.rb",
|
||||||
|
)
|
||||||
|
|
||||||
|
expect(File.exist?(spec_path)).to eq(true), "#{spec_path} is missing"
|
||||||
|
expect(File.read(spec_path)).to include('it_behaves_like "a database entity"')
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
@ -9,15 +9,7 @@ RSpec.describe ::Migrations::Database::Migrator do
|
|||||||
ignore_errors: false
|
ignore_errors: false
|
||||||
)
|
)
|
||||||
if migrations_directory
|
if migrations_directory
|
||||||
migrations_path =
|
migrations_path = File.join(fixture_root, "schema", migrations_directory)
|
||||||
File.join(
|
|
||||||
::Migrations.root_path,
|
|
||||||
"spec",
|
|
||||||
"support",
|
|
||||||
"fixtures",
|
|
||||||
"schema",
|
|
||||||
migrations_directory,
|
|
||||||
)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
temp_path = storage_path = Dir.mktmpdir if storage_path.nil?
|
temp_path = storage_path = Dir.mktmpdir if storage_path.nil?
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
# we need to require the rails_helper from core to load the Rails environment
|
# we need to require the rails_helper from core to load the Rails environment
|
||||||
require_relative "../../spec/rails_helper"
|
require_relative "../../spec/rails_helper"
|
||||||
|
|
||||||
require_relative "../lib/migrations"
|
require_relative "../migrations"
|
||||||
|
|
||||||
::Migrations.configure_zeitwerk
|
::Migrations.configure_zeitwerk
|
||||||
::Migrations.enable_i18n
|
::Migrations.enable_i18n
|
||||||
|
@ -5,3 +5,7 @@ def reset_memoization(instance, *variables)
|
|||||||
instance.remove_instance_variable(var) if instance.instance_variable_defined?(var)
|
instance.remove_instance_variable(var) if instance.instance_variable_defined?(var)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def fixture_root
|
||||||
|
@fixture_root ||= File.join(::Migrations.root_path, "spec", "support", "fixtures")
|
||||||
|
end
|
||||||
|
@ -5,7 +5,7 @@ RSpec.shared_examples "a database entity" do
|
|||||||
expect(subject).to have_constant(:SQL)
|
expect(subject).to have_constant(:SQL)
|
||||||
end
|
end
|
||||||
|
|
||||||
it "responds to .create!" do
|
it "responds to .create" do
|
||||||
expect(subject).to respond_to(:create!)
|
expect(subject).to respond_to(:create)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
Reference in New Issue
Block a user