Use MessageBus to get other processes to failover faster.

This commit is contained in:
Guo Xiang Tan
2017-10-23 15:27:58 +08:00
parent c15c77884e
commit 54455fa40b
2 changed files with 35 additions and 15 deletions

View File

@ -9,10 +9,18 @@ class PostgreSQLFallbackHandler
attr_reader :masters_down
attr_accessor :initialized
DATABASE_DOWN_CHANNEL = '/global/database_down'.freeze
def initialize
@masters_down = DistributedCache.new('masters_down', namespace: false)
@mutex = Mutex.new
@initialized = false
MessageBus.subscribe(DATABASE_DOWN_CHANNEL) do |payload|
RailsMultisite::ConnectionManagement.with_connection(payload.data['db']) do
clear_connections
end
end
end
def verify_master
@ -38,10 +46,11 @@ class PostgreSQLFallbackHandler
synchronize { @masters_down[namespace] }
end
def master_down=(args)
def master_down
synchronize do
@masters_down[namespace] = args
Sidekiq.pause! if args && !Sidekiq.paused?
@masters_down[namespace] = true
Sidekiq.pause! if !Sidekiq.paused?
MessageBus.publish(DATABASE_DOWN_CHANNEL, db: namespace)
end
end
@ -63,8 +72,7 @@ class PostgreSQLFallbackHandler
if is_connection_active
logger.warn "#{log_prefix}: Master server is active. Reconnecting..."
ActiveRecord::Base.clear_active_connections!
ActiveRecord::Base.clear_all_connections!
clear_connections
self.master_up(key)
disable_readonly_mode
Sidekiq.unpause!
@ -82,6 +90,11 @@ class PostgreSQLFallbackHandler
disable_readonly_mode
end
def clear_connections
ActiveRecord::Base.clear_active_connections!
ActiveRecord::Base.clear_all_connections!
end
private
def disable_readonly_mode
@ -130,12 +143,13 @@ module ActiveRecord
connection = postgresql_connection(config)
fallback_handler.initialized ||= true
rescue PG::ConnectionBad => e
fallback_handler.master_down = true
fallback_handler.master_down
fallback_handler.verify_master
if !fallback_handler.initialized
return postgresql_fallback_connection(config)
else
fallback_handler.clear_connections
raise e
end
end