mirror of
https://github.com/discourse/discourse.git
synced 2025-05-22 07:53:49 +08:00
This reverts commit 4f677854d3df2555ddeaa1d02c228960d33e0248.
This commit is contained in:
Krzysztof Kotlarek
committed by
GitHub
GitHub
parent
8eb2147f1f
commit
8cc09fc668
@ -3,84 +3,20 @@
|
||||
require 'rails_helper'
|
||||
|
||||
describe DiscourseRedis do
|
||||
before do
|
||||
DiscourseRedis::FallbackHandlers.instance.instance_variable_set(:@fallback_handlers, {})
|
||||
end
|
||||
|
||||
let(:slave_host) { 'testhost' }
|
||||
let(:slave_port) { 1234 }
|
||||
|
||||
let(:config) do
|
||||
GlobalSetting.redis_config.dup.merge(slave_host: 'testhost', slave_port: 1234, connector: DiscourseRedis::Connector)
|
||||
DiscourseRedis.config.dup.merge(slave_host: 'testhost', slave_port: 1234, connector: DiscourseRedis::Connector)
|
||||
end
|
||||
|
||||
let(:slave_config) { DiscourseRedis.slave_config(config) }
|
||||
let(:fallback_handler) { DiscourseRedis::FallbackHandler.instance }
|
||||
|
||||
it "ignore_readonly returns nil from a pure exception" do
|
||||
result = DiscourseRedis.ignore_readonly { raise Redis::CommandError.new("READONLY") }
|
||||
expect(result).to eq(nil)
|
||||
end
|
||||
|
||||
let!(:master_conn) { mock('master') }
|
||||
|
||||
def self.use_fake_threads
|
||||
attr_reader :execution
|
||||
|
||||
around(:each) do |example|
|
||||
scenario =
|
||||
Concurrency::Scenario.new do |execution|
|
||||
@execution = execution
|
||||
example.run
|
||||
end
|
||||
|
||||
scenario.run(sleep_order: true, runs: 1)
|
||||
end
|
||||
|
||||
after(:each) do
|
||||
# Doing this here, as opposed to after example.run, ensures that it
|
||||
# happens before the mocha expectations are checked.
|
||||
execution.wait_done
|
||||
end
|
||||
end
|
||||
|
||||
def stop_after(time)
|
||||
execution.sleep(time)
|
||||
execution.stop_other_tasks
|
||||
end
|
||||
|
||||
def expect_master_info(conf = config)
|
||||
conf = conf.dup
|
||||
conf.delete(:connector)
|
||||
|
||||
Redis::Client.expects(:new)
|
||||
.with(conf)
|
||||
.returns(master_conn)
|
||||
|
||||
master_conn.expects(:disconnect)
|
||||
master_conn
|
||||
.expects(:call)
|
||||
.with([:info])
|
||||
end
|
||||
|
||||
def info_response(*values)
|
||||
values.map { |x| x.join(':') }.join("\r\n")
|
||||
end
|
||||
|
||||
def expect_fallback(config = slave_config)
|
||||
slave_conn = mock('slave')
|
||||
|
||||
config = config.dup
|
||||
config.delete(:connector)
|
||||
|
||||
Redis::Client.expects(:new)
|
||||
.with(config)
|
||||
.returns(slave_conn)
|
||||
|
||||
slave_conn.expects(:call).with([:client, [:kill, 'type', 'normal']])
|
||||
slave_conn.expects(:call).with([:client, [:kill, 'type', 'pubsub']])
|
||||
slave_conn.expects(:disconnect)
|
||||
end
|
||||
|
||||
describe 'redis commands' do
|
||||
let(:raw_redis) { Redis.new(DiscourseRedis.config) }
|
||||
|
||||
@ -161,349 +97,150 @@ describe DiscourseRedis do
|
||||
end
|
||||
end
|
||||
|
||||
describe DiscourseRedis::RedisStatus do
|
||||
let(:redis_status) { DiscourseRedis::RedisStatus.new(config, slave_config) }
|
||||
|
||||
context "#master_alive?" do
|
||||
it "returns false when the master's hostname cannot be resolved" do
|
||||
expect_master_info
|
||||
.raises(RuntimeError.new('Name or service not known'))
|
||||
|
||||
expect(redis_status.master_alive?).to eq(false)
|
||||
end
|
||||
|
||||
it "raises an error if a runtime error is raised" do
|
||||
error = RuntimeError.new('a random runtime error')
|
||||
expect_master_info.raises(error)
|
||||
|
||||
expect {
|
||||
redis_status.master_alive?
|
||||
}.to raise_error(error)
|
||||
end
|
||||
|
||||
it "returns false if the master is unavailable" do
|
||||
expect_master_info.raises(Redis::ConnectionError.new)
|
||||
|
||||
expect(redis_status.master_alive?).to eq(false)
|
||||
end
|
||||
|
||||
it "returns false if the master is loading" do
|
||||
expect_master_info
|
||||
.returns(info_response(['loading', '1'], ['role', 'master']))
|
||||
|
||||
expect(redis_status.master_alive?).to eq(false)
|
||||
end
|
||||
|
||||
it "returns false if the master is a slave" do
|
||||
expect_master_info
|
||||
.returns(info_response(['loading', '0'], ['role', 'slave']))
|
||||
|
||||
expect(redis_status.master_alive?).to eq(false)
|
||||
end
|
||||
|
||||
it "returns true when the master isn't loading and the role is master" do
|
||||
expect_master_info
|
||||
.returns(info_response(['loading', '0'], ['role', 'master']))
|
||||
|
||||
expect(redis_status.master_alive?).to eq(true)
|
||||
end
|
||||
end
|
||||
|
||||
context "#fallback" do
|
||||
it "instructs redis to kill client connections" do
|
||||
expect_fallback
|
||||
|
||||
redis_status.fallback
|
||||
context 'when redis connection is to a slave redis server' do
|
||||
it 'should check the status of the master server' do
|
||||
begin
|
||||
fallback_handler.master = false
|
||||
Discourse.redis.without_namespace.expects(:set).raises(Redis::CommandError.new("READONLY"))
|
||||
fallback_handler.expects(:verify_master).once
|
||||
Discourse.redis.set('test', '1')
|
||||
ensure
|
||||
fallback_handler.master = true
|
||||
Discourse.redis.del('test')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe DiscourseRedis::Connector do
|
||||
let(:connector) { DiscourseRedis::Connector.new(config) }
|
||||
let(:fallback_handler) { mock('fallback_handler') }
|
||||
|
||||
before do
|
||||
DiscourseRedis::FallbackHandlers.stubs(:handler_for).returns(fallback_handler)
|
||||
after do
|
||||
fallback_handler.master = true
|
||||
end
|
||||
|
||||
it 'should return the master config when master is up' do
|
||||
fallback_handler.expects(:use_master?).returns(true)
|
||||
expect(connector.resolve).to eq(config)
|
||||
end
|
||||
|
||||
class BrokenRedis
|
||||
def initialize(error)
|
||||
@error = error
|
||||
end
|
||||
|
||||
def call(*args)
|
||||
raise @error
|
||||
end
|
||||
|
||||
def disconnect
|
||||
end
|
||||
end
|
||||
|
||||
it 'should return the slave config when master is down' do
|
||||
fallback_handler.expects(:use_master?).returns(false)
|
||||
expect(connector.resolve).to eq(slave_config)
|
||||
error = Redis::CannotConnectError
|
||||
|
||||
expect do
|
||||
connector.resolve(BrokenRedis.new(error))
|
||||
end.to raise_error(Redis::CannotConnectError)
|
||||
|
||||
config = connector.resolve
|
||||
|
||||
expect(config[:host]).to eq(slave_host)
|
||||
expect(config[:port]).to eq(slave_port)
|
||||
end
|
||||
|
||||
it "should return the slave config when master's hostname cannot be resolved" do
|
||||
error = RuntimeError.new('Name or service not known')
|
||||
|
||||
expect do
|
||||
connector.resolve(BrokenRedis.new(error))
|
||||
end.to raise_error(error)
|
||||
|
||||
expect(fallback_handler.master).to eq(false)
|
||||
|
||||
config = connector.resolve
|
||||
|
||||
expect(config[:host]).to eq(slave_host)
|
||||
expect(config[:port]).to eq(slave_port)
|
||||
expect(fallback_handler.master).to eq(false)
|
||||
end
|
||||
|
||||
it "should return the slave config when master is still loading data" do
|
||||
Redis::Client.any_instance
|
||||
.expects(:call)
|
||||
.with([:info, :persistence])
|
||||
.returns("
|
||||
someconfig:haha\r
|
||||
#{DiscourseRedis::FallbackHandler::MASTER_LOADING_STATUS}
|
||||
")
|
||||
|
||||
config = connector.resolve
|
||||
|
||||
expect(config[:host]).to eq(slave_host)
|
||||
expect(config[:port]).to eq(slave_port)
|
||||
end
|
||||
|
||||
it "should raise the right error" do
|
||||
error = RuntimeError.new('test')
|
||||
|
||||
2.times do
|
||||
expect { connector.resolve(BrokenRedis.new(error)) }
|
||||
.to raise_error(error)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe DiscourseRedis::FallbackHandler do
|
||||
use_fake_threads
|
||||
|
||||
let!(:redis_status) { mock }
|
||||
let!(:fallback_handler) { DiscourseRedis::FallbackHandler.new("", redis_status, execution) }
|
||||
|
||||
context "in the initial configuration" do
|
||||
it "tests that the master is alive and returns true if it is" do
|
||||
redis_status.expects(:master_alive?).returns(true)
|
||||
|
||||
expect(fallback_handler.use_master?).to eq(true)
|
||||
end
|
||||
|
||||
it "tests that the master is alive and returns false if it is not" do
|
||||
redis_status.expects(:master_alive?).returns(false)
|
||||
expect(fallback_handler.use_master?).to eq(false)
|
||||
|
||||
stop_after(1)
|
||||
end
|
||||
|
||||
it "tests that the master is alive and returns false if it raises an exception" do
|
||||
error = Exception.new
|
||||
redis_status.expects(:master_alive?).raises(error)
|
||||
|
||||
Discourse.expects(:warn_exception)
|
||||
.with(error, message: "Error running master_alive?")
|
||||
|
||||
expect(fallback_handler.use_master?).to eq(false)
|
||||
|
||||
stop_after(1)
|
||||
end
|
||||
end
|
||||
|
||||
context "after master_alive? has returned false" do
|
||||
before do
|
||||
redis_status.expects(:master_alive?).returns(false)
|
||||
expect(fallback_handler.use_master?).to eq(false)
|
||||
end
|
||||
|
||||
it "responds with false to the next call to use_master? without consulting redis_status" do
|
||||
expect(fallback_handler.use_master?).to eq(false)
|
||||
|
||||
stop_after(1)
|
||||
end
|
||||
|
||||
it "checks that master is alive again after a timeout" do
|
||||
redis_status.expects(:master_alive?).returns(false)
|
||||
|
||||
stop_after(6)
|
||||
end
|
||||
|
||||
it "checks that master is alive again and checks again if an exception is raised" do
|
||||
error = Exception.new
|
||||
redis_status.expects(:master_alive?).raises(error)
|
||||
|
||||
Discourse.expects(:warn_exception)
|
||||
.with(error, message: "Error running master_alive?")
|
||||
|
||||
execution.sleep(6)
|
||||
|
||||
redis_status.expects(:master_alive?).returns(true)
|
||||
redis_status.expects(:fallback)
|
||||
|
||||
stop_after(5)
|
||||
end
|
||||
|
||||
it "triggers a fallback after master_alive? returns true" do
|
||||
redis_status.expects(:master_alive?).returns(true)
|
||||
redis_status.expects(:fallback)
|
||||
|
||||
stop_after(6)
|
||||
end
|
||||
|
||||
context "after falling back" do
|
||||
before do
|
||||
redis_status.expects(:master_alive?).returns(true)
|
||||
redis_status.expects(:fallback)
|
||||
|
||||
stop_after(6)
|
||||
end
|
||||
|
||||
it "tests that the master is alive and returns true if it is" do
|
||||
redis_status.expects(:master_alive?).returns(true)
|
||||
|
||||
expect(fallback_handler.use_master?).to eq(true)
|
||||
end
|
||||
|
||||
it "tests that the master is alive and returns false if it is not" do
|
||||
redis_status.expects(:master_alive?).returns(false)
|
||||
expect(fallback_handler.use_master?).to eq(false)
|
||||
|
||||
stop_after(1)
|
||||
end
|
||||
|
||||
it "tests that the master is alive and returns false if it raises an exception" do
|
||||
error = Exception.new
|
||||
redis_status.expects(:master_alive?).raises(error)
|
||||
|
||||
Discourse.expects(:warn_exception)
|
||||
.with(error, message: "Error running master_alive?")
|
||||
|
||||
expect(fallback_handler.use_master?).to eq(false)
|
||||
|
||||
stop_after(1)
|
||||
end
|
||||
|
||||
it "doesn't do anything to redis_status for a really long time" do
|
||||
stop_after(1e9)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context "when message bus and main are on the same host" do
|
||||
use_fake_threads
|
||||
|
||||
before do
|
||||
# Since config is based on GlobalSetting, we need to fetch it before
|
||||
# stubbing
|
||||
conf = config
|
||||
|
||||
GlobalSetting.stubs(:redis_config).returns(conf)
|
||||
GlobalSetting.stubs(:message_bus_redis_config).returns(conf)
|
||||
|
||||
Concurrency::ThreadedExecution.stubs(:new).returns(execution)
|
||||
@original_keepalive_interval = MessageBus.keepalive_interval
|
||||
end
|
||||
|
||||
context "when the redis master goes down" do
|
||||
it "sets the message bus keepalive interval to 0" do
|
||||
expect_master_info
|
||||
.raises(Redis::ConnectionError.new)
|
||||
|
||||
MessageBus.expects(:keepalive_interval=).with(0)
|
||||
|
||||
DiscourseRedis::Connector.new(config).resolve
|
||||
|
||||
execution.stop_other_tasks
|
||||
end
|
||||
after do
|
||||
fallback_handler.master = true
|
||||
MessageBus.keepalive_interval = @original_keepalive_interval
|
||||
end
|
||||
|
||||
context "when the redis master comes back up" do
|
||||
before do
|
||||
MessageBus.keepalive_interval = 60
|
||||
describe '#initiate_fallback_to_master' do
|
||||
it 'should return the right value if the master server is still down' do
|
||||
fallback_handler.master = false
|
||||
Redis::Client.any_instance.expects(:call).with([:info]).returns("Some other stuff")
|
||||
|
||||
expect_master_info
|
||||
.raises(Redis::ConnectionError.new)
|
||||
|
||||
DiscourseRedis::Connector.new(config).resolve
|
||||
|
||||
expect_master_info
|
||||
.returns(info_response(['loading', '0'], ['role', 'master']))
|
||||
|
||||
expect_fallback
|
||||
expect(fallback_handler.initiate_fallback_to_master).to eq(false)
|
||||
expect(MessageBus.keepalive_interval).to eq(0)
|
||||
end
|
||||
|
||||
it "sets the message bus keepalive interval to its original value" do
|
||||
MessageBus.expects(:keepalive_interval=).with(60)
|
||||
end
|
||||
it 'should fallback to the master server once it is up' do
|
||||
fallback_handler.master = false
|
||||
master_conn = mock('master')
|
||||
slave_conn = mock('slave')
|
||||
|
||||
it "calls clear_readonly! and request_refresh! on Discourse" do
|
||||
Discourse.expects(:clear_readonly!)
|
||||
Discourse.expects(:request_refresh!)
|
||||
end
|
||||
end
|
||||
end
|
||||
Redis::Client.expects(:new)
|
||||
.with(DiscourseRedis.config)
|
||||
.returns(master_conn)
|
||||
|
||||
context "when message bus and main are on different hosts" do
|
||||
use_fake_threads
|
||||
Redis::Client.expects(:new)
|
||||
.with(DiscourseRedis.slave_config)
|
||||
.returns(slave_conn)
|
||||
|
||||
before do
|
||||
# Since config is based on GlobalSetting, we need to fetch it before stubbing
|
||||
conf = config
|
||||
master_conn.expects(:call)
|
||||
.with([:info])
|
||||
.returns("
|
||||
#{DiscourseRedis::FallbackHandler::MASTER_ROLE_STATUS}\r\n
|
||||
#{DiscourseRedis::FallbackHandler::MASTER_LOADED_STATUS}
|
||||
")
|
||||
|
||||
GlobalSetting.stubs(:redis_config).returns(conf)
|
||||
DiscourseRedis::FallbackHandler::CONNECTION_TYPES.each do |connection_type|
|
||||
slave_conn.expects(:call).with(
|
||||
[:client, [:kill, 'type', connection_type]]
|
||||
)
|
||||
end
|
||||
|
||||
message_bus_config = conf.dup
|
||||
message_bus_config[:port] = message_bus_config[:port].to_i + 1
|
||||
message_bus_config[:slave_port] = message_bus_config[:slave_port].to_i + 1
|
||||
master_conn.expects(:disconnect)
|
||||
slave_conn.expects(:disconnect)
|
||||
|
||||
GlobalSetting.stubs(:message_bus_redis_config).returns(message_bus_config)
|
||||
|
||||
Concurrency::ThreadedExecution.stubs(:new).returns(execution)
|
||||
end
|
||||
|
||||
let(:message_bus_master_config) {
|
||||
GlobalSetting.message_bus_redis_config
|
||||
}
|
||||
|
||||
context "when the message bus master goes down" do
|
||||
before do
|
||||
expect_master_info(message_bus_master_config)
|
||||
.raises(Redis::ConnectionError.new)
|
||||
end
|
||||
|
||||
it "sets the message bus keepalive interval to 0" do
|
||||
MessageBus.expects(:keepalive_interval=).with(0)
|
||||
|
||||
DiscourseRedis::Connector.new(message_bus_master_config).resolve
|
||||
|
||||
execution.stop_other_tasks
|
||||
end
|
||||
|
||||
it "does not call clear_readonly! or request_refresh! on Discourse" do
|
||||
Discourse.expects(:clear_readonly!).never
|
||||
Discourse.expects(:request_refresh!).never
|
||||
|
||||
DiscourseRedis::Connector.new(message_bus_master_config).resolve
|
||||
|
||||
execution.stop_other_tasks
|
||||
end
|
||||
end
|
||||
|
||||
context "when the message bus master comes back up" do
|
||||
before do
|
||||
MessageBus.keepalive_interval = 60
|
||||
|
||||
expect_master_info(message_bus_master_config)
|
||||
.raises(Redis::ConnectionError.new)
|
||||
|
||||
DiscourseRedis::Connector.new(message_bus_master_config).resolve
|
||||
|
||||
expect_master_info(message_bus_master_config)
|
||||
.returns(info_response(['loading', '0'], ['role', 'master']))
|
||||
|
||||
expect_fallback(DiscourseRedis.slave_config(message_bus_master_config))
|
||||
end
|
||||
|
||||
it "sets the message bus keepalive interval to its original value" do
|
||||
MessageBus.expects(:keepalive_interval=).with(60)
|
||||
end
|
||||
end
|
||||
|
||||
context "when the main master goes down" do
|
||||
before do
|
||||
expect_master_info
|
||||
.raises(Redis::ConnectionError.new)
|
||||
end
|
||||
|
||||
it "does not change the message bus keepalive interval" do
|
||||
MessageBus.expects(:keepalive_interval=).never
|
||||
|
||||
DiscourseRedis::Connector.new(config).resolve
|
||||
|
||||
execution.stop_other_tasks
|
||||
end
|
||||
end
|
||||
|
||||
context "when the main master comes back up" do
|
||||
before do
|
||||
expect_master_info
|
||||
.raises(Redis::ConnectionError.new)
|
||||
|
||||
DiscourseRedis::Connector.new(config).resolve
|
||||
|
||||
expect_master_info
|
||||
.returns(info_response(['loading', '0'], ['role', 'master']))
|
||||
|
||||
expect_fallback
|
||||
end
|
||||
|
||||
it "does not change the message bus keepalive interval" do
|
||||
MessageBus.expects(:keepalive_interval=).never
|
||||
end
|
||||
|
||||
it "calls clear_readonly! and request_refresh! on Discourse" do
|
||||
Discourse.expects(:clear_readonly!)
|
||||
Discourse.expects(:request_refresh!)
|
||||
expect(fallback_handler.initiate_fallback_to_master).to eq(true)
|
||||
expect(fallback_handler.master).to eq(true)
|
||||
expect(Discourse.recently_readonly?).to eq(false)
|
||||
expect(MessageBus.keepalive_interval).to eq(-1)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user