mirror of
https://git.postgresql.org/git/postgresql.git
synced 2026-02-08 19:37:35 +08:00
ce0fdbfe97
For the initial table data synchronization in logical replication, we use a single transaction to copy the entire table and then synchronize the position in the stream with the main apply worker. There are multiple downsides of this approach: (a) We have to perform the entire copy operation again if there is any error (network breakdown, error in the database operation, etc.) while we synchronize the WAL position between tablesync worker and apply worker; this will be onerous especially for large copies, (b) Using a single transaction in the synchronization-phase (where we can receive WAL from multiple transactions) will have the risk of exceeding the CID limit, (c) The slot will hold the WAL till the entire sync is complete because we never commit till the end. This patch solves all the above downsides by allowing multiple transactions during the tablesync phase. The initial copy is done in a single transaction and after that, we commit each transaction as we receive. To allow recovery after any error or crash, we use a permanent slot and origin to track the progress. The slot and origin will be removed once we finish the synchronization of the table. We also remove slot and origin of tablesync workers if the user performs DROP SUBSCRIPTION .. or ALTER SUBSCRIPTION .. REFERESH and some of the table syncs are still not finished. The commands ALTER SUBSCRIPTION ... REFRESH PUBLICATION and ALTER SUBSCRIPTION ... SET PUBLICATION ... with refresh option as true cannot be executed inside a transaction block because they can now drop the slots for which we have no provision to rollback. This will also open up the path for logical replication of 2PC transactions on the subscriber side. Previously, we can't do that because of the requirement of maintaining a single transaction in tablesync workers. Bump catalog version due to change of state in the catalog (pg_subscription_rel). Author: Peter Smith, Amit Kapila, and Takamichi Osumi Reviewed-by: Ajin Cherian, Petr Jelinek, Hou Zhijie and Amit Kapila Discussion: https://postgr.es/m/CAA4eK1KHJxaZS-fod-0fey=0tq3=Gkn4ho=8N4-5HWiCfu0H1A@mail.gmail.com
176 lines
6.4 KiB
PL/PgSQL
176 lines
6.4 KiB
PL/PgSQL
--
|
|
-- SUBSCRIPTION
|
|
--
|
|
|
|
CREATE ROLE regress_subscription_user LOGIN SUPERUSER;
|
|
CREATE ROLE regress_subscription_user2;
|
|
CREATE ROLE regress_subscription_user_dummy LOGIN NOSUPERUSER;
|
|
SET SESSION AUTHORIZATION 'regress_subscription_user';
|
|
|
|
-- fail - no publications
|
|
CREATE SUBSCRIPTION regress_testsub CONNECTION 'foo';
|
|
|
|
-- fail - no connection
|
|
CREATE SUBSCRIPTION regress_testsub PUBLICATION foo;
|
|
|
|
-- fail - cannot do CREATE SUBSCRIPTION CREATE SLOT inside transaction block
|
|
BEGIN;
|
|
CREATE SUBSCRIPTION regress_testsub CONNECTION 'testconn' PUBLICATION testpub WITH (create_slot);
|
|
COMMIT;
|
|
|
|
-- fail - invalid connection string
|
|
CREATE SUBSCRIPTION regress_testsub CONNECTION 'testconn' PUBLICATION testpub;
|
|
|
|
-- fail - duplicate publications
|
|
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION foo, testpub, foo WITH (connect = false);
|
|
|
|
-- ok
|
|
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false);
|
|
|
|
COMMENT ON SUBSCRIPTION regress_testsub IS 'test subscription';
|
|
SELECT obj_description(s.oid, 'pg_subscription') FROM pg_subscription s;
|
|
|
|
-- fail - name already exists
|
|
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false);
|
|
|
|
-- fail - must be superuser
|
|
SET SESSION AUTHORIZATION 'regress_subscription_user2';
|
|
CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION foo WITH (connect = false);
|
|
SET SESSION AUTHORIZATION 'regress_subscription_user';
|
|
|
|
-- fail - invalid option combinations
|
|
CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, copy_data = true);
|
|
CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, enabled = true);
|
|
CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, create_slot = true);
|
|
CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (slot_name = NONE, enabled = true);
|
|
CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (slot_name = NONE, create_slot = true);
|
|
CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (slot_name = NONE);
|
|
CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (slot_name = NONE, enabled = false);
|
|
CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (slot_name = NONE, create_slot = false);
|
|
|
|
-- ok - with slot_name = NONE
|
|
CREATE SUBSCRIPTION regress_testsub3 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (slot_name = NONE, connect = false);
|
|
-- fail
|
|
ALTER SUBSCRIPTION regress_testsub3 ENABLE;
|
|
ALTER SUBSCRIPTION regress_testsub3 REFRESH PUBLICATION;
|
|
|
|
DROP SUBSCRIPTION regress_testsub3;
|
|
|
|
-- fail - invalid connection string
|
|
ALTER SUBSCRIPTION regress_testsub CONNECTION 'foobar';
|
|
|
|
\dRs+
|
|
|
|
ALTER SUBSCRIPTION regress_testsub SET PUBLICATION testpub2, testpub3 WITH (refresh = false);
|
|
ALTER SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist2';
|
|
ALTER SUBSCRIPTION regress_testsub SET (slot_name = 'newname');
|
|
|
|
-- fail
|
|
ALTER SUBSCRIPTION regress_doesnotexist CONNECTION 'dbname=regress_doesnotexist2';
|
|
ALTER SUBSCRIPTION regress_testsub SET (create_slot = false);
|
|
|
|
\dRs+
|
|
|
|
BEGIN;
|
|
ALTER SUBSCRIPTION regress_testsub ENABLE;
|
|
|
|
\dRs
|
|
|
|
ALTER SUBSCRIPTION regress_testsub DISABLE;
|
|
|
|
\dRs
|
|
|
|
COMMIT;
|
|
|
|
-- fail - must be owner of subscription
|
|
SET ROLE regress_subscription_user_dummy;
|
|
ALTER SUBSCRIPTION regress_testsub RENAME TO regress_testsub_dummy;
|
|
RESET ROLE;
|
|
|
|
ALTER SUBSCRIPTION regress_testsub RENAME TO regress_testsub_foo;
|
|
ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = local);
|
|
ALTER SUBSCRIPTION regress_testsub_foo SET (synchronous_commit = foobar);
|
|
|
|
\dRs+
|
|
|
|
-- rename back to keep the rest simple
|
|
ALTER SUBSCRIPTION regress_testsub_foo RENAME TO regress_testsub;
|
|
|
|
-- fail - new owner must be superuser
|
|
ALTER SUBSCRIPTION regress_testsub OWNER TO regress_subscription_user2;
|
|
ALTER ROLE regress_subscription_user2 SUPERUSER;
|
|
-- now it works
|
|
ALTER SUBSCRIPTION regress_testsub OWNER TO regress_subscription_user2;
|
|
|
|
-- fail - cannot do DROP SUBSCRIPTION inside transaction block with slot name
|
|
BEGIN;
|
|
DROP SUBSCRIPTION regress_testsub;
|
|
COMMIT;
|
|
|
|
ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE);
|
|
|
|
-- now it works
|
|
BEGIN;
|
|
DROP SUBSCRIPTION regress_testsub;
|
|
COMMIT;
|
|
|
|
DROP SUBSCRIPTION IF EXISTS regress_testsub;
|
|
DROP SUBSCRIPTION regress_testsub; -- fail
|
|
|
|
-- fail - binary must be boolean
|
|
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, binary = foo);
|
|
|
|
-- now it works
|
|
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, binary = true);
|
|
|
|
\dRs+
|
|
|
|
ALTER SUBSCRIPTION regress_testsub SET (binary = false);
|
|
ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE);
|
|
|
|
\dRs+
|
|
|
|
DROP SUBSCRIPTION regress_testsub;
|
|
|
|
-- fail - streaming must be boolean
|
|
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = foo);
|
|
|
|
-- now it works
|
|
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, streaming = true);
|
|
|
|
\dRs+
|
|
|
|
ALTER SUBSCRIPTION regress_testsub SET (streaming = false);
|
|
ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE);
|
|
|
|
\dRs+
|
|
|
|
DROP SUBSCRIPTION regress_testsub;
|
|
|
|
CREATE SUBSCRIPTION regress_testsub CONNECTION 'dbname=postgres' PUBLICATION mypub
|
|
WITH (enabled = true, create_slot = false, copy_data = false);
|
|
|
|
-- fail - ALTER SUBSCRIPTION with refresh is not allowed in a transaction
|
|
-- block or function
|
|
BEGIN;
|
|
ALTER SUBSCRIPTION regress_testsub SET PUBLICATION mypub WITH (refresh = true);
|
|
END;
|
|
|
|
BEGIN;
|
|
ALTER SUBSCRIPTION regress_testsub REFRESH PUBLICATION;
|
|
END;
|
|
|
|
CREATE FUNCTION func() RETURNS VOID AS
|
|
$$ ALTER SUBSCRIPTION regress_testsub SET PUBLICATION mypub WITH (refresh = true) $$ LANGUAGE SQL;
|
|
SELECT func();
|
|
|
|
ALTER SUBSCRIPTION regress_testsub DISABLE;
|
|
ALTER SUBSCRIPTION regress_testsub SET (slot_name = NONE);
|
|
DROP SUBSCRIPTION regress_testsub;
|
|
DROP FUNCTION func;
|
|
|
|
RESET SESSION AUTHORIZATION;
|
|
DROP ROLE regress_subscription_user;
|
|
DROP ROLE regress_subscription_user2;
|
|
DROP ROLE regress_subscription_user_dummy;
|