From 48982dedc18ea35db5530a81beb3405ad264d89a Mon Sep 17 00:00:00 2001 From: laishenghao Date: Mon, 2 Sep 2024 10:08:49 +0800 Subject: [PATCH] =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E5=8F=AF=E8=A7=81=E6=80=A7?= =?UTF-8?q?=E4=B8=8Evacuum=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/gs_guc/cluster_guc.conf | 1 + .../backend/utils/misc/guc/guc_storage.cpp | 14 +++ .../utils/misc/postgresql_single.conf.sample | 1 + src/gausskernel/optimizer/commands/vacuum.cpp | 7 +- .../storage/access/heap/heapam.cpp | 50 ++++++----- src/gausskernel/storage/ipc/procarray.cpp | 55 ++++++++++++ .../knl/knl_guc/knl_session_attr_storage.h | 1 + src/include/storage/procarray.h | 1 + .../regress/expected/single_node_vacuum.out | 85 +++++++++++++++++++ src/test/regress/sql/single_node_vacuum.sql | 30 +++++++ 10 files changed, 221 insertions(+), 24 deletions(-) diff --git a/src/bin/gs_guc/cluster_guc.conf b/src/bin/gs_guc/cluster_guc.conf index 87ebb5d0f..1a7f7ed28 100755 --- a/src/bin/gs_guc/cluster_guc.conf +++ b/src/bin/gs_guc/cluster_guc.conf @@ -616,6 +616,7 @@ plan_cache_mode|enum|auto,force_generic_plan,force_custom_plan|NULL|NULL| plan_cache_type_validation|bool|0,0|NULL|NULL| remote_read_mode|enum|off,non_authentication,authentication|NULL|NULL| enable_debug_vacuum|bool|0,0|NULL|NULL| +enable_vacuum_extreme_xmin|bool|0,0|NULL|Use extreme xmin to vacuum.| enable_early_free|bool|0,0|NULL|NULL| resource_track_cost|int|-1,2147483647|NULL|NULL| resource_track_duration|int|0,2147483647|s|NULL| diff --git a/src/common/backend/utils/misc/guc/guc_storage.cpp b/src/common/backend/utils/misc/guc/guc_storage.cpp index ed30d2721..a96661af3 100755 --- a/src/common/backend/utils/misc/guc/guc_storage.cpp +++ b/src/common/backend/utils/misc/guc/guc_storage.cpp @@ -842,6 +842,20 @@ static void InitStorageConfigureNamesBool() NULL, NULL, NULL}, + + {{"enable_vacuum_extreme_xmin", + PGC_SIGHUP, + NODE_SINGLENODE, + AUTOVACUUM, + gettext_noop("Use extreme xmin to vacuum."), + NULL, + }, + &u_sess->attr.attr_storage.enableVacuumExtremeXmin, + false, + NULL, + NULL, + NULL}, + {{"enable_adio_debug", PGC_SUSET, NODE_ALL, diff --git a/src/common/backend/utils/misc/postgresql_single.conf.sample b/src/common/backend/utils/misc/postgresql_single.conf.sample index d37d95ad3..bfb4b527e 100644 --- a/src/common/backend/utils/misc/postgresql_single.conf.sample +++ b/src/common/backend/utils/misc/postgresql_single.conf.sample @@ -313,6 +313,7 @@ synchronous_standby_names = '*' # standby servers that provide sync rep # as standbalone after sync standby failure # It's global control for all transactions #vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed +#enable_vacuum_extreme_xmin = off # Use extreme xmin to vacuum #data_replicate_buffer_size = 16MB # data replication buffer size walsender_max_send_size = 8MB # Size of walsender max send size #enable_data_replicate = on diff --git a/src/gausskernel/optimizer/commands/vacuum.cpp b/src/gausskernel/optimizer/commands/vacuum.cpp index 73196b092..e04b1258c 100644 --- a/src/gausskernel/optimizer/commands/vacuum.cpp +++ b/src/gausskernel/optimizer/commands/vacuum.cpp @@ -1100,7 +1100,12 @@ void vacuum_set_xid_limits(Relation rel, int64 freeze_min_age, int64 freeze_tabl * working on a particular table at any time, and that each vacuum is * always an independent transaction. */ - *oldestXmin = GetOldestXmin(rel); + if (u_sess->attr.attr_storage.enableVacuumExtremeXmin) { + *oldestXmin = GetVacuumExtremeOldestXmin(); + } else { + *oldestXmin = GetOldestXmin(rel); + } + if (IsCatalogRelation(rel) || RelationIsAccessibleInLogicalDecoding(rel)) { TransactionId CatalogXmin = GetReplicationSlotCatalogXmin(); if (TransactionIdIsNormal(CatalogXmin) && TransactionIdPrecedes(CatalogXmin, *oldestXmin)) { diff --git a/src/gausskernel/storage/access/heap/heapam.cpp b/src/gausskernel/storage/access/heap/heapam.cpp index 02385ea1d..c572d1348 100755 --- a/src/gausskernel/storage/access/heap/heapam.cpp +++ b/src/gausskernel/storage/access/heap/heapam.cpp @@ -395,36 +395,40 @@ void heapgetpage(TableScanDesc sscan, BlockNumber page, bool* has_cur_xact_write */ all_visible = PageIsAllVisible(dp) && !snapshot->takenDuringRecovery; - for (line_off = FirstOffsetNumber, lpp = HeapPageGetItemId(dp, line_off); line_off <= lines; line_off++, lpp++) { - if (ItemIdIsNormal(lpp)) { - HeapTupleData loctup; - bool valid = false; + bool isSerializableXact = IsSerializableXact(); - if (likely(all_visible && (!IsSerializableXact()))) { + if (all_visible && !isSerializableXact) { + for (line_off = FirstOffsetNumber, lpp = HeapPageGetItemId(dp, line_off); line_off <= lines; line_off++, lpp++) { + if (ItemIdIsNormal(lpp)) { scan->rs_base.rs_vistuples[ntup++] = line_off; - continue; } + } + } else { + HeapTupleData loctup; + bool valid = false; + bool shouldLog = (log_min_messages <= DEBUG1); + for (line_off = FirstOffsetNumber, lpp = HeapPageGetItemId(dp, line_off); line_off <= lines; line_off++, lpp++) { + if (ItemIdIsNormal(lpp)) { + loctup.t_tableOid = RelationGetRelid(scan->rs_base.rs_rd); + loctup.t_bucketId = RelationGetBktid(scan->rs_base.rs_rd); + loctup.t_data = (HeapTupleHeader)PageGetItem((Page)dp, lpp); + loctup.t_len = ItemIdGetLength(lpp); + HeapTupleCopyBaseFromPage(&loctup, dp); + ItemPointerSet(&(loctup.t_self), page, line_off); - loctup.t_tableOid = RelationGetRelid(scan->rs_base.rs_rd); - loctup.t_bucketId = RelationGetBktid(scan->rs_base.rs_rd); - loctup.t_data = (HeapTupleHeader)PageGetItem((Page)dp, lpp); - loctup.t_len = ItemIdGetLength(lpp); - HeapTupleCopyBaseFromPage(&loctup, dp); - ItemPointerSet(&(loctup.t_self), page, line_off); - - if (all_visible) - valid = true; - else valid = HeapTupleSatisfiesVisibility(&loctup, snapshot, buffer, has_cur_xact_write); + if (unlikely(isSerializableXact)) { + CheckForSerializableConflictOut(valid, scan->rs_base.rs_rd, (void*)&loctup, buffer, snapshot); + } + if (valid) { + scan->rs_base.rs_vistuples[ntup++] = line_off; + } - CheckForSerializableConflictOut(valid, scan->rs_base.rs_rd, (void*)&loctup, buffer, snapshot); - - if (valid) { - scan->rs_base.rs_vistuples[ntup++] = line_off; + if (unlikely(shouldLog)) { + ereport(DEBUG1, (errmsg("heapgetpage xid %lu ctid(%u,%d) valid %d", + GetCurrentTransactionIdIfAny(), page, line_off, valid))); + } } - - ereport(DEBUG1, (errmsg("heapgetpage xid %lu ctid(%u,%d) valid %d", GetCurrentTransactionIdIfAny(), page, - line_off, valid))); } } diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index 4f012ce0a..9795f6a71 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -5540,3 +5540,58 @@ void GetOldestGlobalProcXmin(TransactionId *globalProcXmin) } LWLockRelease(ProcArrayLock); } + +TransactionId GetVacuumExtremeOldestXmin() +{ + pg_read_barrier(); + (void)LWLockAcquire(ProcArrayLock, LW_SHARED); + + /* initialize minXmin calculation with xmax: latestCompletedXid + 1 */ + TransactionId minXmin = t_thrd.xact_cxt.ShmemVariableCache->latestCompletedXid; + Assert(TransactionIdIsNormal(minXmin)); + TransactionIdAdvance(minXmin); + + /* + * Spin over procArray checking xid, xmin, and subxids. The goal is + * to gather all active xids, find the lowest xmin, and try to record subxids. + */ + ProcArrayStruct* arrayP = g_instance.proc_array_idx; + int* pgprocnos = arrayP->pgprocnos; + int numProcs = arrayP->numProcs; + TransactionId xid = InvalidTransactionId; + for (int i = 0; i < numProcs; i++) { + int procNo = pgprocnos[i]; + volatile PGXACT* pgxact = &g_instance.proc_base_all_xacts[procNo]; + + /* + * Ignore procs doing logical decoding which manages xmin + * separately or running LAZY VACUUM + */ + if ((pgxact->vacuumFlags & PROC_IN_LOGICAL_DECODING) || + (pgxact->vacuumFlags & PROC_IN_VACUUM)) { + continue; + } + + /* Fetch xid just once - see GetNewTransactionId */ + xid = pgxact->xid; + /* If no XID assigned, use xid passed down from CN */ + if (!TransactionIdIsNormal(xid)) { + xid = pgxact->next_xid; + } + if (TransactionIdIsNormal(xid) && + TransactionIdPrecedes(xid, minXmin)) { + minXmin = xid; + } + } + + uint64 deferAge = (uint64)u_sess->attr.attr_storage.vacuum_defer_cleanup_age; + if (TransactionIdPrecedes(minXmin, deferAge)) { + minXmin = FirstNormalTransactionId; + } else { + minXmin -= deferAge; + } + + LWLockRelease(ProcArrayLock); + + return minXmin; +} diff --git a/src/include/knl/knl_guc/knl_session_attr_storage.h b/src/include/knl/knl_guc/knl_session_attr_storage.h index 7aff20299..0ac0b7eaf 100755 --- a/src/include/knl/knl_guc/knl_session_attr_storage.h +++ b/src/include/knl/knl_guc/knl_session_attr_storage.h @@ -112,6 +112,7 @@ typedef struct knl_session_attr_storage { bool guc_most_available_sync; bool enable_show_any_tuples; bool enable_debug_vacuum; + bool enableVacuumExtremeXmin; bool enable_adio_debug; bool gds_debug_mod; bool log_pagewriter; diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h index 164018c89..ba87fcd8b 100755 --- a/src/include/storage/procarray.h +++ b/src/include/storage/procarray.h @@ -96,6 +96,7 @@ extern TransactionId GetOldestCatalogXmin(); extern TransactionId GetRecentGlobalXmin(void); extern TransactionId GetOldestXmin(Relation rel, bool bFixRecentGlobalXmin = false, bool bRecentGlobalXminNoCheck = false); +extern TransactionId GetVacuumExtremeOldestXmin(); extern TransactionId GetGlobalOldestXmin(void); extern TransactionId GetOldestXminForUndo(TransactionId * recycleXmin); extern void CheckCurrentTimeline(GTM_Timeline timeline); diff --git a/src/test/regress/expected/single_node_vacuum.out b/src/test/regress/expected/single_node_vacuum.out index 262843422..cdccad4d0 100644 --- a/src/test/regress/expected/single_node_vacuum.out +++ b/src/test/regress/expected/single_node_vacuum.out @@ -84,3 +84,88 @@ VACUUM ANALYZE vaccluster(i,i); ANALYZE vaccluster(i,i); DROP TABLE vaccluster; DROP TABLE vactst; +-- test vacuum opt +set enable_vacuum_extreme_xmin=off; -- should error +ERROR: parameter "enable_vacuum_extreme_xmin" cannot be changed now +set enable_vacuum_extreme_xmin=on; -- should error +ERROR: parameter "enable_vacuum_extreme_xmin" cannot be changed now +alter system set enable_vacuum_extreme_xmin=on; +select pg_sleep(1); + pg_sleep +---------- + +(1 row) + +show enable_vacuum_extreme_xmin; + enable_vacuum_extreme_xmin +---------------------------- + on +(1 row) + +drop table if exists vac_opt_t; +NOTICE: table "vac_opt_t" does not exist, skipping +drop table if exists vac_opt_t2; +NOTICE: table "vac_opt_t2" does not exist, skipping +create table vac_opt_t(c1 int); +insert into vac_opt_t values(generate_series(1,10)); +vacuum vac_opt_t; +insert into vac_opt_t values(generate_series(1,10)); +vacuum analyze vac_opt_t; +delete from vac_opt_t where c1 < 5; +vacuum full vac_opt_t ; +select * from vac_opt_t order by c1; + c1 +---- + 5 + 5 + 6 + 6 + 7 + 7 + 8 + 8 + 9 + 9 + 10 + 10 +(12 rows) + +update vac_opt_t set c1 = -1; +create table vac_opt_t2(c1 int, c2 varchar(20)); +insert into vac_opt_t2 values(generate_series(1,1000), 'hello'); +Delete from vac_opt_t2 where c1 in (8, 9, 10); +select * from vac_opt_t order by c1; + c1 +---- + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 +(12 rows) + +select * from vac_opt_t2 order by c1 limit 10; + c1 | c2 +----+------- + 1 | hello + 2 | hello + 3 | hello + 4 | hello + 5 | hello + 6 | hello + 7 | hello + 11 | hello + 12 | hello + 13 | hello +(10 rows) + +alter system set enable_vacuum_extreme_xmin=off; +drop table vac_opt_t; +drop table vac_opt_t2; diff --git a/src/test/regress/sql/single_node_vacuum.sql b/src/test/regress/sql/single_node_vacuum.sql index b2f35804b..be34e3ce0 100644 --- a/src/test/regress/sql/single_node_vacuum.sql +++ b/src/test/regress/sql/single_node_vacuum.sql @@ -68,3 +68,33 @@ ANALYZE vaccluster(i,i); DROP TABLE vaccluster; DROP TABLE vactst; + +-- test vacuum opt +set enable_vacuum_extreme_xmin=off; -- should error +set enable_vacuum_extreme_xmin=on; -- should error + +alter system set enable_vacuum_extreme_xmin=on; +select pg_sleep(1); +show enable_vacuum_extreme_xmin; +drop table if exists vac_opt_t; +drop table if exists vac_opt_t2; +create table vac_opt_t(c1 int); +insert into vac_opt_t values(generate_series(1,10)); + +vacuum vac_opt_t; +insert into vac_opt_t values(generate_series(1,10)); +vacuum analyze vac_opt_t; +delete from vac_opt_t where c1 < 5; +vacuum full vac_opt_t ; +select * from vac_opt_t order by c1; +update vac_opt_t set c1 = -1; +create table vac_opt_t2(c1 int, c2 varchar(20)); +insert into vac_opt_t2 values(generate_series(1,1000), 'hello'); +Delete from vac_opt_t2 where c1 in (8, 9, 10); + +select * from vac_opt_t order by c1; +select * from vac_opt_t2 order by c1 limit 10; + +alter system set enable_vacuum_extreme_xmin=off; +drop table vac_opt_t; +drop table vac_opt_t2;