From e37b598028469d260650e5e1671c8275d30e22b6 Mon Sep 17 00:00:00 2001 From: Thomas Munro Date: Sat, 31 May 2025 22:50:22 +1200 Subject: [PATCH] Add file_extend_method=posix_fallocate,write_zeros. Provide a way to disable the use of posix_fallocate() for relation files. It was introduced by commit 4d330a61bb1. The new setting file_extend_method=write_zeros can be used as a workaround for problems reported from the field: * BTRFS compression is disabled by the use of posix_fallocate() * XFS could produce spurious ENOSPC errors in some Linux kernel versions, though that problem is reported to have been fixed The default is file_extend_method=posix_fallocate if available, as before. The write_zeros option is similar to PostgreSQL < 16, except that now it's multi-block. Backpatch-through: 16 Reviewed-by: Jakub Wartak Reported-by: Dimitrios Apostolou Discussion: https://postgr.es/m/b1843124-fd22-e279-a31f-252dffb6fbf2%40gmx.net --- doc/src/sgml/config.sgml | 36 +++++++++++++++++++ src/backend/storage/file/fd.c | 3 ++ src/backend/storage/smgr/md.c | 21 ++++++++--- src/backend/utils/misc/guc_tables.c | 19 ++++++++++ src/backend/utils/misc/postgresql.conf.sample | 5 +++ src/include/storage/fd.h | 11 ++++++ 6 files changed, 90 insertions(+), 5 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index d0dc5bac439..f675523e8c8 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -2210,6 +2210,42 @@ include_dir 'conf.d' + + file_extend_method (enum) + + file_extend_method configuration parameter + + + + + Specifies the method used to extend data files during bulk operations + such as COPY. The first available option is used as + the default, depending on the operating system: + + + + posix_fallocate (Unix) uses the standard POSIX + interface for allocating disk space, but is missing on some systems. + If it is present but the underlying file system doesn't support it, + this option silently falls back to write_zeros. + Current versions of BTRFS are known to disable compression when + this option is used. + This is the default on systems that have the function. + + + + + write_zeros extends files by writing out blocks + of zero bytes. This is the default on systems that don't have the + function posix_fallocate. + + + + The write_zeros method is always used when data + files are extended by 8 blocks or fewer. + + + diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index ee91bb79aca..b54c42d9f84 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -164,6 +164,9 @@ bool data_sync_retry = false; /* How SyncDataDirectory() should do its job. */ int recovery_init_sync_method = RECOVERY_INIT_SYNC_METHOD_FSYNC; +/* How data files should be bulk-extended with zeros. */ +int file_extend_method = DEFAULT_FILE_EXTEND_METHOD; + /* Which kinds of files should be opened with PG_O_DIRECT. */ int io_direct_flags; diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 47c01ed7271..e6f1ad21fe8 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -577,13 +577,24 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, * that decision should be made though? For now just use a cutoff of * 8, anything between 4 and 8 worked OK in some local testing. */ - if (numblocks > 8) + if (numblocks > 8 && + file_extend_method != FILE_EXTEND_METHOD_WRITE_ZEROS) { - int ret; + int ret = 0; - ret = FileFallocate(v->mdfd_vfd, - seekpos, (off_t) BLCKSZ * numblocks, - WAIT_EVENT_DATA_FILE_EXTEND); +#ifdef HAVE_POSIX_FALLOCATE + if (file_extend_method == FILE_EXTEND_METHOD_POSIX_FALLOCATE) + { + ret = FileFallocate(v->mdfd_vfd, + seekpos, (off_t) BLCKSZ * numblocks, + WAIT_EVENT_DATA_FILE_EXTEND); + } + else +#endif + { + elog(ERROR, "unsupported file_extend_method: %d", + file_extend_method); + } if (ret != 0) { ereport(ERROR, diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index ab97dea63c4..093567535bd 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -66,6 +66,7 @@ #include "replication/slot.h" #include "replication/syncrep.h" #include "storage/bufmgr.h" +#include "storage/fd.h" #include "storage/large_object.h" #include "storage/pg_shmem.h" #include "storage/predicate.h" @@ -470,6 +471,14 @@ static const struct config_enum_entry wal_compression_options[] = { {NULL, 0, false} }; +static const struct config_enum_entry file_extend_method_options[] = { +#ifdef HAVE_POSIX_FALLOCATE + {"posix_fallocate", FILE_EXTEND_METHOD_POSIX_FALLOCATE, false}, +#endif + {"write_zeros", FILE_EXTEND_METHOD_WRITE_ZEROS, false}, + {NULL, 0, false} +}; + /* * Options for enum values stored in other modules */ @@ -4847,6 +4856,16 @@ struct config_enum ConfigureNamesEnum[] = NULL, NULL, NULL }, + { + {"file_extend_method", PGC_SIGHUP, RESOURCES_DISK, + gettext_noop("Selects the method used for extending data files."), + NULL + }, + &file_extend_method, + DEFAULT_FILE_EXTEND_METHOD, file_extend_method_options, + NULL, NULL, NULL + }, + { {"wal_sync_method", PGC_SIGHUP, WAL_SETTINGS, gettext_noop("Selects the method used for forcing WAL updates to disk."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index de0ed211f38..c8ffcdf7b16 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -167,6 +167,11 @@ #temp_file_limit = -1 # limits per-process temp file space # in kilobytes, or -1 for no limit +#file_extend_method = posix_fallocate # the default is the first option supported + # by the operating system: + # posix_fallocate (most Unix-like systems) + # write_zeros + # - Kernel Resources - #max_files_per_process = 1000 # min 64 diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 6791a406fc1..26809899591 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -59,12 +59,23 @@ typedef int File; #define IO_DIRECT_WAL 0x02 #define IO_DIRECT_WAL_INIT 0x04 +enum FileExtendMethod +{ +#ifdef HAVE_POSIX_FALLOCATE + FILE_EXTEND_METHOD_POSIX_FALLOCATE, +#endif + FILE_EXTEND_METHOD_WRITE_ZEROS, +}; + +/* Default to the first available file_extend_method. */ +#define DEFAULT_FILE_EXTEND_METHOD 0 /* GUC parameter */ extern PGDLLIMPORT int max_files_per_process; extern PGDLLIMPORT bool data_sync_retry; extern PGDLLIMPORT int recovery_init_sync_method; extern PGDLLIMPORT int io_direct_flags; +extern PGDLLIMPORT int file_extend_method; /* * This is private to fd.c, but exported for save/restore_backend_variables()