database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2026-01-27 21:43:08 +03:00
Code Activity

Replace off_t by pgoff_t in I/O routines

Browse Source 
PostgreSQL's Windows port has never been able to handle files larger
than 2GB due to the use of off_t for file offsets, only 32-bit on
Windows.  This causes signed integer overflow at exactly 2^31 bytes when
trying to handle files larger than 2GB, for the routines touched by this
commit.

Note that large files are forbidden by ./configure (3c6248a828) and
meson (recent change, see 79cd66f28c).  This restriction also exists
in v16 and older versions for the now-dead MSVC scripts.

The code base already defines pgoff_t as __int64 (64-bit) on Windows for
this purpose, and some function declarations in headers use it, but many
internals still rely on off_t.  This commit switches more routines to
use pgoff_t, offering more portability, for areas mainly related to file
extensions and storage.

These are not critical for WAL segments yet, which have currently a
maximum size allowed of 1GB (well, this opens the door at allowing a
larger size for them).  This matters more for segment files if we want
to lift the large file restriction in ./configure and meson in the
future, which would make sense to remove once/if all traces of off_t are
gone from the tree.  This can additionally matter for out-of-core code
that may want files larger than 2GB in places where off_t is four bytes
in size.

Note that off_t is still used in other parts of the tree like
buffile.c, WAL sender/receiver, base backup, pg_combinebackup, etc.
These other code paths can be addressed separately, and their update
will be required if we want to remove the large file restriction in the
future.  This commit is a good first cut in itself towards more
portability, hopefully.

On Unix-like systems, pgoff_t is defined as off_t, so this change only
affects Windows behavior.

Author: Bryan Green <dbryan.green@gmail.com>
Reviewed-by: Thomas Munro <thomas.munro@gmail.com>
Reviewed-by: Michael Paquier <michael@paquier.xyz>
Discussion: https://postgr.es/m/0f238ff4-c442-42f5-adb8-01b762c94ca1@gmail.com
This commit is contained in:
Michael Paquier
2025-11-13 12:41:40 +09:00
parent 705601c5ae
commit 84fb27511d
13 changed files with 76 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/include/common/file_utils.h
View File

@@ -55,9 +55,9 @@ extern int compute_remaining_iovec(struct iovec *destination,
extern ssize_t pg_pwritev_with_retry(int fd,
const struct iovec *iov,
int iovcnt,
off_t offset);
pgoff_t offset);
extern ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset);
extern ssize_t pg_pwrite_zeros(int fd, size_t size, pgoff_t offset);
/* Filename components */
#define PG_TEMP_FILES_DIR "pgsql_tmp"

4
src/include/port/pg_iovec.h
View File

@@ -51,7 +51,7 @@ struct iovec
* this changes the current file position.
*/
static inline ssize_t
pg_preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset)
pg_preadv(int fd, const struct iovec *iov, int iovcnt, pgoff_t offset)
{
#if HAVE_DECL_PREADV
/*
@@ -90,7 +90,7 @@ pg_preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset)
* this changes the current file position.
*/
static inline ssize_t
pg_pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset)
pg_pwritev(int fd, const struct iovec *iov, int iovcnt, pgoff_t offset)
{
#if HAVE_DECL_PWRITEV
/*

4
src/include/port/win32_port.h
View File

@@ -584,9 +584,9 @@ typedef unsigned short mode_t;
#endif
/* in port/win32pread.c */
extern ssize_t pg_pread(int fd, void *buf, size_t nbyte, off_t offset);
extern ssize_t pg_pread(int fd, void *buf, size_t nbyte, pgoff_t offset);
/* in port/win32pwrite.c */
extern ssize_t pg_pwrite(int fd, const void *buf, size_t nbyte, off_t offset);
extern ssize_t pg_pwrite(int fd, const void *buf, size_t nbyte, pgoff_t offset);
#endif /* PG_WIN32_PORT_H */

26
src/include/storage/fd.h
View File

@@ -108,17 +108,17 @@ extern File PathNameOpenFile(const char *fileName, int fileFlags);
extern File PathNameOpenFilePerm(const char *fileName, int fileFlags, mode_t fileMode);
extern File OpenTemporaryFile(bool interXact);
extern void FileClose(File file);
extern int FilePrefetch(File file, off_t offset, off_t amount, uint32 wait_event_info);
extern ssize_t FileReadV(File file, const struct iovec *iov, int iovcnt, off_t offset, uint32 wait_event_info);
extern ssize_t FileWriteV(File file, const struct iovec *iov, int iovcnt, off_t offset, uint32 wait_event_info);
extern int FileStartReadV(struct PgAioHandle *ioh, File file, int iovcnt, off_t offset, uint32 wait_event_info);
extern int FilePrefetch(File file, pgoff_t offset, pgoff_t amount, uint32 wait_event_info);
extern ssize_t FileReadV(File file, const struct iovec *iov, int iovcnt, pgoff_t offset, uint32 wait_event_info);
extern ssize_t FileWriteV(File file, const struct iovec *iov, int iovcnt, pgoff_t offset, uint32 wait_event_info);
extern int FileStartReadV(struct PgAioHandle *ioh, File file, int iovcnt, pgoff_t offset, uint32 wait_event_info);
extern int FileSync(File file, uint32 wait_event_info);
extern int FileZero(File file, off_t offset, off_t amount, uint32 wait_event_info);
extern int FileFallocate(File file, off_t offset, off_t amount, uint32 wait_event_info);
extern int FileZero(File file, pgoff_t offset, pgoff_t amount, uint32 wait_event_info);
extern int FileFallocate(File file, pgoff_t offset, pgoff_t amount, uint32 wait_event_info);
extern off_t FileSize(File file);
extern int FileTruncate(File file, off_t offset, uint32 wait_event_info);
extern void FileWriteback(File file, off_t offset, off_t nbytes, uint32 wait_event_info);
extern pgoff_t FileSize(File file);
extern int FileTruncate(File file, pgoff_t offset, uint32 wait_event_info);
extern void FileWriteback(File file, pgoff_t offset, pgoff_t nbytes, uint32 wait_event_info);
extern char *FilePathName(File file);
extern int FileGetRawDesc(File file);
extern int FileGetRawFlags(File file);
@@ -186,8 +186,8 @@ extern int pg_fsync_no_writethrough(int fd);
extern int pg_fsync_writethrough(int fd);
extern int pg_fdatasync(int fd);
extern bool pg_file_exists(const char *name);
extern void pg_flush_data(int fd, off_t offset, off_t nbytes);
extern int pg_truncate(const char *path, off_t length);
extern void pg_flush_data(int fd, pgoff_t offset, pgoff_t nbytes);
extern int pg_truncate(const char *path, pgoff_t length);
extern void fsync_fname(const char *fname, bool isdir);
extern int fsync_fname_ext(const char *fname, bool isdir, bool ignore_perm, int elevel);
extern int durable_rename(const char *oldfile, const char *newfile, int elevel);
@@ -196,7 +196,7 @@ extern void SyncDataDirectory(void);
extern int data_sync_elevel(int elevel);
static inline ssize_t
FileRead(File file, void *buffer, size_t amount, off_t offset,
FileRead(File file, void *buffer, size_t amount, pgoff_t offset,
uint32 wait_event_info)
{
struct iovec iov = {
@@ -208,7 +208,7 @@ FileRead(File file, void *buffer, size_t amount, off_t offset,
}
static inline ssize_t
FileWrite(File file, const void *buffer, size_t amount, off_t offset,
FileWrite(File file, const void *buffer, size_t amount, pgoff_t offset,
uint32 wait_event_info)
{
struct iovec iov = {