diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 98b6e83e531..fa613397bc5 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -7765,6 +7765,38 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir' + + data_sync_retry (boolean) + + data_sync_retry configuration parameter + + + + + When set to false, which is the default, PostgreSQL + will raise a PANIC-level error on failure to flush modified data files + to the filesystem. This causes the database server to crash. + + + On some operating systems, the status of data in the kernel's page + cache is unknown after a write-back failure. In some cases it might + have been entirely forgotten, making it unsafe to retry; the second + attempt may be reported as successful, when in fact the data has been + lost. In these circumstances, the only way to avoid data loss is to + recover from the WAL after any failure is reported, preferably + after investigating the root cause of the failure and replacing any + faulty hardware. + + + If set to true, PostgreSQL will instead + report an error but continue to run so that the data flushing + operation can be retried in a later checkpoint. Only set it to true + after investigating the operating system's treatment of buffered data + in case of write-back failure. + + + + diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c index 09b21c927cb..62c7683f148 100644 --- a/src/backend/access/heap/rewriteheap.c +++ b/src/backend/access/heap/rewriteheap.c @@ -977,7 +977,7 @@ logical_end_heap_rewrite(RewriteState state) while ((src = (RewriteMappingFile *) hash_seq_search(&seq_status)) != NULL) { if (FileSync(src->vfd, WAIT_EVENT_LOGICAL_REWRITE_SYNC) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", src->path))); FileClose(src->vfd); @@ -1200,7 +1200,7 @@ heap_xlog_logical_rewrite(XLogReaderState *r) */ pgstat_report_wait_start(WAIT_EVENT_LOGICAL_REWRITE_MAPPING_SYNC); if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", path))); pgstat_report_wait_end(); @@ -1299,7 +1299,7 @@ CheckPointLogicalRewriteHeap(void) */ pgstat_report_wait_start(WAIT_EVENT_LOGICAL_REWRITE_CHECKPOINT_SYNC); if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", path))); pgstat_report_wait_end(); diff --git a/src/backend/access/transam/slru.c b/src/backend/access/transam/slru.c index e0d26b75cf4..56ca77c4074 100644 --- a/src/backend/access/transam/slru.c +++ b/src/backend/access/transam/slru.c @@ -929,7 +929,7 @@ SlruReportIOError(SlruCtl ctl, int pageno, TransactionId xid) path, offset))); break; case SLRU_FSYNC_FAILED: - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not access status of transaction %u", xid), errdetail("Could not fsync file \"%s\": %m.", diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index 63db8a981dc..9975548f4b7 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -407,7 +407,7 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, pgstat_report_wait_start(WAIT_EVENT_TIMELINE_HISTORY_SYNC); if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmppath))); pgstat_report_wait_end(); @@ -487,7 +487,7 @@ writeTimeLineHistoryFile(TimeLineID tli, char *content, int size) pgstat_report_wait_start(WAIT_EVENT_TIMELINE_HISTORY_FILE_SYNC); if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmppath))); pgstat_report_wait_end(); diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 45ca90ca6c2..a93017feecf 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -3445,7 +3445,7 @@ XLogFileCopy(XLogSegNo destsegno, TimeLineID srcTLI, XLogSegNo srcsegno, pgstat_report_wait_start(WAIT_EVENT_WAL_COPY_SYNC); if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmppath))); pgstat_report_wait_end(); diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c index 8549cd506df..67e939883bc 100644 --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@ -1630,6 +1630,9 @@ SnapBuildSerialize(SnapBuild *builder, XLogRecPtr lsn) * fsync the file before renaming so that even if we crash after this we * have either a fully valid file or nothing. * + * It's safe to just ERROR on fsync() here because we'll retry the whole + * operation including the writes. + * * TODO: Do the fsync() via checkpoints/restartpoints, doing it here has * some noticeable overhead since it's performed synchronously during * decoding? diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index 3172092a914..0e50e4f4bac 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -138,6 +138,8 @@ int max_files_per_process = 1000; */ int max_safe_fds = 32; /* default if not changed */ +/* Whether it is safe to continue running after fsync() fails. */ +bool data_sync_retry = false; /* Debugging.... */ @@ -433,11 +435,9 @@ pg_flush_data(int fd, off_t offset, off_t nbytes) */ rc = sync_file_range(fd, offset, nbytes, SYNC_FILE_RANGE_WRITE); - - /* don't error out, this is just a performance optimization */ if (rc != 0) { - ereport(WARNING, + ereport(data_sync_elevel(WARNING), (errcode_for_file_access(), errmsg("could not flush dirty data: %m"))); } @@ -509,7 +509,7 @@ pg_flush_data(int fd, off_t offset, off_t nbytes) rc = msync(p, (size_t) nbytes, MS_ASYNC); if (rc != 0) { - ereport(WARNING, + ereport(data_sync_elevel(WARNING), (errcode_for_file_access(), errmsg("could not flush dirty data: %m"))); /* NB: need to fall through to munmap()! */ @@ -565,7 +565,7 @@ pg_flush_data(int fd, off_t offset, off_t nbytes) void fsync_fname(const char *fname, bool isdir) { - fsync_fname_ext(fname, isdir, false, ERROR); + fsync_fname_ext(fname, isdir, false, data_sync_elevel(ERROR)); } /* @@ -1031,7 +1031,8 @@ LruDelete(File file) * to leak the FD than to mess up our internal state. */ if (close(vfdP->fd)) - elog(LOG, "could not close file \"%s\": %m", vfdP->fileName); + elog(vfdP->fdstate & FD_TEMPORARY ? LOG : data_sync_elevel(LOG), + "could not close file \"%s\": %m", vfdP->fileName); vfdP->fd = VFD_CLOSED; --nfile; @@ -1510,7 +1511,14 @@ FileClose(File file) { /* close the file */ if (close(vfdP->fd)) - elog(LOG, "could not close file \"%s\": %m", vfdP->fileName); + { + /* + * We may need to panic on failure to close non-temporary files; + * see LruDelete. + */ + elog(vfdP->fdstate & FD_TEMPORARY ? LOG : data_sync_elevel(LOG), + "could not close file \"%s\": %m", vfdP->fileName); + } --nfile; vfdP->fd = VFD_CLOSED; @@ -2949,6 +2957,9 @@ looks_like_temp_rel_name(const char *name) * harmless cases such as read-only files in the data directory, and that's * not good either. * + * Note that if we previously crashed due to a PANIC on fsync(), we'll be + * rewriting all changes again during recovery. + * * Note we assume we're chdir'd into PGDATA to begin with. */ void @@ -3235,3 +3246,26 @@ fsync_parent_path(const char *fname, int elevel) return 0; } + +/* + * Return the passed-in error level, or PANIC if data_sync_retry is off. + * + * Failure to fsync any data file is cause for immediate panic, unless + * data_sync_retry is enabled. Data may have been written to the operating + * system and removed from our buffer pool already, and if we are running on + * an operating system that forgets dirty data on write-back failure, there + * may be only one copy of the data remaining: in the WAL. A later attempt to + * fsync again might falsely report success. Therefore we must not allow any + * further checkpoints to be attempted. data_sync_retry can in theory be + * enabled on systems known not to drop dirty buffered data on write-back + * failure (with the likely outcome that checkpoints will continue to fail + * until the underlying problem is fixed). + * + * Any code that reports a failure from fsync() or related functions should + * filter the error level with this function. + */ +int +data_sync_elevel(int elevel) +{ + return data_sync_retry ? elevel : PANIC; +} diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 64455a41550..d08e837c931 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -1040,7 +1040,7 @@ mdimmedsync(SMgrRelation reln, ForkNumber forknum) MdfdVec *v = &reln->md_seg_fds[forknum][segno - 1]; if (FileSync(v->mdfd_vfd, WAIT_EVENT_DATA_FILE_IMMEDIATE_SYNC) < 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", FilePathName(v->mdfd_vfd)))); @@ -1285,7 +1285,7 @@ mdsync(void) bms_join(new_requests, requests); errno = save_errno; - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", path))); @@ -1459,7 +1459,7 @@ register_dirty_segment(SMgrRelation reln, ForkNumber forknum, MdfdVec *seg) (errmsg("could not forward fsync request because request queue is full"))); if (FileSync(seg->mdfd_vfd, WAIT_EVENT_DATA_FILE_SYNC) < 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", FilePathName(seg->mdfd_vfd)))); diff --git a/src/backend/utils/cache/relmapper.c b/src/backend/utils/cache/relmapper.c index f5394dc43d4..8d4215cb981 100644 --- a/src/backend/utils/cache/relmapper.c +++ b/src/backend/utils/cache/relmapper.c @@ -798,7 +798,7 @@ write_relmap_file(bool shared, RelMapFile *newmap, */ pgstat_report_wait_start(WAIT_EVENT_RELATION_MAP_SYNC); if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync relation mapping file \"%s\": %m", mapfilename))); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 40da074d7b2..4298a0dde23 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -1667,6 +1667,15 @@ static struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"data_sync_retry", PGC_POSTMASTER, ERROR_HANDLING_OPTIONS, + gettext_noop("Whether to continue running after a failure to sync data files."), + }, + &data_sync_retry, + false, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 22923350581..3e68afd2750 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -635,6 +635,7 @@ #exit_on_error = off # terminate session on any error? #restart_after_crash = on # reinitialize after backend crash? +#data_sync_retry = off # retry or panic on failure to fsync data? #------------------------------------------------------------------------------ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index ffb3f4e10f1..3043af31b01 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -53,6 +53,7 @@ typedef int File; /* GUC parameter */ extern PGDLLIMPORT int max_files_per_process; +extern PGDLLIMPORT bool data_sync_retry; /* * This is private to fd.c, but exported for save/restore_backend_variables() @@ -124,6 +125,7 @@ extern int durable_rename(const char *oldfile, const char *newfile, int loglevel extern int durable_unlink(const char *fname, int loglevel); extern int durable_link_or_rename(const char *oldfile, const char *newfile, int loglevel); extern void SyncDataDirectory(void); +extern int data_sync_elevel(int elevel); /* Filename components for OpenTemporaryFile */ #define PG_TEMP_FILES_DIR "pgsql_tmp"