diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 54dfa14963b..d4ef8869f15 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -7512,6 +7512,38 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir' + + data_sync_retry (boolean) + + data_sync_retry configuration parameter + + + + + When set to false, which is the default, PostgreSQL + will raise a PANIC-level error on failure to flush modified data files + to the filesystem. This causes the database server to crash. + + + On some operating systems, the status of data in the kernel's page + cache is unknown after a write-back failure. In some cases it might + have been entirely forgotten, making it unsafe to retry; the second + attempt may be reported as successful, when in fact the data has been + lost. In these circumstances, the only way to avoid data loss is to + recover from the WAL after any failure is reported, preferably + after investigating the root cause of the failure and replacing any + faulty hardware. + + + If set to true, PostgreSQL will instead + report an error but continue to run so that the data flushing + operation can be retried in a later checkpoint. Only set it to true + after investigating the operating system's treatment of buffered data + in case of write-back failure. + + + + diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c index e700824d12d..93ab5249b64 100644 --- a/src/backend/access/heap/rewriteheap.c +++ b/src/backend/access/heap/rewriteheap.c @@ -974,7 +974,7 @@ logical_end_heap_rewrite(RewriteState state) while ((src = (RewriteMappingFile *) hash_seq_search(&seq_status)) != NULL) { if (FileSync(src->vfd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", src->path))); FileClose(src->vfd); @@ -1192,7 +1192,7 @@ heap_xlog_logical_rewrite(XLogReaderState *r) * doesn't seem worth the trouble. */ if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", path))); @@ -1289,7 +1289,7 @@ CheckPointLogicalRewriteHeap(void) * but it's currently not deemed worth the effort. */ else if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", path))); CloseTransientFile(fd); diff --git a/src/backend/access/transam/slru.c b/src/backend/access/transam/slru.c index 9789d470780..e708967e2f3 100644 --- a/src/backend/access/transam/slru.c +++ b/src/backend/access/transam/slru.c @@ -921,7 +921,7 @@ SlruReportIOError(SlruCtl ctl, int pageno, TransactionId xid) path, offset))); break; case SLRU_FSYNC_FAILED: - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not access status of transaction %u", xid), errdetail("Could not fsync file \"%s\": %m.", diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index bd91573708b..5d73cc942cd 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -402,7 +402,7 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, } if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmppath))); @@ -478,7 +478,7 @@ writeTimeLineHistoryFile(TimeLineID tli, char *content, int size) } if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmppath))); diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index d7505aa454b..004700febeb 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -3283,7 +3283,7 @@ XLogFileCopy(XLogSegNo destsegno, TimeLineID srcTLI, XLogSegNo srcsegno, } if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmppath))); diff --git a/src/backend/replication/logical/snapbuild.c b/src/backend/replication/logical/snapbuild.c index d8d8f03cdf9..69ab728ed67 100644 --- a/src/backend/replication/logical/snapbuild.c +++ b/src/backend/replication/logical/snapbuild.c @@ -1606,6 +1606,9 @@ SnapBuildSerialize(SnapBuild *builder, XLogRecPtr lsn) * fsync the file before renaming so that even if we crash after this we * have either a fully valid file or nothing. * + * It's safe to just ERROR on fsync() here because we'll retry the whole + * operation including the writes. + * * TODO: Do the fsync() via checkpoints/restartpoints, doing it here has * some noticeable overhead since it's performed synchronously during * decoding? diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index 9354652d914..96c884a0634 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -138,6 +138,8 @@ int max_files_per_process = 1000; */ int max_safe_fds = 32; /* default if not changed */ +/* Whether it is safe to continue running after fsync() fails. */ +bool data_sync_retry = false; /* Debugging.... */ @@ -433,11 +435,9 @@ pg_flush_data(int fd, off_t offset, off_t nbytes) */ rc = sync_file_range(fd, offset, nbytes, SYNC_FILE_RANGE_WRITE); - - /* don't error out, this is just a performance optimization */ if (rc != 0) { - ereport(WARNING, + ereport(data_sync_elevel(WARNING), (errcode_for_file_access(), errmsg("could not flush dirty data: %m"))); } @@ -509,7 +509,7 @@ pg_flush_data(int fd, off_t offset, off_t nbytes) rc = msync(p, (size_t) nbytes, MS_ASYNC); if (rc != 0) { - ereport(WARNING, + ereport(data_sync_elevel(WARNING), (errcode_for_file_access(), errmsg("could not flush dirty data: %m"))); /* NB: need to fall through to munmap()! */ @@ -565,7 +565,7 @@ pg_flush_data(int fd, off_t offset, off_t nbytes) void fsync_fname(const char *fname, bool isdir) { - fsync_fname_ext(fname, isdir, false, ERROR); + fsync_fname_ext(fname, isdir, false, data_sync_elevel(ERROR)); } /* @@ -994,7 +994,8 @@ LruDelete(File file) * to leak the FD than to mess up our internal state. */ if (close(vfdP->fd)) - elog(LOG, "could not close file \"%s\": %m", vfdP->fileName); + elog(vfdP->fdstate & FD_TEMPORARY ? LOG : data_sync_elevel(LOG), + "could not close file \"%s\": %m", vfdP->fileName); vfdP->fd = VFD_CLOSED; --nfile; @@ -1473,7 +1474,14 @@ FileClose(File file) { /* close the file */ if (close(vfdP->fd)) - elog(LOG, "could not close file \"%s\": %m", vfdP->fileName); + { + /* + * We may need to panic on failure to close non-temporary files; + * see LruDelete. + */ + elog(vfdP->fdstate & FD_TEMPORARY ? LOG : data_sync_elevel(LOG), + "could not close file \"%s\": %m", vfdP->fileName); + } --nfile; vfdP->fd = VFD_CLOSED; @@ -2898,6 +2906,9 @@ looks_like_temp_rel_name(const char *name) * harmless cases such as read-only files in the data directory, and that's * not good either. * + * Note that if we previously crashed due to a PANIC on fsync(), we'll be + * rewriting all changes again during recovery. + * * Note we assume we're chdir'd into PGDATA to begin with. */ void @@ -3184,3 +3195,26 @@ fsync_parent_path(const char *fname, int elevel) return 0; } + +/* + * Return the passed-in error level, or PANIC if data_sync_retry is off. + * + * Failure to fsync any data file is cause for immediate panic, unless + * data_sync_retry is enabled. Data may have been written to the operating + * system and removed from our buffer pool already, and if we are running on + * an operating system that forgets dirty data on write-back failure, there + * may be only one copy of the data remaining: in the WAL. A later attempt to + * fsync again might falsely report success. Therefore we must not allow any + * further checkpoints to be attempted. data_sync_retry can in theory be + * enabled on systems known not to drop dirty buffered data on write-back + * failure (with the likely outcome that checkpoints will continue to fail + * until the underlying problem is fixed). + * + * Any code that reports a failure from fsync() or related functions should + * filter the error level with this function. + */ +int +data_sync_elevel(int elevel) +{ + return data_sync_retry ? elevel : PANIC; +} diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 91a290c357f..b34a2e552a0 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -1037,7 +1037,7 @@ mdimmedsync(SMgrRelation reln, ForkNumber forknum) while (v != NULL) { if (FileSync(v->mdfd_vfd) < 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", FilePathName(v->mdfd_vfd)))); @@ -1282,7 +1282,7 @@ mdsync(void) bms_join(new_requests, requests); errno = save_errno; - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", path))); @@ -1456,7 +1456,7 @@ register_dirty_segment(SMgrRelation reln, ForkNumber forknum, MdfdVec *seg) (errmsg("could not forward fsync request because request queue is full"))); if (FileSync(seg->mdfd_vfd) < 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", FilePathName(seg->mdfd_vfd)))); diff --git a/src/backend/utils/cache/relmapper.c b/src/backend/utils/cache/relmapper.c index 6a63b5e1d74..6010594f6e6 100644 --- a/src/backend/utils/cache/relmapper.c +++ b/src/backend/utils/cache/relmapper.c @@ -792,7 +792,7 @@ write_relmap_file(bool shared, RelMapFile *newmap, * CheckPointRelationMap. */ if (pg_fsync(fd) != 0) - ereport(ERROR, + ereport(data_sync_elevel(ERROR), (errcode_for_file_access(), errmsg("could not fsync relation mapping file \"%s\": %m", mapfilename))); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 5c27397ccc8..ec83d1dd456 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -1658,6 +1658,15 @@ static struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"data_sync_retry", PGC_POSTMASTER, ERROR_HANDLING_OPTIONS, + gettext_noop("Whether to continue running after a failure to sync data files."), + }, + &data_sync_retry, + false, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b6b683ec280..756dfb396a7 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -620,6 +620,7 @@ #exit_on_error = off # terminate session on any error? #restart_after_crash = on # reinitialize after backend crash? +#data_sync_retry = off # retry or panic on failure to fsync data? #------------------------------------------------------------------------------ diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index b6b5c7ab4b2..ee063f7cc48 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -53,6 +53,7 @@ typedef int File; /* GUC parameter */ extern PGDLLIMPORT int max_files_per_process; +extern PGDLLIMPORT bool data_sync_retry; /* * This is private to fd.c, but exported for save/restore_backend_variables() @@ -123,6 +124,7 @@ extern void fsync_fname(const char *fname, bool isdir); extern int durable_rename(const char *oldfile, const char *newfile, int loglevel); extern int durable_link_or_rename(const char *oldfile, const char *newfile, int loglevel); extern void SyncDataDirectory(void); +extern int data_sync_elevel(int elevel); /* Filename components for OpenTemporaryFile */ #define PG_TEMP_FILES_DIR "pgsql_tmp"