database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-28 23:42:10 +03:00
Code Activity

Make WAL segment size configurable at initdb time.

Browse Source 
For performance reasons a larger segment size than the default 16MB
can be useful. A larger segment size has two main benefits: Firstly,
in setups using archiving, it makes it easier to write scripts that
can keep up with higher amounts of WAL, secondly, the WAL has to be
written and synced to disk less frequently.

But at the same time large segment size are disadvantageous for
smaller databases. So far the segment size had to be configured at
compile time, often making it unrealistic to choose one fitting to a
particularly load. Therefore change it to a initdb time setting.

This includes a breaking changes to the xlogreader.h API, which now
requires the current segment size to be configured.  For that and
similar reasons a number of binaries had to be taught how to recognize
the current segment size.

Author: Beena Emerson, editorialized by Andres Freund
Reviewed-By: Andres Freund, David Steele, Kuntal Ghosh, Michael
    Paquier, Peter Eisentraut, Robert Hass, Tushar Ahuja
Discussion: https://postgr.es/m/CAOG9ApEAcQ--1ieKbhFzXSQPw_YLmepaa4hNdnY5+ZULpt81Mw@mail.gmail.com
This commit is contained in:
Andres Freund
2017-09-19 22:03:48 -07:00
parent 5ada1fcd0c
commit fc49e24fa6
46 changed files with 898 additions and 501 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
contrib/pg_standby/pg_standby.c
View File

@ -36,6 +36,8 @@
const char *progname;
int WalSegSz = -1;
/* Options and defaults */
int sleeptime = 5; /* amount of time to sleep between file checks */
int waittime = -1; /* how long we have been waiting, -1 no wait
@ -100,6 +102,10 @@ int nextWALFileType;
struct stat stat_buf;
static bool SetWALFileNameForCleanup(void);
static bool SetWALSegSize(void);
/* =====================================================================
*
* Customizable section
@ -175,6 +181,35 @@ CustomizableNextWALFileReady(void)
{
if (stat(WALFilePath, &stat_buf) == 0)
{
/*
* If we've not seen any WAL segments, we don't know the WAL segment
* size, which we need. If it looks like a WAL segment, determine size
* of segments for the cluster.
*/
if (WalSegSz == -1 && IsXLogFileName(nextWALFileName))
{
if (SetWALSegSize())
{
/*
* Successfully determined WAL segment size. Can compute
* cleanup cutoff now.
*/
need_cleanup = SetWALFileNameForCleanup();
if (debug)
{
fprintf(stderr,
_("WAL segment size: %d \n"), WalSegSz);
fprintf(stderr, "Keep archive history: ");
if (need_cleanup)
fprintf(stderr, "%s and later\n",
exclusiveCleanupFileName);
else
fprintf(stderr, "no cleanup required\n");
}
}
}
/*
* If it's a backup file, return immediately. If it's a regular file
* return only if it's the right size already.
@ -184,7 +219,7 @@ CustomizableNextWALFileReady(void)
nextWALFileType = XLOG_BACKUP_LABEL;
return true;
}
else if (stat_buf.st_size == XLOG_SEG_SIZE)
else if (WalSegSz > 0 && stat_buf.st_size == WalSegSz)
{
#ifdef WIN32
@ -204,7 +239,7 @@ CustomizableNextWALFileReady(void)
/*
* If still too small, wait until it is the correct size
*/
if (stat_buf.st_size > XLOG_SEG_SIZE)
if (WalSegSz > 0 && stat_buf.st_size > WalSegSz)
{
if (debug)
{
@ -218,8 +253,6 @@ CustomizableNextWALFileReady(void)
return false;
}
#define MaxSegmentsPerLogFile ( 0xFFFFFFFF / XLOG_SEG_SIZE )
static void
CustomizableCleanupPriorWALFiles(void)
{
@ -315,6 +348,7 @@ SetWALFileNameForCleanup(void)
uint32 log_diff = 0,
seg_diff = 0;
bool cleanup = false;
int max_segments_per_logfile = (0xFFFFFFFF / WalSegSz);
if (restartWALFileName)
{
@ -336,12 +370,12 @@ SetWALFileNameForCleanup(void)
sscanf(nextWALFileName, "%08X%08X%08X", &tli, &log, &seg);
if (tli > 0 && seg > 0)
{
log_diff = keepfiles / MaxSegmentsPerLogFile;
seg_diff = keepfiles % MaxSegmentsPerLogFile;
log_diff = keepfiles / max_segments_per_logfile;
seg_diff = keepfiles % max_segments_per_logfile;
if (seg_diff > seg)
{
log_diff++;
seg = MaxSegmentsPerLogFile - (seg_diff - seg);
seg = max_segments_per_logfile - (seg_diff - seg);
}
else
seg -= seg_diff;
@ -364,6 +398,66 @@ SetWALFileNameForCleanup(void)
return cleanup;
}
/*
* Try to set the wal segment size from the WAL file specified by WALFilePath.
*
* Return true if size could be determined, false otherwise.
*/
static bool
SetWALSegSize(void)
{
bool ret_val = false;
int fd;
char *buf = (char *) malloc(XLOG_BLCKSZ);
Assert(WalSegSz == -1);
if ((fd = open(WALFilePath, O_RDWR, 0)) < 0)
{
fprintf(stderr, "%s: couldn't open WAL file \"%s\"\n",
progname, WALFilePath);
return false;
}
if (read(fd, buf, XLOG_BLCKSZ) == XLOG_BLCKSZ)
{
XLogLongPageHeader longhdr = (XLogLongPageHeader) buf;
WalSegSz = longhdr->xlp_seg_size;
if (IsValidWalSegSize(WalSegSz))
{
/* successfully retrieved WAL segment size */
ret_val = true;
}
else
fprintf(stderr,
"%s: WAL segment size must be a power of two between 1MB and 1GB, but the WAL file header specifies %d bytes\n",
progname, WalSegSz);
close(fd);
}
else
{
/*
* Don't complain loudly, this is to be expected for segments being
* created.
*/
if (errno != 0)
{
if (debug)
fprintf(stderr, "could not read file \"%s\": %s",
WALFilePath, strerror(errno));
}
else
{
if (debug)
fprintf(stderr, "not enough data in file \"%s\"", WALFilePath);
}
}
fflush(stderr);
return ret_val;
}
/*
* CheckForExternalTrigger()
*
@ -708,8 +802,6 @@ main(int argc, char **argv)
CustomizableInitialize();
need_cleanup = SetWALFileNameForCleanup();
if (debug)
{
fprintf(stderr, "Trigger file: %s\n", triggerPath ? triggerPath : "<not set>");
@ -721,11 +813,6 @@ main(int argc, char **argv)
fprintf(stderr, "Max wait interval: %d %s\n",
maxwaittime, (maxwaittime > 0 ? "seconds" : "forever"));
fprintf(stderr, "Command for restore: %s\n", restoreCommand);
fprintf(stderr, "Keep archive history: ");
if (need_cleanup)
fprintf(stderr, "%s and later\n", exclusiveCleanupFileName);
else
fprintf(stderr, "no cleanup required\n");
fflush(stderr);
}