database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-11-10 17:42:29 +03:00
Code Activity

Support base backup targets.

Browse Source 
pg_basebackup now has a --target=TARGET[:DETAIL] option. If specfied,
it is sent to the server as the value of the TARGET option to the
BASE_BACKUP command. If DETAIL is included, it is sent as the value of
the new TARGET_DETAIL option to the BASE_BACKUP command.  If the
target is anything other than 'client', pg_basebackup assumes that it
will now be the server's job to write the backup in a location somehow
defined by the target, and that it therefore needs to write nothing
locally. However, the server will still send messages to the client
for progress reporting purposes.

On the server side, we now support two additional types of backup
targets.  There is a 'blackhole' target, which just throws away the
backup data without doing anything at all with it. Naturally, this
should only be used for testing and debugging purposes, since you will
not actually have a backup when it finishes running. More usefully,
there is also a 'server' target, so you can now use something like
'pg_basebackup -Xnone -t server:/SOME/PATH' to write a backup to some
location on the server. We can extend this to more types of targets
in the future, and might even want to create an extensibility
mechanism for adding new target types.

Since WAL fetching is handled with separate client-side logic, it's
not part of this mechanism; thus, backups with non-default targets
must use -Xnone or -Xfetch.

Patch by me, with a bug fix by Jeevan Ladhe.  The patch set of which
this is a part has also had review and/or testing from Tushar Ahuja,
Suraj Kharage, Dipesh Pandit, and Mark Dilger.

Discussion: http://postgr.es/m/CA+TgmoaYZbz0=Yk797aOJwkGJC-LK3iXn+wzzMx7KdwNpZhS5g@mail.gmail.com
This commit is contained in:
Robert Haas
2021-11-16 15:20:50 -05:00
parent f80900be06
commit 3500ccc39b
11 changed files with 677 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
src/bin/pg_basebackup/pg_basebackup.c
View File

@@ -115,7 +115,7 @@ typedef enum
static char *basedir = NULL;
static TablespaceList tablespace_dirs = {NULL, NULL};
static char *xlog_dir = NULL;
static char format = 'p'; /* p(lain)/t(ar) */
static char format = '\0'; /* p(lain)/t(ar) */
static char *label = "pg_basebackup base backup";
static bool noclean = false;
static bool checksum_failure = false;
@@ -132,6 +132,7 @@ static pg_time_t last_progress_report = 0;
static int32 maxrate = 0; /* no limit by default */
static char *replication_slot = NULL;
static bool temp_replication_slot = true;
static char *backup_target = NULL;
static bool create_slot = false;
static bool no_slot = false;
static bool verify_checksums = true;
@@ -364,6 +365,8 @@ usage(void)
printf(_("Usage:\n"));
printf(_(" %s [OPTION]...\n"), progname);
printf(_("\nOptions controlling the output:\n"));
printf(_(" -t, --target=TARGET[:DETAIL]\n"
" backup target (if other than client)\n"));
printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n"));
printf(_(" -F, --format=p|t output format (plain (default), tar)\n"));
printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer data directory\n"
@@ -1232,15 +1235,22 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data)
}
/*
* Create an appropriate backup streamer. We know that
* recovery GUCs are supported, because this protocol can only
* be used on v15+.
* Create an appropriate backup streamer, unless a backup
* target was specified. In that case, it's up to the server
* to put the backup wherever it needs to go.
*/
state->streamer =
CreateBackupStreamer(archive_name,
spclocation,
&state->manifest_inject_streamer,
true, false);
if (backup_target == NULL)
{
/*
* We know that recovery GUCs are supported, because this
* protocol can only be used on v15+.
*/
state->streamer =
CreateBackupStreamer(archive_name,
spclocation,
&state->manifest_inject_streamer,
true, false);
}
break;
}
@@ -1312,24 +1322,32 @@ ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data)
GetCopyDataEnd(r, copybuf, cursor);
/*
* If we're supposed inject the manifest into the archive, we
* prepare to buffer it in memory; otherwise, we prepare to
* write it to a temporary file.
* If a backup target was specified, figuring out where to put
* the manifest is the server's problem. Otherwise, we need to
* deal with it.
*/
if (state->manifest_inject_streamer != NULL)
state->manifest_buffer = createPQExpBuffer();
else
if (backup_target == NULL)
{
snprintf(state->manifest_filename,
sizeof(state->manifest_filename),
"%s/backup_manifest.tmp", basedir);
state->manifest_file =
fopen(state->manifest_filename, "wb");
if (state->manifest_file == NULL)
/*
* If we're supposed inject the manifest into the archive,
* we prepare to buffer it in memory; otherwise, we
* prepare to write it to a temporary file.
*/
if (state->manifest_inject_streamer != NULL)
state->manifest_buffer = createPQExpBuffer();
else
{
pg_log_error("could not create file \"%s\": %m",
state->manifest_filename);
exit(1);
snprintf(state->manifest_filename,
sizeof(state->manifest_filename),
"%s/backup_manifest.tmp", basedir);
state->manifest_file =
fopen(state->manifest_filename, "wb");
if (state->manifest_file == NULL)
{
pg_log_error("could not create file \"%s\": %m",
state->manifest_filename);
exit(1);
}
}
}
break;
@@ -1698,13 +1716,41 @@ BaseBackup(void)
if (manifest)
{
AppendStringCommandOption(&buf, use_new_option_syntax, "MANIFEST",
manifest_force_encode ? "force-encode" : "yes");
manifest_force_encode ? "force-encode" : "yes");
if (manifest_checksums != NULL)
AppendStringCommandOption(&buf, use_new_option_syntax,
"MANIFEST_CHECKSUMS", manifest_checksums);
"MANIFEST_CHECKSUMS", manifest_checksums);
}
if (serverMajor >= 1500)
if (backup_target != NULL)
{
char *colon;
if (serverMajor < 1500)
{
pg_log_error("backup targets are not supported by this server version");
exit(1);
}
AppendPlainCommandOption(&buf, use_new_option_syntax, "TABLESPACE_MAP");
if ((colon = strchr(backup_target, ':')) == NULL)
{
AppendStringCommandOption(&buf, use_new_option_syntax,
"TARGET", backup_target);
}
else
{
char *target;
target = pnstrdup(backup_target, colon - backup_target);
AppendStringCommandOption(&buf, use_new_option_syntax,
"TARGET", target);
AppendStringCommandOption(&buf, use_new_option_syntax,
"TARGET_DETAIL", colon + 1);
}
}
else if (serverMajor >= 1500)
AppendStringCommandOption(&buf, use_new_option_syntax,
"TARGET", "client");
@@ -1799,8 +1845,13 @@ BaseBackup(void)
* Verify tablespace directories are empty. Don't bother with the
* first once since it can be relocated, and it will be checked before
* we do anything anyway.
*
* Note that this is skipped for tar format backups and backups that
* the server is storing to a target location, since in that case
* we won't be storing anything into these directories and thus should
* not create them.
*/
if (format == 'p' && !PQgetisnull(res, i, 1))
if (backup_target == NULL && format == 'p' && !PQgetisnull(res, i, 1))
{
char *path = unconstify(char *, get_tablespace_mapping(PQgetvalue(res, i, 1)));
@@ -1811,7 +1862,8 @@ BaseBackup(void)
/*
* When writing to stdout, require a single tablespace
*/
writing_to_stdout = format == 't' && strcmp(basedir, "-") == 0;
writing_to_stdout = format == 't' && basedir != NULL &&
strcmp(basedir, "-") == 0;
if (writing_to_stdout && PQntuples(res) > 1)
{
pg_log_error("can only write single tablespace to stdout, database has %d",
@@ -1894,7 +1946,7 @@ BaseBackup(void)
res = PQgetResult(conn);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
pg_log_error("could not get write-ahead log end position from server: %s",
pg_log_error("backup failed: %s",
PQerrorMessage(conn));
exit(1);
}
@@ -2028,8 +2080,11 @@ BaseBackup(void)
* synced after being completed. In plain format, all the data of the
* base directory is synced, taking into account all the tablespaces.
* Errors are not considered fatal.
*
* If, however, there's a backup target, we're not writing anything
* locally, so in that case we skip this step.
*/
if (do_sync)
if (do_sync && backup_target == NULL)
{
if (verbose)
pg_log_info("syncing data to disk ...");
@@ -2051,7 +2106,7 @@ BaseBackup(void)
* without a backup_manifest file, decreasing the chances that a directory
* we leave behind will be mistaken for a valid backup.
*/
if (!writing_to_stdout && manifest)
if (!writing_to_stdout && manifest && backup_target == NULL)
{
char tmp_filename[MAXPGPATH];
char filename[MAXPGPATH];
@@ -2085,6 +2140,7 @@ main(int argc, char **argv)
{"max-rate", required_argument, NULL, 'r'},
{"write-recovery-conf", no_argument, NULL, 'R'},
{"slot", required_argument, NULL, 'S'},
{"target", required_argument, NULL, 't'},
{"tablespace-mapping", required_argument, NULL, 'T'},
{"wal-method", required_argument, NULL, 'X'},
{"gzip", no_argument, NULL, 'z'},
@@ -2135,7 +2191,7 @@ main(int argc, char **argv)
atexit(cleanup_directories_atexit);
while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP",
while ((c = getopt_long(argc, argv, "CD:F:r:RS:t:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP",
long_options, &option_index)) != -1)
{
switch (c)
@@ -2176,6 +2232,9 @@ main(int argc, char **argv)
case 2:
no_slot = true;
break;
case 't':
backup_target = pg_strdup(optarg);
break;
case 'T':
tablespace_list_append(optarg);
break;
@@ -2308,27 +2367,72 @@ main(int argc, char **argv)
}
/*
* Required arguments
* Setting the backup target to 'client' is equivalent to leaving out the
* option. This logic allows us to assume elsewhere that the backup is
* being stored locally if and only if backup_target == NULL.
*/
if (basedir == NULL)
if (backup_target != NULL && strcmp(backup_target, "client") == 0)
{
pg_log_error("no target directory specified");
pg_free(backup_target);
backup_target = NULL;
}
/*
* Can't use --format with --target. Without --target, default format is
* tar.
*/
if (backup_target != NULL && format != '\0')
{
pg_log_error("cannot specify both format and backup target");
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
progname);
exit(1);
}
if (format == '\0')
format = 'p';
/*
* Either directory or backup target should be specified, but not both
*/
if (basedir == NULL && backup_target == NULL)
{
pg_log_error("must specify output directory or backup target");
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
progname);
exit(1);
}
if (basedir != NULL && backup_target != NULL)
{
pg_log_error("cannot specify both output directory and backup target");
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
progname);
exit(1);
}
/*
* Mutually exclusive arguments
* Compression doesn't make sense unless tar format is in use.
*/
if (format == 'p' && compresslevel != 0)
{
pg_log_error("only tar mode backups can be compressed");
if (backup_target == NULL)
pg_log_error("only tar mode backups can be compressed");
else
pg_log_error("client-side compression is not possible when a backup target is specfied");
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
progname);
exit(1);
}
/*
* Sanity checks for WAL method.
*/
if (backup_target != NULL && includewal == STREAM_WAL)
{
pg_log_error("WAL cannot be streamed when a backup target is specified");
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
progname);
exit(1);
}
if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0)
{
pg_log_error("cannot stream write-ahead logs in tar mode to stdout");
@@ -2345,6 +2449,9 @@ main(int argc, char **argv)
exit(1);
}
/*
* Sanity checks for replication slot options.
*/
if (no_slot)
{
if (replication_slot)
@@ -2378,8 +2485,18 @@ main(int argc, char **argv)
}
}
/*
* Sanity checks on WAL directory.
*/
if (xlog_dir)
{
if (backup_target != NULL)
{
pg_log_error("WAL directory location cannot be specified along with a backup target");
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
progname);
exit(1);
}
if (format != 'p')
{
pg_log_error("WAL directory location can only be specified in plain mode");
@@ -2400,6 +2517,7 @@ main(int argc, char **argv)
}
#ifndef HAVE_LIBZ
/* Sanity checks for compression level. */
if (compresslevel != 0)
{
pg_log_error("this build does not support compression");
@@ -2407,6 +2525,9 @@ main(int argc, char **argv)
}
#endif
/*
* Sanity checks for progress reporting options.
*/
if (showprogress && !estimatesize)
{
pg_log_error("%s and %s are incompatible options",
@@ -2416,6 +2537,9 @@ main(int argc, char **argv)
exit(1);
}
/*
* Sanity checks for backup manifest options.
*/
if (!manifest && manifest_checksums != NULL)
{
pg_log_error("%s and %s are incompatible options",
@@ -2458,11 +2582,11 @@ main(int argc, char **argv)
manifest = false;
/*
* Verify that the target directory exists, or create it. For plaintext
* backups, always require the directory. For tar backups, require it
* unless we are writing to stdout.
* If an output directory was specified, verify that it exists, or create
* it. Note that for a tar backup, an output directory of "-" means we are
* writing to stdout, so do nothing in that case.
*/
if (format == 'p' || strcmp(basedir, "-") != 0)
if (basedir != NULL && (format == 'p' || strcmp(basedir, "-") != 0))
verify_dir_is_empty_or_create(basedir, &made_new_pgdata, &found_existing_pgdata);
/* determine remote server's xlog segment size */