Add -c/--restore-target-wal to pg_rewind

pg_rewind needs to copy from the source cluster to the target cluster a
set of relation blocks changed from the previous checkpoint where WAL
forked up to the end of WAL on the target.  Building this list of
relation blocks requires a range of WAL segments that may not be present
anymore on the target's pg_wal, causing pg_rewind to fail.  It is
possible to work around this issue by copying manually the WAL segments
needed but this may lead to some extra and actually useless work.

This commit introduces a new option allowing pg_rewind to use a
restore_command while doing the rewind by grabbing the parameter value
of restore_command from the target cluster configuration.  This allows
the rewind operation to be more reliable, so as only the WAL segments
needed by the rewind are restored from the archives.

In order to be able to do that, a new routine is added to src/common/ to
allow frontend tools to restore files from archives using an
already-built restore command.  This version is more simple than the
backend equivalent as there is no need to handle the non-recovery case.

Author: Alexey Kondratov
Reviewed-by: Andrey Borodin, Andres Freund, Alvaro Herrera, Alexander
Korotkov, Michael Paquier
Discussion: https://postgr.es/m/a3acff50-5a0d-9a2c-b3b2-ee36168955c1@postgrespro.ru

src/common/Makefile

@@ -88,6 +88,7 @@ endif
 # (Mkvcbuild.pm has a copy of this list, too)
 OBJS_FRONTEND = \
 	$(OBJS_COMMON) \
+	fe_archive.o \
 	fe_memutils.o \
 	file_utils.o \
 	logging.o \

src/common/exec.c

@@ -51,7 +51,6 @@
 
 static int	validate_exec(const char *path);
 static int	resolve_symlinks(char *path);
-static char *pipe_read_line(char *cmd, char *line, int maxsize);
 
 #ifdef WIN32
 static BOOL GetTokenUser(HANDLE hToken, PTOKEN_USER *ppTokenUser);
@@ -356,7 +355,7 @@ find_other_exec(const char *argv0, const char *target,
 /*
  * Execute a command in a pipe and read the first line from it.
  */
-static char *
+char *
 pipe_read_line(char *cmd, char *line, int maxsize)
 {
 	FILE	   *pgver;

src/common/fe_archive.c

@@ -0,0 +1,128 @@
+/*-------------------------------------------------------------------------
+ *
+ * fe_archive.c
+ *	  Routines to access WAL archives from frontend
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ *	  src/common/fe_archive.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#error "This file is not expected to be compiled for backend code"
+#endif
+
+#include "postgres_fe.h"
+
+#include <unistd.h>
+#include <sys/stat.h>
+
+#include "access/xlog_internal.h"
+#include "common/archive.h"
+#include "common/fe_archive.h"
+#include "common/logging.h"
+
+
+/*
+ * RestoreArchivedFile
+ *
+ * Attempt to retrieve the specified file from off-line archival storage.
+ * If successful, return a file descriptor of the restored file, else
+ * return -1.
+ *
+ * For fixed-size files, the caller may pass the expected size as an
+ * additional crosscheck on successful recovery.  If the file size is not
+ * known, set expectedSize = 0.
+ */
+int
+RestoreArchivedFile(const char *path, const char *xlogfname,
+					off_t expectedSize, const char *restoreCommand)
+{
+	char		xlogpath[MAXPGPATH];
+	char	   *xlogRestoreCmd;
+	int			rc;
+	struct stat stat_buf;
+
+	snprintf(xlogpath, MAXPGPATH, "%s/" XLOGDIR "/%s", path, xlogfname);
+
+	xlogRestoreCmd = BuildRestoreCommand(restoreCommand, xlogpath,
+										 xlogfname, NULL);
+	if (xlogRestoreCmd == NULL)
+	{
+		pg_log_fatal("could not use restore_command with %%r alias");
+		exit(1);
+	}
+
+	/*
+	 * Execute restore_command, which should copy the missing file from
+	 * archival storage.
+	 */
+	rc = system(xlogRestoreCmd);
+	pfree(xlogRestoreCmd);
+
+	if (rc == 0)
+	{
+		/*
+		 * Command apparently succeeded, but let's make sure the file is
+		 * really there now and has the correct size.
+		 */
+		if (stat(xlogpath, &stat_buf) == 0)
+		{
+			if (expectedSize > 0 && stat_buf.st_size != expectedSize)
+			{
+				pg_log_fatal("unexpected file size for \"%s\": %lu instead of %lu",
+							 xlogfname, (unsigned long) stat_buf.st_size,
+							 (unsigned long) expectedSize);
+				exit(1);
+			}
+			else
+			{
+				int			xlogfd = open(xlogpath, O_RDONLY | PG_BINARY, 0);
+
+				if (xlogfd < 0)
+				{
+					pg_log_fatal("could not open file \"%s\" restored from archive: %m",
+								 xlogpath);
+					exit(1);
+				}
+				else
+					return xlogfd;
+			}
+		}
+		else
+		{
+			if (errno != ENOENT)
+			{
+				pg_log_fatal("could not stat file \"%s\": %m",
+							 xlogpath);
+				exit(1);
+			}
+		}
+	}
+
+	/*
+	 * If the failure was due to a signal, then it would be misleading to
+	 * return with a failure at restoring the file.  So just bail out and
+	 * exit.  Hard shell errors such as "command not found" are treated as
+	 * fatal too.
+	 */
+	if (wait_result_is_any_signal(rc, true))
+	{
+		pg_log_fatal("restore_command failed due to the signal: %s",
+					 wait_result_to_str(rc));
+		exit(1);
+	}
+
+	/*
+	 * The file is not available, so just let the caller decide what to do
+	 * next.
+	 */
+	pg_log_error("could not restore file \"%s\" from archive",
+				 xlogfname);
+	return -1;
+}