Move some code from src/bin/scripts to src/fe_utils to permit reuse.

The parallel slots infrastructure (which implements client-side
multiplexing of server connections doing similar things, not
threading or multiple processes or anything like that) are moved from
src/bin/scripts/scripts_parallel.c to src/fe_utils/parallel_slot.c.

The functions consumeQueryResult() and processQueryResult() which were
previously part of src/bin/scripts/common.c are now moved into that
file as well, becoming static helper functions. This might need to be
changed in the future, but currently they're not used for anything
else.

Some other functions from src/bin/scripts/common.c are moved to to
src/fe_utils and are split up among several files.  connectDatabase(),
connectMaintenanceDatabase(), and disconnectDatabase() are moved to
connect_utils.c.  executeQuery(), executeCommand(), and
executeMaintenanceCommand() are move to query_utils.c.
handle_help_version_opts() is moved to option_utils.c.

Mark Dilger, reviewed by me. The larger patch series of which this is
a part has also had review from Peter Geoghegan, Andres Freund, Álvaro
Herrera, Michael Paquier, and Amul Sul, but I don't know whether any
of them have reviewed this bit specifically.

Discussion: http://postgr.es/m/12ED3DA8-25F0-4B68-937D-D907CFBF08E7@enterprisedb.com
Discussion: http://postgr.es/m/5F743835-3399-419C-8324-2D424237E999@enterprisedb.com
Discussion: http://postgr.es/m/70655DF3-33CE-4527-9A4D-DDEB582B6BA0@enterprisedb.com

src/fe_utils/Makefile

@@ -23,9 +23,13 @@ OBJS = \
 	archive.o \
 	cancel.o \
 	conditional.o \
+	connect_utils.o \
 	mbprint.o \
+	option_utils.o \
+	parallel_slot.o \
 	print.o \
 	psqlscan.o \
+	query_utils.o \
 	recovery_gen.o \
 	simple_list.o \
 	string_utils.o

src/fe_utils/connect_utils.c

@@ -0,0 +1,181 @@
+/*-------------------------------------------------------------------------
+ *
+ * Facilities for frontend code to connect to and disconnect from databases.
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/fe_utils/connect_utils.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/connect.h"
+#include "common/logging.h"
+#include "common/string.h"
+#include "fe_utils/connect_utils.h"
+#include "fe_utils/query_utils.h"
+
+/*
+ * Make a database connection with the given parameters.
+ *
+ * An interactive password prompt is automatically issued if needed and
+ * allowed by cparams->prompt_password.
+ *
+ * If allow_password_reuse is true, we will try to re-use any password
+ * given during previous calls to this routine.  (Callers should not pass
+ * allow_password_reuse=true unless reconnecting to the same database+user
+ * as before, else we might create password exposure hazards.)
+ */
+PGconn *
+connectDatabase(const ConnParams *cparams, const char *progname,
+				bool echo, bool fail_ok, bool allow_password_reuse)
+{
+	PGconn	   *conn;
+	bool		new_pass;
+	static char *password = NULL;
+
+	/* Callers must supply at least dbname; other params can be NULL */
+	Assert(cparams->dbname);
+
+	if (!allow_password_reuse && password)
+	{
+		free(password);
+		password = NULL;
+	}
+
+	if (cparams->prompt_password == TRI_YES && password == NULL)
+		password = simple_prompt("Password: ", false);
+
+	/*
+	 * Start the connection.  Loop until we have a password if requested by
+	 * backend.
+	 */
+	do
+	{
+		const char *keywords[8];
+		const char *values[8];
+		int			i = 0;
+
+		/*
+		 * If dbname is a connstring, its entries can override the other
+		 * values obtained from cparams; but in turn, override_dbname can
+		 * override the dbname component of it.
+		 */
+		keywords[i] = "host";
+		values[i++] = cparams->pghost;
+		keywords[i] = "port";
+		values[i++] = cparams->pgport;
+		keywords[i] = "user";
+		values[i++] = cparams->pguser;
+		keywords[i] = "password";
+		values[i++] = password;
+		keywords[i] = "dbname";
+		values[i++] = cparams->dbname;
+		if (cparams->override_dbname)
+		{
+			keywords[i] = "dbname";
+			values[i++] = cparams->override_dbname;
+		}
+		keywords[i] = "fallback_application_name";
+		values[i++] = progname;
+		keywords[i] = NULL;
+		values[i++] = NULL;
+		Assert(i <= lengthof(keywords));
+
+		new_pass = false;
+		conn = PQconnectdbParams(keywords, values, true);
+
+		if (!conn)
+		{
+			pg_log_error("could not connect to database %s: out of memory",
+						 cparams->dbname);
+			exit(1);
+		}
+
+		/*
+		 * No luck?  Trying asking (again) for a password.
+		 */
+		if (PQstatus(conn) == CONNECTION_BAD &&
+			PQconnectionNeedsPassword(conn) &&
+			cparams->prompt_password != TRI_NO)
+		{
+			PQfinish(conn);
+			if (password)
+				free(password);
+			password = simple_prompt("Password: ", false);
+			new_pass = true;
+		}
+	} while (new_pass);
+
+	/* check to see that the backend connection was successfully made */
+	if (PQstatus(conn) == CONNECTION_BAD)
+	{
+		if (fail_ok)
+		{
+			PQfinish(conn);
+			return NULL;
+		}
+		pg_log_error("%s", PQerrorMessage(conn));
+		exit(1);
+	}
+
+	/* Start strict; callers may override this. */
+	PQclear(executeQuery(conn, ALWAYS_SECURE_SEARCH_PATH_SQL, echo));
+
+	return conn;
+}
+
+/*
+ * Try to connect to the appropriate maintenance database.
+ *
+ * This differs from connectDatabase only in that it has a rule for
+ * inserting a default "dbname" if none was given (which is why cparams
+ * is not const).  Note that cparams->dbname should typically come from
+ * a --maintenance-db command line parameter.
+ */
+PGconn *
+connectMaintenanceDatabase(ConnParams *cparams,
+						   const char *progname, bool echo)
+{
+	PGconn	   *conn;
+
+	/* If a maintenance database name was specified, just connect to it. */
+	if (cparams->dbname)
+		return connectDatabase(cparams, progname, echo, false, false);
+
+	/* Otherwise, try postgres first and then template1. */
+	cparams->dbname = "postgres";
+	conn = connectDatabase(cparams, progname, echo, true, false);
+	if (!conn)
+	{
+		cparams->dbname = "template1";
+		conn = connectDatabase(cparams, progname, echo, false, false);
+	}
+	return conn;
+}
+
+/*
+ * Disconnect the given connection, canceling any statement if one is active.
+ */
+void
+disconnectDatabase(PGconn *conn)
+{
+	char		errbuf[256];
+
+	Assert(conn != NULL);
+
+	if (PQtransactionStatus(conn) == PQTRANS_ACTIVE)
+	{
+		PGcancel   *cancel;
+
+		if ((cancel = PQgetCancel(conn)))
+		{
+			(void) PQcancel(cancel, errbuf, sizeof(errbuf));
+			PQfreeCancel(cancel);
+		}
+	}
+
+	PQfinish(conn);
+}

src/fe_utils/option_utils.c

@@ -0,0 +1,38 @@
+/*-------------------------------------------------------------------------
+ *
+ * Command line option processing facilities for frontend code
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/fe_utils/option_utils.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres_fe.h"
+
+#include "fe_utils/option_utils.h"
+
+/*
+ * Provide strictly harmonized handling of --help and --version
+ * options.
+ */
+void
+handle_help_version_opts(int argc, char *argv[],
+						 const char *fixed_progname, help_handler hlp)
+{
+	if (argc > 1)
+	{
+		if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
+		{
+			hlp(get_progname(argv[0]));
+			exit(0);
+		}
+		if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
+		{
+			printf("%s (PostgreSQL) " PG_VERSION "\n", fixed_progname);
+			exit(0);
+		}
+	}
+}

src/fe_utils/parallel_slot.c

@@ -0,0 +1,337 @@
+/*-------------------------------------------------------------------------
+ *
+ *	parallel_slot.c
+ *		Parallel support for front-end parallel database connections
+ *
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/fe_utils/parallel_slot.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifdef WIN32
+#define FD_SETSIZE 1024			/* must set before winsock2.h is included */
+#endif
+
+#include "postgres_fe.h"
+
+#ifdef HAVE_SYS_SELECT_H
+#include <sys/select.h>
+#endif
+
+#include "common/logging.h"
+#include "fe_utils/cancel.h"
+#include "fe_utils/parallel_slot.h"
+
+#define ERRCODE_UNDEFINED_TABLE  "42P01"
+
+static void init_slot(ParallelSlot *slot, PGconn *conn);
+static int	select_loop(int maxFd, fd_set *workerset);
+static bool processQueryResult(PGconn *conn, PGresult *result);
+
+static void
+init_slot(ParallelSlot *slot, PGconn *conn)
+{
+	slot->connection = conn;
+	/* Initially assume connection is idle */
+	slot->isFree = true;
+}
+
+/*
+ * Process (and delete) a query result.  Returns true if there's no error,
+ * false otherwise -- but errors about trying to work on a missing relation
+ * are reported and subsequently ignored.
+ */
+static bool
+processQueryResult(PGconn *conn, PGresult *result)
+{
+	/*
+	 * If it's an error, report it.  Errors about a missing table are harmless
+	 * so we continue processing; but die for other errors.
+	 */
+	if (PQresultStatus(result) != PGRES_COMMAND_OK)
+	{
+		char	   *sqlState = PQresultErrorField(result, PG_DIAG_SQLSTATE);
+
+		pg_log_error("processing of database \"%s\" failed: %s",
+					 PQdb(conn), PQerrorMessage(conn));
+
+		if (sqlState && strcmp(sqlState, ERRCODE_UNDEFINED_TABLE) != 0)
+		{
+			PQclear(result);
+			return false;
+		}
+	}
+
+	PQclear(result);
+	return true;
+}
+
+/*
+ * Consume all the results generated for the given connection until
+ * nothing remains.  If at least one error is encountered, return false.
+ * Note that this will block if the connection is busy.
+ */
+static bool
+consumeQueryResult(PGconn *conn)
+{
+	bool		ok = true;
+	PGresult   *result;
+
+	SetCancelConn(conn);
+	while ((result = PQgetResult(conn)) != NULL)
+	{
+		if (!processQueryResult(conn, result))
+			ok = false;
+	}
+	ResetCancelConn();
+	return ok;
+}
+
+/*
+ * Wait until a file descriptor from the given set becomes readable.
+ *
+ * Returns the number of ready descriptors, or -1 on failure (including
+ * getting a cancel request).
+ */
+static int
+select_loop(int maxFd, fd_set *workerset)
+{
+	int			i;
+	fd_set		saveSet = *workerset;
+
+	if (CancelRequested)
+		return -1;
+
+	for (;;)
+	{
+		/*
+		 * On Windows, we need to check once in a while for cancel requests;
+		 * on other platforms we rely on select() returning when interrupted.
+		 */
+		struct timeval *tvp;
+#ifdef WIN32
+		struct timeval tv = {0, 1000000};
+
+		tvp = &tv;
+#else
+		tvp = NULL;
+#endif
+
+		*workerset = saveSet;
+		i = select(maxFd + 1, workerset, NULL, NULL, tvp);
+
+#ifdef WIN32
+		if (i == SOCKET_ERROR)
+		{
+			i = -1;
+
+			if (WSAGetLastError() == WSAEINTR)
+				errno = EINTR;
+		}
+#endif
+
+		if (i < 0 && errno == EINTR)
+			continue;			/* ignore this */
+		if (i < 0 || CancelRequested)
+			return -1;			/* but not this */
+		if (i == 0)
+			continue;			/* timeout (Win32 only) */
+		break;
+	}
+
+	return i;
+}
+
+/*
+ * ParallelSlotsGetIdle
+ *		Return a connection slot that is ready to execute a command.
+ *
+ * This returns the first slot we find that is marked isFree, if one is;
+ * otherwise, we loop on select() until one socket becomes available.  When
+ * this happens, we read the whole set and mark as free all sockets that
+ * become available.  If an error occurs, NULL is returned.
+ */
+ParallelSlot *
+ParallelSlotsGetIdle(ParallelSlot *slots, int numslots)
+{
+	int			i;
+	int			firstFree = -1;
+
+	/*
+	 * Look for any connection currently free.  If there is one, mark it as
+	 * taken and let the caller know the slot to use.
+	 */
+	for (i = 0; i < numslots; i++)
+	{
+		if (slots[i].isFree)
+		{
+			slots[i].isFree = false;
+			return slots + i;
+		}
+	}
+
+	/*
+	 * No free slot found, so wait until one of the connections has finished
+	 * its task and return the available slot.
+	 */
+	while (firstFree < 0)
+	{
+		fd_set		slotset;
+		int			maxFd = 0;
+
+		/* We must reconstruct the fd_set for each call to select_loop */
+		FD_ZERO(&slotset);
+
+		for (i = 0; i < numslots; i++)
+		{
+			int			sock = PQsocket(slots[i].connection);
+
+			/*
+			 * We don't really expect any connections to lose their sockets
+			 * after startup, but just in case, cope by ignoring them.
+			 */
+			if (sock < 0)
+				continue;
+
+			FD_SET(sock, &slotset);
+			if (sock > maxFd)
+				maxFd = sock;
+		}
+
+		SetCancelConn(slots->connection);
+		i = select_loop(maxFd, &slotset);
+		ResetCancelConn();
+
+		/* failure? */
+		if (i < 0)
+			return NULL;
+
+		for (i = 0; i < numslots; i++)
+		{
+			int			sock = PQsocket(slots[i].connection);
+
+			if (sock >= 0 && FD_ISSET(sock, &slotset))
+			{
+				/* select() says input is available, so consume it */
+				PQconsumeInput(slots[i].connection);
+			}
+
+			/* Collect result(s) as long as any are available */
+			while (!PQisBusy(slots[i].connection))
+			{
+				PGresult   *result = PQgetResult(slots[i].connection);
+
+				if (result != NULL)
+				{
+					/* Check and discard the command result */
+					if (!processQueryResult(slots[i].connection, result))
+						return NULL;
+				}
+				else
+				{
+					/* This connection has become idle */
+					slots[i].isFree = true;
+					if (firstFree < 0)
+						firstFree = i;
+					break;
+				}
+			}
+		}
+	}
+
+	slots[firstFree].isFree = false;
+	return slots + firstFree;
+}
+
+/*
+ * ParallelSlotsSetup
+ *		Prepare a set of parallel slots to use on a given database.
+ *
+ * This creates and initializes a set of connections to the database
+ * using the information given by the caller, marking all parallel slots
+ * as free and ready to use.  "conn" is an initial connection set up
+ * by the caller and is associated with the first slot in the parallel
+ * set.
+ */
+ParallelSlot *
+ParallelSlotsSetup(const ConnParams *cparams,
+				   const char *progname, bool echo,
+				   PGconn *conn, int numslots)
+{
+	ParallelSlot *slots;
+	int			i;
+
+	Assert(conn != NULL);
+
+	slots = (ParallelSlot *) pg_malloc(sizeof(ParallelSlot) * numslots);
+	init_slot(slots, conn);
+	if (numslots > 1)
+	{
+		for (i = 1; i < numslots; i++)
+		{
+			conn = connectDatabase(cparams, progname, echo, false, true);
+
+			/*
+			 * Fail and exit immediately if trying to use a socket in an
+			 * unsupported range.  POSIX requires open(2) to use the lowest
+			 * unused file descriptor and the hint given relies on that.
+			 */
+			if (PQsocket(conn) >= FD_SETSIZE)
+			{
+				pg_log_fatal("too many jobs for this platform -- try %d", i);
+				exit(1);
+			}
+
+			init_slot(slots + i, conn);
+		}
+	}
+
+	return slots;
+}
+
+/*
+ * ParallelSlotsTerminate
+ *		Clean up a set of parallel slots
+ *
+ * Iterate through all connections in a given set of ParallelSlots and
+ * terminate all connections.
+ */
+void
+ParallelSlotsTerminate(ParallelSlot *slots, int numslots)
+{
+	int			i;
+
+	for (i = 0; i < numslots; i++)
+	{
+		PGconn	   *conn = slots[i].connection;
+
+		if (conn == NULL)
+			continue;
+
+		disconnectDatabase(conn);
+	}
+}
+
+/*
+ * ParallelSlotsWaitCompletion
+ *
+ * Wait for all connections to finish, returning false if at least one
+ * error has been found on the way.
+ */
+bool
+ParallelSlotsWaitCompletion(ParallelSlot *slots, int numslots)
+{
+	int			i;
+
+	for (i = 0; i < numslots; i++)
+	{
+		if (!consumeQueryResult((slots + i)->connection))
+			return false;
+	}
+
+	return true;
+}

src/fe_utils/query_utils.c

@@ -0,0 +1,92 @@
+/*-------------------------------------------------------------------------
+ *
+ * Facilities for frontend code to query a databases.
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/fe_utils/query_utils.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres_fe.h"
+
+#include "common/logging.h"
+#include "fe_utils/cancel.h"
+#include "fe_utils/query_utils.h"
+
+/*
+ * Run a query, return the results, exit program on failure.
+ */
+PGresult *
+executeQuery(PGconn *conn, const char *query, bool echo)
+{
+	PGresult   *res;
+
+	if (echo)
+		printf("%s\n", query);
+
+	res = PQexec(conn, query);
+	if (!res ||
+		PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		pg_log_error("query failed: %s", PQerrorMessage(conn));
+		pg_log_info("query was: %s", query);
+		PQfinish(conn);
+		exit(1);
+	}
+
+	return res;
+}
+
+
+/*
+ * As above for a SQL command (which returns nothing).
+ */
+void
+executeCommand(PGconn *conn, const char *query, bool echo)
+{
+	PGresult   *res;
+
+	if (echo)
+		printf("%s\n", query);
+
+	res = PQexec(conn, query);
+	if (!res ||
+		PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		pg_log_error("query failed: %s", PQerrorMessage(conn));
+		pg_log_info("query was: %s", query);
+		PQfinish(conn);
+		exit(1);
+	}
+
+	PQclear(res);
+}
+
+
+/*
+ * As above for a SQL maintenance command (returns command success).
+ * Command is executed with a cancel handler set, so Ctrl-C can
+ * interrupt it.
+ */
+bool
+executeMaintenanceCommand(PGconn *conn, const char *query, bool echo)
+{
+	PGresult   *res;
+	bool		r;
+
+	if (echo)
+		printf("%s\n", query);
+
+	SetCancelConn(conn);
+	res = PQexec(conn, query);
+	ResetCancelConn();
+
+	r = (res && PQresultStatus(res) == PGRES_COMMAND_OK);
+
+	if (res)
+		PQclear(res);
+
+	return r;
+}