Move libpq's write_failed mechanism down to pqsecure_raw_write().

Commit 1f39a1c06 implemented write-failure postponement in pqSendSome,
which is above SSL/GSS processing.  However, we've now seen failures
indicating that (some versions of?) OpenSSL have a tendency to report
write failures prematurely too.  Hence, move the primary responsibility
for postponing write failures down to pqsecure_raw_write(), below
SSL/GSS processing.  pqSendSome now sets write_failed only in corner
cases where we'd lost the connection already.

A side-effect of this change is that errors detected in the SSL/GSS
layer itself will be reported immediately (as if they were read
errors) rather than being postponed like write errors.  That's
reverting an effect of 1f39a1c06, and I think it's fine: if there's
not a socket-level error, it's hard to be sure whether an OpenSSL
error ought to be considered a read or write failure anyway.

Another important point is that write-failure postponement is now
effective during connection setup.  OpenSSL's misbehavior of this
sort occurs during SSL_connect(), so that's a change we want.

Per bug #17391 from Nazir Bilal Yavuz.  Possibly this should be
back-patched, but I think it prudent to let it age awhile in HEAD
first.

Discussion: https://postgr.es/m/17391-304f81bcf724b58b@postgresql.org

src/interfaces/libpq/fe-misc.c

@@ -777,19 +777,19 @@ definitelyFailed:
  * (putting it in conn->inBuffer) in any situation where we can't send
  * all the specified data immediately.
  *
- * Upon write failure, conn->write_failed is set and the error message is
- * saved in conn->write_err_msg, but we clear the output buffer and return
- * zero anyway; this is because callers should soldier on until it's possible
- * to read from the server and check for an error message.  write_err_msg
- * should be reported only when we are unable to obtain a server error first.
- * (Thus, a -1 result is returned only for an internal *read* failure.)
+ * If a socket-level write failure occurs, conn->write_failed is set and the
+ * error message is saved in conn->write_err_msg, but we clear the output
+ * buffer and return zero anyway; this is because callers should soldier on
+ * until we have read what we can from the server and checked for an error
+ * message.  write_err_msg should be reported only when we are unable to
+ * obtain a server error first.  Much of that behavior is implemented at
+ * lower levels, but this function deals with some edge cases.
  */
 static int
 pqSendSome(PGconn *conn, int len)
 {
 	char	   *ptr = conn->outBuffer;
 	int			remaining = conn->outCount;
-	int			oldmsglen = conn->errorMessage.len;
 	int			result = 0;
 
 	/*
@@ -817,7 +817,7 @@ pqSendSome(PGconn *conn, int len)
 	if (conn->sock == PGINVALID_SOCKET)
 	{
 		conn->write_failed = true;
-		/* Insert error message into conn->write_err_msg, if possible */
+		/* Store error message in conn->write_err_msg, if possible */
 		/* (strdup failure is OK, we'll cope later) */
 		conn->write_err_msg = strdup(libpq_gettext("connection not open\n"));
 		/* Discard queued data; no chance it'll ever be sent */
@@ -859,24 +859,6 @@ pqSendSome(PGconn *conn, int len)
 					continue;
 
 				default:
-					/* pqsecure_write set the error message for us */
-					conn->write_failed = true;
-
-					/*
-					 * Transfer error message to conn->write_err_msg, if
-					 * possible (strdup failure is OK, we'll cope later).
-					 *
-					 * We only want to transfer whatever has been appended to
-					 * conn->errorMessage since we entered this routine.
-					 */
-					if (!PQExpBufferBroken(&conn->errorMessage))
-					{
-						conn->write_err_msg = strdup(conn->errorMessage.data +
-													 oldmsglen);
-						conn->errorMessage.len = oldmsglen;
-						conn->errorMessage.data[oldmsglen] = '\0';
-					}
-
 					/* Discard queued data; no chance it'll ever be sent */
 					conn->outCount = 0;
 
@@ -886,7 +868,18 @@ pqSendSome(PGconn *conn, int len)
 						if (pqReadData(conn) < 0)
 							return -1;
 					}
-					return 0;
+
+					/*
+					 * Lower-level code should already have filled
+					 * conn->write_err_msg (and set conn->write_failed) or
+					 * conn->errorMessage.  In the former case, we pretend
+					 * there's no problem; the write_failed condition will be
+					 * dealt with later.  Otherwise, report the error now.
+					 */
+					if (conn->write_failed)
+						return 0;
+					else
+						return -1;
 			}
 		}
 		else

src/interfaces/libpq/fe-secure.c

@@ -280,9 +280,22 @@ pqsecure_raw_read(PGconn *conn, void *ptr, size_t len)
 /*
  *	Write data to a secure connection.
  *
- * On failure, this function is responsible for appending a suitable message
- * to conn->errorMessage.  The caller must still inspect errno, but only
- * to determine whether to continue/retry after error.
+ * Returns the number of bytes written, or a negative value (with errno
+ * set) upon failure.  The write count could be less than requested.
+ *
+ * Note that socket-level hard failures are masked from the caller,
+ * instead setting conn->write_failed and storing an error message
+ * in conn->write_err_msg; see pqsecure_raw_write.  This allows us to
+ * postpone reporting of write failures until we're sure no error
+ * message is available from the server.
+ *
+ * However, errors detected in the SSL or GSS management level are reported
+ * via a negative result, with message appended to conn->errorMessage.
+ * It's frequently unclear whether such errors should be considered read or
+ * write errors, so we don't attempt to postpone reporting them.
+ *
+ * The caller must still inspect errno upon failure, but only to determine
+ * whether to continue/retry; a message has been saved someplace in any case.
  */
 ssize_t
 pqsecure_write(PGconn *conn, const void *ptr, size_t len)
@@ -310,16 +323,50 @@ pqsecure_write(PGconn *conn, const void *ptr, size_t len)
 	return n;
 }
 
+/*
+ * Low-level implementation of pqsecure_write.
+ *
+ * This is used directly for an unencrypted connection.  For encrypted
+ * connections, this does the physical I/O on behalf of pgtls_write or
+ * pg_GSS_write.
+ *
+ * This function reports failure (i.e., returns a negative result) only
+ * for retryable errors such as EINTR.  Looping for such cases is to be
+ * handled at some outer level, maybe all the way up to the application.
+ * For hard failures, we set conn->write_failed and store an error message
+ * in conn->write_err_msg, but then claim to have written the data anyway.
+ * This is because we don't want to report write failures so long as there
+ * is a possibility of reading from the server and getting an error message
+ * that could explain why the connection dropped.  Many TCP stacks have
+ * race conditions such that a write failure may or may not be reported
+ * before all incoming data has been read.
+ *
+ * Note that this error behavior happens below the SSL management level when
+ * we are using SSL.  That's because at least some versions of OpenSSL are
+ * too quick to report a write failure when there's still a possibility to
+ * get a more useful error from the server.
+ */
 ssize_t
 pqsecure_raw_write(PGconn *conn, const void *ptr, size_t len)
 {
 	ssize_t		n;
 	int			flags = 0;
 	int			result_errno = 0;
+	char		msgbuf[1024];
 	char		sebuf[PG_STRERROR_R_BUFLEN];
 
 	DECLARE_SIGPIPE_INFO(spinfo);
 
+	/*
+	 * If we already had a write failure, we will never again try to send data
+	 * on that connection.  Even if the kernel would let us, we've probably
+	 * lost message boundary sync with the server.  conn->write_failed
+	 * therefore persists until the connection is reset, and we just discard
+	 * all data presented to be written.
+	 */
+	if (conn->write_failed)
+		return len;
+
 #ifdef MSG_NOSIGNAL
 	if (conn->sigpipe_flag)
 		flags |= MSG_NOSIGNAL;
@@ -369,17 +416,29 @@ retry_masked:
 				/* FALL THRU */
 
 			case ECONNRESET:
-				appendPQExpBufferStr(&conn->errorMessage,
-									 libpq_gettext("server closed the connection unexpectedly\n"
-												   "\tThis probably means the server terminated abnormally\n"
-												   "\tbefore or while processing the request.\n"));
+				conn->write_failed = true;
+				/* Store error message in conn->write_err_msg, if possible */
+				/* (strdup failure is OK, we'll cope later) */
+				snprintf(msgbuf, sizeof(msgbuf),
+						 libpq_gettext("server closed the connection unexpectedly\n"
+									   "\tThis probably means the server terminated abnormally\n"
+									   "\tbefore or while processing the request.\n"));
+				conn->write_err_msg = strdup(msgbuf);
+				/* Now claim the write succeeded */
+				n = len;
 				break;
 
 			default:
-				appendPQExpBuffer(&conn->errorMessage,
-								  libpq_gettext("could not send data to server: %s\n"),
-								  SOCK_STRERROR(result_errno,
-												sebuf, sizeof(sebuf)));
+				conn->write_failed = true;
+				/* Store error message in conn->write_err_msg, if possible */
+				/* (strdup failure is OK, we'll cope later) */
+				snprintf(msgbuf, sizeof(msgbuf),
+						 libpq_gettext("could not send data to server: %s\n"),
+						 SOCK_STRERROR(result_errno,
+									   sebuf, sizeof(sebuf)));
+				conn->write_err_msg = strdup(msgbuf);
+				/* Now claim the write succeeded */
+				n = len;
 				break;
 		}
 	}