Fix SHMEM_ATTACH_UNKNOWN_SIZE when the shmem area doesn't exist

SHMEM_ATTACH_UNKNOWN_SIZE can be passed as argument to
ShmemRequestStruct() when the caller wants to attach to an existing
shared memory structure whose size it doesn't know.  If the shared
memory structure doesn't exist, the request should fail, but instead,
ProcessShmemRequestsAfterStartup() tried to create the structure with
size = -1.  That led to integer overflow in ShmemAllocRaw() and memory
corruption.

Fix by rejecting requests with SHMEM_ATTACH_UNKNOWN_SIZE when the
structure doesn't exist.  Also add an integer overflow check in
ShmemAllocRaw(), to protect from this kind of confusion or simply too
large requests.  Also document SHMEM_ATTACH_UNKNOWN_SIZE.

Author: Ashutosh Bapat <ashutosh.bapat.oss@gmail.com>
Discussion: https://www.postgresql.org/message-id/CAExHW5u_fTsOAS85kG981Vu6eR1GV-344rup6zYew7xMjEDREw@mail.gmail.com
Backpatch-through: 19

doc/src/sgml/xfunc.sgml

@@ -3745,6 +3745,16 @@ my_shmem_init(void *arg)
       areas are left in an abandoned state and any attempt to attach or
       re-initialize them will fail until the server is restarted.
      </para>
+     <para>
+      A backend attaching to an already allocated shared memory area must
+      normally pass the same <literal>size</literal> that was used to allocate
+      it to <function>ShmemRequestStruct()</function>; a mismatch results in
+      an error.  If the attaching backend does not know the size in advance,
+      it can pass <literal>SHMEM_ATTACH_UNKNOWN_SIZE</literal> instead, to
+      skip this cross-check.  <literal>SHMEM_ATTACH_UNKNOWN_SIZE</literal> can
+      only be used to attach to an area that already exists; using it before
+      allocating the area results in an error.
+     </para>
     </sect3>
 
     <sect3 id="xfunc-shared-addin-dynamic">

src/backend/storage/ipc/shmem.c

@@ -131,6 +131,7 @@
 #include <unistd.h>
 
 #include "access/slru.h"
+#include "common/int.h"
 #include "fmgr.h"
 #include "funcapi.h"
 #include "miscadmin.h"
@@ -535,6 +536,9 @@ InitShmemIndexEntry(ShmemRequest *request)
 	size_t		allocated_size;
 	void	   *structPtr;
 
+	/* Size must be known at this point. */
+	Assert(request->options->size != SHMEM_ATTACH_UNKNOWN_SIZE);
+
 	/* look it up in the shmem index */
 	index_entry = (ShmemIndexEnt *)
 		hash_search(ShmemIndex, name, HASH_ENTER_NULL, &found);
@@ -836,6 +840,8 @@ ShmemAllocNoError(Size size)
  *
  * Also sets *allocated_size to the number of bytes allocated, which will
  * be equal to the number requested plus any padding we choose to add.
+ *
+ * Returns NULL in case space can not be allocated.
  */
 static void *
 ShmemAllocRaw(Size size, Size alignment, Size *allocated_size)
@@ -866,8 +872,13 @@ ShmemAllocRaw(Size size, Size alignment, Size *allocated_size)
 	rawStart = ShmemAllocator->free_offset;
 	newStart = TYPEALIGN(alignment, rawStart);
 
-	newFree = newStart + size;
-	if (newFree <= ShmemSegHdr->totalsize)
+	/*
+	 * newFree = newStart + size, which is the start of the remaining space
+	 * after the allocation.  If it exceeds the shmem segment size, we don't
+	 * have enough space available.
+	 */
+	if (!pg_add_size_overflow(newStart, size, &newFree) &&
+		newFree <= ShmemSegHdr->totalsize)
 	{
 		newSpace = (char *) ShmemBase + newStart;
 		ShmemAllocator->free_offset = newFree;
@@ -1024,7 +1035,14 @@ ProcessShmemRequestsAfterStartup(const ShmemCallbacks *callbacks)
 			found_any = true;
 		}
 		else
+		{
+			if (request->options->size == SHMEM_ATTACH_UNKNOWN_SIZE)
+				ereport(ERROR,
+						(errmsg("cannot attach to shared memory struct \"%s\" because it does not exist",
+								request->options->name),
+						 errdetail("SHMEM_ATTACH_UNKNOWN_SIZE can only be used to attach to an existing shared memory structure.")));
 			notfound_any = true;
+		}
 	}
 	if (found_any && notfound_any)
 		elog(ERROR, "some of the requested shmem areas have already been initialized");

src/test/modules/test_shmem/t/001_late_shmem_alloc.pl

@@ -31,6 +31,23 @@ cmp_ok($attach_count2, '>', $attach_count1,
 
 $node->stop;
 
+###
+# Test that trying to allocate a new shmem area with size =
+# SHMEM_ATTACH_UNKNOWN_SIZE (-1) fails.
+###
+$node->append_conf('postgresql.conf', "test_shmem.area_size = -1");
+$node->start;
+
+my (undef, undef, $stderr) =
+  $node->psql("postgres", "SELECT get_test_shmem_attach_count();");
+like(
+	$stderr,
+	qr/cannot attach to shared memory struct "test_shmem area" because it does not exist/,
+	"unknown size request for a nonexistent area fails");
+
+$node->stop;
+$node->adjust_conf('postgresql.conf', 'test_shmem.area_size', undef);
+
 ###
 # Test allocating memory after startup in single-user mode
 ###

src/test/modules/test_shmem/test_shmem.c

@@ -35,12 +35,11 @@ typedef struct TestShmemData
 
 static TestShmemData *TestShmem;
 
-#define MIN_TEST_AREA_BYTES sizeof(TestShmemData)
-#define DEFAULT_TEST_AREA_BYTES MIN_TEST_AREA_BYTES
+#define DEFAULT_TEST_AREA_BYTES sizeof(TestShmemData)
 #define MAX_TEST_AREA_BYTES 1000000
 
 static bool attached_or_initialized = false;
-static int	test_shmem_area_size = MIN_TEST_AREA_BYTES;
+static int	test_shmem_area_size = DEFAULT_TEST_AREA_BYTES;
 static bool test_shmem_guc_defined = false;
 
 static void test_shmem_request(void *arg);
@@ -100,12 +99,16 @@ _PG_init(void)
 
 	if (!test_shmem_guc_defined)
 	{
+		/*
+		 * The minimum size that makes sense is sizeof(TestShmemData), but we
+		 * allow -1 so that we can test passing SHMEM_ATTACH_UNKNOWN_SIZE.
+		 */
 		DefineCustomIntVariable("test_shmem.area_size",
 								"Size of the shmem area to request.",
 								NULL,
 								&test_shmem_area_size,
 								DEFAULT_TEST_AREA_BYTES,
-								MIN_TEST_AREA_BYTES,
+								-1,
 								MAX_TEST_AREA_BYTES,
 								PGC_USERSET,
 								GUC_UNIT_BYTE,