Reduce memory overheads for storing a Memoize tuple

This uses 16 fewer bytes per tuple stored in each MemoizeEntry.  24 bytes
can be saved by not doing palloc_object(MemoizeTuple), and instead we
can store the pointer to the next tuple by requesting that
ExecCopySlotMinimalTupleExtra() allocate an extra 8 bytes per
MinimalTuple, a net saving of 16 bytes per cached tuple, which equates
to 33% less memory in the most extreme case (assuming very few entries
with many tuples and the narrowest possible MinimalTuple).  Real-world
cases will save less, as we're not saving anything on the MemoizeEntry
itself, only tuples stored within it.

Saving memory in Memoize allows more entries to exist concurrently and
reduces the chances that an entry will be evicted from the cache due to
memory pressure.  That increases the chances of getting cache hits.

Author: David Rowley <dgrowleyml@gmail.com>
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/CAApHDvoNurQzpR2nZ4aLq1v_4gstsSdimZP-m34PMTBPptn+zQ@mail.gmail.com

src/backend/executor/nodeMemoize.c

@@ -84,20 +84,18 @@
 #define MEMO_END_OF_SCAN			5	/* Ready for rescan */
 
 
+/*
+ * The number of extra bytes we request from ExecCopySlotMinimalTupleExtra to
+ * allow storage of the pointer to the next cached tuple for a MemoizeEntry.
+ */
+#define MEMOIZE_NEXT_TUPLE_EXTRA_BYTES MAXALIGN(sizeof(MinimalTuple))
+
 /* Helper macros for memory accounting */
 #define EMPTY_ENTRY_MEMORY_BYTES(e)		(sizeof(MemoizeEntry) + \
 										 sizeof(MemoizeKey) + \
-										 (e)->key->params->t_len);
+										 (e)->key->params->t_len)
-#define CACHE_TUPLE_BYTES(t)			(sizeof(MemoizeTuple) + \
+#define CACHE_TUPLE_BYTES(t)			((t)->t_len + \
-										 (t)->mintuple->t_len)
+										 MEMOIZE_NEXT_TUPLE_EXTRA_BYTES)
-
- /* MemoizeTuple Stores an individually cached tuple */
-typedef struct MemoizeTuple
-{
-	MinimalTuple mintuple;		/* Cached tuple */
-	struct MemoizeTuple *next;	/* The next tuple with the same parameter
-								 * values or NULL if it's the last one */
-} MemoizeTuple;
 
 /*
  * MemoizeKey
@@ -116,13 +114,46 @@ typedef struct MemoizeKey
 typedef struct MemoizeEntry
 {
 	MemoizeKey *key;			/* Hash key for hash table lookups */
-	MemoizeTuple *tuplehead;	/* Pointer to the first tuple or NULL if no
+	MinimalTuple tuplehead;		/* Pointer to the first tuple or NULL if no
 								 * tuples are cached for this entry */
 	uint32		hash;			/* Hash value (cached) */
 	char		status;			/* Hash status */
 	bool		complete;		/* Did we read the outer plan to completion? */
 } MemoizeEntry;
 
+/*
+ * Tuples stored in a MemoizeEntry are stored as MinimalTuples.  To allow
+ * these MinimalTuples to be formed into a linked list containing all tuples
+ * for the entry, we make use of ExecCopySlotMinimalTupleExtra() so that the
+ * palloc for the MinimalTuple has enough extra bytes to store the pointer to
+ * the next tuple for the cache entry, or NULL when it's the last tuple.
+ *
+ * The helper functions below allow us to avoid having to repeat the memory
+ * address calculations for the next tuple pointer and allow us to fetch and
+ * set the pointer to the next tuple.
+ */
+
+/*
+ * Calculate the address of the "next" pointer from the MinimalTuple
+ */
+#define MemoizeNextTupleAddress(t) \
+	((MinimalTuple *) ((char *) (t) - MEMOIZE_NEXT_TUPLE_EXTRA_BYTES))
+
+/* Fetch a pointer to the MinimalTupleData for the next tuple after 'tup' */
+static pg_always_inline MinimalTuple
+MemoizeGetNextTuple(MinimalTuple tup)
+{
+	return *MemoizeNextTupleAddress(tup);
+}
+
+/* Set the next pointer in 'tup' to 'next' or NULL when it's the last tuple */
+static pg_always_inline void
+MemoizeSetNextTuple(MinimalTuple tup, MinimalTuple next)
+{
+	MinimalTuple *next_ptr = MemoizeNextTupleAddress(tup);
+
+	*next_ptr = next;
+}
 
 #define SH_PREFIX memoize
 #define SH_ELEMENT_TYPE MemoizeEntry
@@ -344,18 +375,17 @@ prepare_probe_slot(MemoizeState *mstate, MemoizeKey *key)
 static inline void
 entry_purge_tuples(MemoizeState *mstate, MemoizeEntry *entry)
 {
-	MemoizeTuple *tuple = entry->tuplehead;
+	MinimalTuple tuple = entry->tuplehead;
 	uint64		freed_mem = 0;
 
 	while (tuple != NULL)
 	{
-		MemoizeTuple *next = tuple->next;
+		MinimalTuple next = MemoizeGetNextTuple(tuple);
 
 		freed_mem += CACHE_TUPLE_BYTES(tuple);
 
 		/* Free memory used for this tuple */
-		pfree(tuple->mintuple);
+		pfree(MemoizeNextTupleAddress(tuple));
-		pfree(tuple);
 
 		tuple = next;
 	}
@@ -625,21 +655,30 @@ cache_lookup(MemoizeState *mstate, bool *found)
 static bool
 cache_store_tuple(MemoizeState *mstate, TupleTableSlot *slot)
 {
-	MemoizeTuple *tuple;
 	MemoizeEntry *entry = mstate->entry;
 	MemoryContext oldcontext;
+	MinimalTuple mintuple;
 
 	Assert(slot != NULL);
 	Assert(entry != NULL);
 
 	oldcontext = MemoryContextSwitchTo(mstate->tableContext);
 
-	tuple = palloc_object(MemoizeTuple);
+	/*
-	tuple->mintuple = ExecCopySlotMinimalTuple(slot);
+	 * Form a MinimalTuple with extra space to store a "next" pointer so that
-	tuple->next = NULL;
+	 * we can form a singly linked list of tuples belonging to this
+	 * MemoizeEntry.
+	 */
+	mintuple = ExecCopySlotMinimalTupleExtra(slot,
+											 MEMOIZE_NEXT_TUPLE_EXTRA_BYTES);
+
+	/*
+	 * No need to use MemoizeSetNextTuple to point the next tuple to NULL as
+	 * ExecCopySlotMinimalTupleExtra zeros the extra bytes.
+	 */
 
 	/* Account for the memory we just consumed */
-	mstate->mem_used += CACHE_TUPLE_BYTES(tuple);
+	mstate->mem_used += CACHE_TUPLE_BYTES(mintuple);
 
 	if (entry->tuplehead == NULL)
 	{
@@ -647,15 +686,15 @@ cache_store_tuple(MemoizeState *mstate, TupleTableSlot *slot)
 		 * This is the first tuple for this entry, so just point the list head
 		 * to it.
 		 */
-		entry->tuplehead = tuple;
+		entry->tuplehead = mintuple;
 	}
 	else
 	{
 		/* push this tuple onto the tail of the list */
-		mstate->last_tuple->next = tuple;
+		MemoizeSetNextTuple(mstate->last_tuple, mintuple);
 	}
 
-	mstate->last_tuple = tuple;
+	mstate->last_tuple = mintuple;
 	MemoryContextSwitchTo(oldcontext);
 
 	/*
@@ -758,8 +797,7 @@ ExecMemoize(PlanState *pstate)
 						node->mstatus = MEMO_CACHE_FETCH_NEXT_TUPLE;
 
 						slot = node->ss.ps.ps_ResultTupleSlot;
-						ExecStoreMinimalTuple(entry->tuplehead->mintuple,
+						ExecStoreMinimalTuple(entry->tuplehead, slot, false);
-											  slot, false);
 
 						return slot;
 					}
@@ -846,7 +884,7 @@ ExecMemoize(PlanState *pstate)
 				Assert(node->last_tuple != NULL);
 
 				/* Skip to the next tuple to output */
-				node->last_tuple = node->last_tuple->next;
+				node->last_tuple = MemoizeGetNextTuple(node->last_tuple);
 
 				/* No more tuples in the cache */
 				if (node->last_tuple == NULL)
@@ -856,8 +894,7 @@ ExecMemoize(PlanState *pstate)
 				}
 
 				slot = node->ss.ps.ps_ResultTupleSlot;
-				ExecStoreMinimalTuple(node->last_tuple->mintuple, slot,
+				ExecStoreMinimalTuple(node->last_tuple, slot, false);
-									  false);
 
 				return slot;
 			}
@@ -1094,13 +1131,13 @@ ExecEndMemoize(MemoizeState *node)
 		count = 0;
 		while ((entry = memoize_iterate(node->hashtable, &i)) != NULL)
 		{
-			MemoizeTuple *tuple = entry->tuplehead;
+			MinimalTuple tuple = entry->tuplehead;
 
 			mem += EMPTY_ENTRY_MEMORY_BYTES(entry);
 			while (tuple != NULL)
 			{
 				mem += CACHE_TUPLE_BYTES(tuple);
-				tuple = tuple->next;
+				tuple = MemoizeGetNextTuple(tuple);
 			}
 			count++;
 		}
@@ -1167,13 +1204,14 @@ ExecReScanMemoize(MemoizeState *node)
 /*
  * ExecEstimateCacheEntryOverheadBytes
  *		For use in the query planner to help it estimate the amount of memory
- *		required to store a single entry in the cache.
+ *		required to store a single entry in the cache and each of the tuples
+ *		for that entry.
  */
 double
 ExecEstimateCacheEntryOverheadBytes(double ntuples)
 {
-	return sizeof(MemoizeEntry) + sizeof(MemoizeKey) + sizeof(MemoizeTuple) *
+	return sizeof(MemoizeEntry) + sizeof(MemoizeKey) +
-		ntuples;
+		MEMOIZE_NEXT_TUPLE_EXTRA_BYTES * ntuples;
 }
 
 /* ----------------------------------------------------------------

src/include/nodes/execnodes.h

@@ -2291,7 +2291,6 @@ typedef struct MaterialState
 } MaterialState;
 
 struct MemoizeEntry;
-struct MemoizeTuple;
 struct MemoizeKey;
 
 /* ----------------
@@ -2319,10 +2318,9 @@ typedef struct MemoizeState
 	uint64		mem_limit;		/* memory limit in bytes for the cache */
 	MemoryContext tableContext; /* memory context to store cache data */
 	dlist_head	lru_list;		/* least recently used entry list */
-	struct MemoizeTuple *last_tuple;	/* Used to point to the last tuple
+	MinimalTuple last_tuple;	/* Used to point to the last tuple returned
-										 * returned during a cache hit and the
+								 * during a cache hit and the tuple we last
-										 * tuple we last stored when
+								 * stored when populating the cache. */
-										 * populating the cache. */
 	struct MemoizeEntry *entry; /* the entry that 'last_tuple' belongs to or
 								 * NULL if 'last_tuple' is NULL. */
 	bool		singlerow;		/* true if the cache entry is to be marked as

src/tools/pgindent/typedefs.list

@@ -1761,7 +1761,6 @@ MemoizeInstrumentation
 MemoizeKey
 MemoizePath
 MemoizeState
-MemoizeTuple
 MemoryChunk
 MemoryContext
 MemoryContextCallback