This patch implements holdable cursors, following the proposal

(materialization into a tuple store) discussed on pgsql-hackers earlier.
I've updated the documentation and the regression tests.

Notes on the implementation:

- I needed to change the tuple store API slightly -- it assumes that it
won't be used to hold data across transaction boundaries, so the temp
files that it uses for on-disk storage are automatically reclaimed at
end-of-transaction. I added a flag to tuplestore_begin_heap() to control
this behavior. Is changing the tuple store API in this fashion OK?

- in order to store executor results in a tuple store, I added a new
CommandDest. This works well for the most part, with one exception: the
current DestFunction API doesn't provide enough information to allow the
Executor to store results into an arbitrary tuple store (where the
particular tuple store to use is chosen by the call site of
ExecutorRun). To workaround this, I've temporarily hacked up a solution
that works, but is not ideal: since the receiveTuple DestFunction is
passed the portal name, we can use that to lookup the Portal data
structure for the cursor and then use that to get at the tuple store the
Portal is using. This unnecessarily ties the Portal code with the
tupleReceiver code, but it works...

The proper fix for this is probably to change the DestFunction API --
Tom suggested passing the full QueryDesc to the receiveTuple function.
In that case, callers of ExecutorRun could "subclass" QueryDesc to add
any additional fields that their particular CommandDest needed to get
access to. This approach would work, but I'd like to think about it for
a little bit longer before deciding which route to go. In the mean time,
the code works fine, so I don't think a fix is urgent.

- (semi-related) I added a NO SCROLL keyword to DECLARE CURSOR, and
adjusted the behavior of SCROLL in accordance with the discussion on
-hackers.

- (unrelated) Cleaned up some SGML markup in sql.sgml, copy.sgml

Neil Conway

src/backend/access/transam/xact.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/access/transam/xact.c,v 1.144 2003/03/21 04:33:15 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/access/transam/xact.c,v 1.145 2003/03/27 16:51:27 momjian Exp $
  *
  * NOTES
  *		Transaction aborts can now occur two ways:
@@ -926,7 +926,7 @@ CommitTransaction(void)
 	 * access, and in fact could still cause an error...)
 	 */
 
-	AtEOXact_portals();
+	AtEOXact_portals(true);
 
 	/* handle commit for large objects [ PA, 7/17/98 ] */
 	/* XXX probably this does not belong here */
@@ -1057,7 +1057,7 @@ AbortTransaction(void)
 	 * do abort processing
 	 */
 	DeferredTriggerAbortXact();
-	AtEOXact_portals();
+	AtEOXact_portals(false);
 	lo_commit(false);			/* 'false' means it's abort */
 	AtAbort_Notify();
 	AtEOXact_UpdatePasswordFile(false);

src/backend/commands/copy.c

@@ -1,15 +1,14 @@
 /*-------------------------------------------------------------------------
  *
  * copy.c
- *		COPY command.
- *
+ *		Implements the COPY utility command.
  *
  * Portions Copyright (c) 1996-2002, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/commands/copy.c,v 1.189 2003/02/03 21:15:43 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/commands/copy.c,v 1.190 2003/03/27 16:51:27 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -50,6 +49,9 @@
 #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
 #define OCTVALUE(c) ((c) - '0')
 
+/*
+ * Represents the type of data returned by CopyReadAttribute()
+ */
 typedef enum CopyReadResult
 {
 	NORMAL_ATTR,
@@ -1311,7 +1313,7 @@ GetTypeElement(Oid type)
  * *result is set to indicate what terminated the read:
  *		NORMAL_ATTR:	column delimiter
  *		END_OF_LINE:	newline
- *		END_OF_FILE:	EOF indication
+ *		END_OF_FILE:	EOF indicator
  * In all cases, the string read up to the terminator is returned.
  *
  * Note: This function does not care about SQL NULL values -- it

src/backend/commands/portalcmds.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/commands/portalcmds.c,v 1.10 2003/03/11 19:40:22 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/commands/portalcmds.c,v 1.11 2003/03/27 16:51:27 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -17,18 +17,23 @@
 
 #include <limits.h>
 
+#include "miscadmin.h"
 #include "commands/portalcmds.h"
 #include "executor/executor.h"
 #include "optimizer/planner.h"
 #include "rewrite/rewriteHandler.h"
-
+#include "utils/memutils.h"
 
 static long DoRelativeFetch(Portal portal,
 							bool forward,
 							long count,
 							CommandDest dest);
+static long DoRelativeStoreFetch(Portal portal,
+								 bool forward,
+								 long count,
+								 CommandDest dest);
 static void DoPortalRewind(Portal portal);
-static Portal PreparePortal(char *portalName);
+static Portal PreparePortal(DeclareCursorStmt *stmt);
 
 
 /*
@@ -46,8 +51,15 @@ PerformCursorOpen(DeclareCursorStmt *stmt, CommandDest dest)
 	char	   *cursorName;
 	QueryDesc  *queryDesc;
 
-	/* Check for invalid context (must be in transaction block) */
-	RequireTransactionChain((void *) stmt, "DECLARE CURSOR");
+	/*
+	 * If this is a non-holdable cursor, we ensure that this statement
+	 * has been executed inside a transaction block (or else, it would
+	 * have no user-visible effect).
+	 *
+	 * XXX: surely there is a better way to check this?
+	 */
+	if (!(stmt->options & CURSOR_OPT_HOLD))
+		RequireTransactionChain((void *) stmt, "DECLARE CURSOR");
 
 	/*
 	 * The query has been through parse analysis, but not rewriting or
@@ -76,7 +88,7 @@ PerformCursorOpen(DeclareCursorStmt *stmt, CommandDest dest)
 	/*
 	 * Create a portal and copy the query and plan into its memory context.
 	 */
-	portal = PreparePortal(stmt->portalname);
+	portal = PreparePortal(stmt);
 
 	oldContext = MemoryContextSwitchTo(PortalGetHeapMemory(portal));
 	query = copyObject(query);
@@ -130,6 +142,7 @@ PerformPortalFetch(FetchStmt *stmt,
 	portal = GetPortalByName(stmt->portalname);
 	if (!PortalIsValid(portal))
 	{
+		/* FIXME: shouldn't this be an ERROR? */
 		elog(WARNING, "PerformPortalFetch: portal \"%s\" not found",
 			 stmt->portalname);
 		return;
@@ -343,6 +356,9 @@ DoRelativeFetch(Portal portal,
 	ScanDirection direction;
 	QueryDesc	temp_queryDesc;
 
+	if (portal->holdStore)
+		return DoRelativeStoreFetch(portal, forward, count, dest);
+
 	queryDesc = PortalGetQueryDesc(portal);
 	estate = queryDesc->estate;
 
@@ -407,7 +423,7 @@ DoRelativeFetch(Portal portal,
 	}
 	else
 	{
-		if (!portal->backwardOK)
+		if (portal->scrollType == DISABLE_SCROLL)
 			elog(ERROR, "Cursor can only scan forward"
 				 "\n\tDeclare it with SCROLL option to enable backward scan");
 
@@ -452,17 +468,85 @@ DoRelativeFetch(Portal portal,
 	return estate->es_processed;
 }
 
+/*
+ * DoRelativeStoreFetch
+ *		Do fetch for a simple N-rows-forward-or-backward case, getting
+ *		the results from the portal's tuple store.
+ */
+static long
+DoRelativeStoreFetch(Portal portal,
+					 bool forward,
+					 long count,
+					 CommandDest dest)
+{
+	DestReceiver *destfunc;
+	QueryDesc *queryDesc = portal->queryDesc;
+	long rows_fetched = 0;
+
+	if (!forward && portal->scrollType == DISABLE_SCROLL)
+			elog(ERROR, "Cursor can only scan forward"
+				 "\n\tDeclare it with SCROLL option to enable backward scan");
+
+	destfunc = DestToFunction(dest);
+	(*destfunc->setup) (destfunc, queryDesc->operation,
+						portal->name, queryDesc->tupDesc);
+
+	for (;;)
+	{
+		HeapTuple tup;
+		bool should_free;
+
+		if (rows_fetched >= count)
+			break;
+		if (portal->atEnd && forward)
+			break;
+		if (portal->atStart && !forward)
+			break;
+
+		tup = tuplestore_getheaptuple(portal->holdStore, forward, &should_free);
+
+		if (tup == NULL)
+		{
+			if (forward)
+				portal->atEnd = true;
+			else
+				portal->atStart = true;
+
+			break;
+		}
+
+		(*destfunc->receiveTuple) (tup, queryDesc->tupDesc, destfunc);
+
+		rows_fetched++;
+		if (forward)
+			portal->portalPos++;
+		else
+			portal->portalPos--;
+
+		if (forward && portal->atStart)
+			portal->atStart = false;
+		if (!forward && portal->atEnd)
+			portal->atEnd = false;
+
+		if (should_free)
+			pfree(tup);
+	}
+
+	(*destfunc->cleanup) (destfunc);
+
+	return rows_fetched;
+}
+
 /*
  * DoPortalRewind - rewind a Portal to starting point
  */
 static void
 DoPortalRewind(Portal portal)
 {
-	QueryDesc  *queryDesc;
-
-	queryDesc = PortalGetQueryDesc(portal);
-
-	ExecutorRewind(queryDesc);
+	if (portal->holdStore)
+		tuplestore_rescan(portal->holdStore);
+	else
+		ExecutorRewind(PortalGetQueryDesc(portal));
 
 	portal->atStart = true;
 	portal->atEnd = false;
@@ -493,22 +577,25 @@ PerformPortalClose(char *name)
 	/*
 	 * Note: PortalCleanup is called as a side-effect
 	 */
-	PortalDrop(portal);
+	PortalDrop(portal, false);
 }
 
-
 /*
  * PreparePortal
+ *		Given a DECLARE CURSOR statement, returns the Portal data
+ *		structure based on that statement that is used to manage the
+ *		Portal internally. If a portal with specified name already
+ *		exists, it is replaced.
  */
 static Portal
-PreparePortal(char *portalName)
+PreparePortal(DeclareCursorStmt *stmt)
 {
 	Portal		portal;
 
 	/*
 	 * Check for already-in-use portal name.
 	 */
-	portal = GetPortalByName(portalName);
+	portal = GetPortalByName(stmt->portalname);
 	if (PortalIsValid(portal))
 	{
 		/*
@@ -516,19 +603,30 @@ PreparePortal(char *portalName)
 		 * portal?
 		 */
 		elog(WARNING, "Closing pre-existing portal \"%s\"",
-			 portalName);
-		PortalDrop(portal);
+			 stmt->portalname);
+		PortalDrop(portal, false);
 	}
 
 	/*
 	 * Create the new portal.
 	 */
-	portal = CreatePortal(portalName);
+	portal = CreatePortal(stmt->portalname);
+
+	/*
+	 * Modify the newly created portal based on the options specified in
+	 * the DECLARE CURSOR statement.
+	 */
+	if (stmt->options & CURSOR_OPT_SCROLL)
+		portal->scrollType = ENABLE_SCROLL;
+	else if (stmt->options & CURSOR_OPT_NO_SCROLL)
+		portal->scrollType = DISABLE_SCROLL;
+
+	if (stmt->options & CURSOR_OPT_HOLD)
+		portal->holdOpen = true;
 
 	return portal;
 }
 
-
 /*
  * PortalCleanup
  *
@@ -545,14 +643,128 @@ PortalCleanup(Portal portal)
 	AssertArg(PortalIsValid(portal));
 	AssertArg(portal->cleanup == PortalCleanup);
 
-	/*
-	 * tell the executor to shutdown the query
-	 */
-	ExecutorEnd(PortalGetQueryDesc(portal));
+	if (portal->holdStore)
+		tuplestore_end(portal->holdStore);
+	else
+		ExecutorEnd(PortalGetQueryDesc(portal));
+
+}
+
+/*
+ * PersistHoldablePortal
+ *
+ * Prepare the specified Portal for access outside of the current
+ * transaction. When this function returns, all future accesses to the
+ * portal must be done via the Tuplestore (not by invoking the
+ * executor).
+ */
+void
+PersistHoldablePortal(Portal portal)
+{
+	MemoryContext oldcxt;
+	QueryDesc *queryDesc = PortalGetQueryDesc(portal);
 
 	/*
-	 * This should be unnecessary since the querydesc should be in the
-	 * portal's memory context, but do it anyway for symmetry.
+	 * If we're preserving a holdable portal, we had better be
+	 * inside the transaction that originally created it.
 	 */
-	FreeQueryDesc(PortalGetQueryDesc(portal));
+	Assert(portal->createXact == GetCurrentTransactionId());
+	Assert(portal->holdStore == NULL);
+
+	/*
+	 * This context is used to store portal data that needs to persist
+	 * between transactions.
+	 */
+	oldcxt = MemoryContextSwitchTo(portal->holdContext);
+
+	/* XXX: Should SortMem be used for this? */
+	portal->holdStore = tuplestore_begin_heap(true, true, SortMem);
+
+	/* Set the destination to output to the tuplestore */
+	queryDesc->dest = Tuplestore;
+
+	/*
+	 * Rewind the executor: we need to store the entire result set in
+	 * the tuplestore, so that subsequent backward FETCHs can be
+	 * processed.
+	 */
+	ExecutorRewind(queryDesc);
+
+	/* Fetch the result set into the tuplestore */
+	ExecutorRun(queryDesc, ForwardScanDirection, 0);
+
+	/*
+	 * Reset the position in the result set: ideally, this could be
+	 * implemented by just skipping straight to the tuple # that we need
+	 * to be at, but the tuplestore API doesn't support that. So we
+	 * start at the beginning of the tuplestore and iterate through it
+	 * until we reach where we need to be.
+	 */
+	if (!portal->atEnd)
+	{
+		int store_pos = 0;
+		bool should_free;
+
+		tuplestore_rescan(portal->holdStore);
+
+		while (store_pos < portal->portalPos)
+		{
+			HeapTuple tmp = tuplestore_gettuple(portal->holdStore,
+												true, &should_free);
+
+			if (tmp == NULL)
+				elog(ERROR,
+					 "PersistHoldablePortal: unexpected end of tuple stream");
+
+			store_pos++;
+
+			/*
+			 * This could probably be optimized by creating and then
+			 * deleting a separate memory context for this series of
+			 * operations.
+			 */
+			if (should_free)
+				pfree(tmp);
+		}
+	}
+
+	/*
+	 * The current Portal structure contains some data that will be
+	 * needed by the holdable cursor, but it has been allocated in a
+	 * memory context that is not sufficiently long-lived: we need to
+	 * copy it into the portal's long-term memory context.
+	 */
+	{
+		TupleDesc tupDescCopy;
+		QueryDesc *queryDescCopy;
+
+		/*
+		 * We need to use this order as ExecutorEnd invalidates the
+		 * queryDesc's tuple descriptor
+		 */
+		tupDescCopy = CreateTupleDescCopy(queryDesc->tupDesc);
+
+		ExecutorEnd(queryDesc);
+
+		queryDescCopy = palloc(sizeof(*queryDescCopy));
+
+		/*
+		 * This doesn't copy all the dependant data in the QueryDesc,
+		 * but that's okay -- the only complex field we need to keep is
+		 * the query's tupledesc, which we've copied ourselves.
+		 */
+		memcpy(queryDescCopy, queryDesc, sizeof(*queryDesc));
+
+		FreeQueryDesc(queryDesc);
+
+		queryDescCopy->tupDesc = tupDescCopy;
+		portal->queryDesc = queryDescCopy;
+	}
+
+	/*
+	 * We no longer need the portal's short-term memory context.
+	 */
+	MemoryContextDelete(PortalGetHeapMemory(portal));
+
+	PortalGetHeapMemory(portal) = NULL;
 }

src/backend/executor/Makefile

@@ -4,7 +4,7 @@
 #    Makefile for executor
 #
 # IDENTIFICATION
-#    $Header: /cvsroot/pgsql/src/backend/executor/Makefile,v 1.20 2003/01/10 23:54:24 tgl Exp $
+#    $Header: /cvsroot/pgsql/src/backend/executor/Makefile,v 1.21 2003/03/27 16:51:27 momjian Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -18,7 +18,7 @@ OBJS = execAmi.o execGrouping.o execJunk.o execMain.o \
        nodeHashjoin.o nodeIndexscan.o nodeMaterial.o nodeMergejoin.o \
        nodeNestloop.o nodeFunctionscan.o nodeResult.o nodeSeqscan.o \
        nodeSetOp.o nodeSort.o nodeUnique.o nodeLimit.o nodeGroup.o \
-       nodeSubplan.o nodeSubqueryscan.o nodeTidscan.o spi.o
+       nodeSubplan.o nodeSubqueryscan.o nodeTidscan.o tstoreReceiver.o spi.o
 
 all: SUBSYS.o
 

src/backend/executor/execMain.c

@@ -26,7 +26,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/executor/execMain.c,v 1.204 2003/03/27 14:33:11 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/executor/execMain.c,v 1.205 2003/03/27 16:51:27 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -217,8 +217,8 @@ ExecutorRun(QueryDesc *queryDesc,
 	estate->es_lastoid = InvalidOid;
 
 	destfunc = DestToFunction(dest);
-	(*destfunc->setup) (destfunc, (int) operation,
-						queryDesc->portalName, queryDesc->tupDesc);
+	(*destfunc->setup) (destfunc, operation, queryDesc->portalName,
+						queryDesc->tupDesc);
 
 	/*
 	 * run plan
@@ -421,15 +421,6 @@ ExecCheckRTEPerms(RangeTblEntry *rte, CmdType operation)
 	}
 }
 
-
-/* ===============================================================
- * ===============================================================
-						 static routines follow
- * ===============================================================
- * ===============================================================
- */
-
-
 static void
 ExecCheckXactReadOnly(Query *parsetree, CmdType operation)
 {

src/backend/executor/execQual.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/executor/execQual.c,v 1.126 2003/03/09 02:19:13 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/executor/execQual.c,v 1.127 2003/03/27 16:51:27 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -1054,8 +1054,7 @@ ExecMakeTableFunctionResult(ExprState *funcexpr,
 									   0,
 									   false);
 				}
-				tupstore = tuplestore_begin_heap(true,	/* randomAccess */
-												 SortMem);
+				tupstore = tuplestore_begin_heap(true, false, SortMem);
 				MemoryContextSwitchTo(oldcontext);
 				rsinfo.setResult = tupstore;
 				rsinfo.setDesc = tupdesc;

src/backend/executor/nodeHash.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/executor/nodeHash.c,v 1.74 2003/01/10 23:54:24 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/executor/nodeHash.c,v 1.75 2003/03/27 16:51:27 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -64,7 +64,7 @@ ExecHash(HashState *node)
 		 * buffers are palloc'd in regular executor context.
 		 */
 		for (i = 0; i < nbatch; i++)
-			hashtable->innerBatchFile[i] = BufFileCreateTemp();
+			hashtable->innerBatchFile[i] = BufFileCreateTemp(false);
 	}
 
 	/*

src/backend/executor/nodeHashjoin.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/executor/nodeHashjoin.c,v 1.48 2003/01/27 20:51:48 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/executor/nodeHashjoin.c,v 1.49 2003/03/27 16:51:27 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -138,7 +138,7 @@ ExecHashJoin(HashJoinState *node)
 		 * buffers are palloc'd in regular executor context.
 		 */
 		for (i = 0; i < hashtable->nbatch; i++)
-			hashtable->outerBatchFile[i] = BufFileCreateTemp();
+			hashtable->outerBatchFile[i] = BufFileCreateTemp(false);
 	}
 	else if (hashtable == NULL)
 		return NULL;

src/backend/executor/nodeMaterial.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/executor/nodeMaterial.c,v 1.41 2003/03/09 02:19:13 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/executor/nodeMaterial.c,v 1.42 2003/03/27 16:51:27 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -62,8 +62,7 @@ ExecMaterial(MaterialState *node)
 	 */
 	if (tuplestorestate == NULL)
 	{
-		tuplestorestate = tuplestore_begin_heap(true,	/* randomAccess */
-												SortMem);
+		tuplestorestate = tuplestore_begin_heap(true, false, SortMem);
 
 		node->tuplestorestate = (void *) tuplestorestate;
 	}

src/backend/executor/spi.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/executor/spi.c,v 1.88 2003/03/11 19:40:22 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/executor/spi.c,v 1.89 2003/03/27 16:51:28 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -866,7 +866,7 @@ SPI_cursor_close(Portal portal)
 	if (!PortalIsValid(portal))
 		elog(ERROR, "invalid portal in SPI cursor operation");
 
-	PortalDrop(portal);
+	PortalDrop(portal, false);
 }
 
 /* =================== private functions =================== */

src/backend/parser/analyze.c

@@ -6,7 +6,7 @@
  * Portions Copyright (c) 1996-2002, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- *	$Header: /cvsroot/pgsql/src/backend/parser/analyze.c,v 1.265 2003/03/10 03:53:50 tgl Exp $
+ *	$Header: /cvsroot/pgsql/src/backend/parser/analyze.c,v 1.266 2003/03/27 16:51:28 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -2294,6 +2294,13 @@ transformDeclareCursorStmt(ParseState *pstate, DeclareCursorStmt *stmt)
 	result->commandType = CMD_UTILITY;
 	result->utilityStmt = (Node *) stmt;
 
+	/*
+	 * Don't allow both SCROLL and NO SCROLL to be specified
+	 */
+	if ((stmt->options & CURSOR_OPT_SCROLL) &&
+		(stmt->options & CURSOR_OPT_NO_SCROLL))
+		elog(ERROR, "Both SCROLL and NO SCROLL cannot be specified.");
+
 	stmt->query = (Node *) transformStmt(pstate, stmt->query,
 										 &extras_before, &extras_after);
 

src/backend/parser/gram.y

@@ -11,7 +11,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/parser/gram.y,v 2.408 2003/03/20 18:52:47 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/parser/gram.y,v 2.409 2003/03/27 16:51:28 momjian Exp $
  *
  * HISTORY
  *	  AUTHOR			DATE			MAJOR EVENT
@@ -246,7 +246,7 @@ static void doNegateFloat(Value *v);
 %type <boolean> opt_freeze opt_default opt_recheck
 %type <defelt>	opt_binary opt_oids copy_delimiter
 
-%type <boolean> copy_from
+%type <boolean> copy_from opt_hold
 
 %type <ival>	reindex_type drop_type fetch_count
 				opt_column event comment_type cursor_options
@@ -348,7 +348,7 @@ static void doNegateFloat(Value *v);
 
 	GLOBAL GRANT GROUP_P
 
-	HANDLER HAVING HOUR_P
+	HANDLER HAVING HOLD HOUR_P
 
 	ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IN_P INCREMENT
 	INDEX INHERITS INITIALLY INNER_P INOUT INPUT
@@ -4230,22 +4230,31 @@ UpdateStmt: UPDATE relation_expr
  *				CURSOR STATEMENTS
  *
  *****************************************************************************/
-DeclareCursorStmt: DECLARE name cursor_options CURSOR FOR SelectStmt
+DeclareCursorStmt: DECLARE name cursor_options CURSOR opt_hold FOR SelectStmt
 				{
 					DeclareCursorStmt *n = makeNode(DeclareCursorStmt);
 					n->portalname = $2;
 					n->options = $3;
-					n->query = $6;
+					n->query = $7;
+
+					if ($5)
+						n->options |= CURSOR_OPT_HOLD;
+
 					$$ = (Node *)n;
 				}
 		;
 
 cursor_options: /*EMPTY*/					{ $$ = 0; }
-			| cursor_options BINARY			{ $$ = $1 | CURSOR_OPT_BINARY; }
+			| cursor_options NO SCROLL		{ $$ = $1 | CURSOR_OPT_NO_SCROLL; }
 			| cursor_options SCROLL			{ $$ = $1 | CURSOR_OPT_SCROLL; }
+			| cursor_options BINARY			{ $$ = $1 | CURSOR_OPT_BINARY; }
 			| cursor_options INSENSITIVE	{ $$ = $1 | CURSOR_OPT_INSENSITIVE; }
 		;
 
+opt_hold: /* EMPTY */						{ $$ = FALSE; }
+			| WITH HOLD						{ $$ = TRUE; }
+			| WITHOUT HOLD					{ $$ = FALSE; }
+
 /*****************************************************************************
  *
  *		QUERY:

src/backend/parser/keywords.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/parser/keywords.c,v 1.136 2003/03/20 07:02:10 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/parser/keywords.c,v 1.137 2003/03/27 16:51:28 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -143,6 +143,7 @@ static const ScanKeyword ScanKeywords[] = {
 	{"group", GROUP_P},
 	{"handler", HANDLER},
 	{"having", HAVING},
+	{"hold", HOLD},
 	{"hour", HOUR_P},
 	{"ilike", ILIKE},
 	{"immediate", IMMEDIATE},

src/backend/storage/file/buffile.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/storage/file/buffile.c,v 1.14 2002/09/05 00:43:07 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/storage/file/buffile.c,v 1.15 2003/03/27 16:51:29 momjian Exp $
  *
  * NOTES:
  *
@@ -64,6 +64,7 @@ struct BufFile
 	 */
 
 	bool		isTemp;			/* can only add files if this is TRUE */
+	bool		isInterTxn;		/* keep open over transactions? */
 	bool		dirty;			/* does buffer need to be written? */
 
 	/*
@@ -118,7 +119,7 @@ extendBufFile(BufFile *file)
 	File		pfile;
 
 	Assert(file->isTemp);
-	pfile = OpenTemporaryFile();
+	pfile = OpenTemporaryFile(file->isInterTxn);
 	Assert(pfile >= 0);
 
 	file->files = (File *) repalloc(file->files,
@@ -136,16 +137,17 @@ extendBufFile(BufFile *file)
  * written to it).
  */
 BufFile *
-BufFileCreateTemp(void)
+BufFileCreateTemp(bool interTxn)
 {
 	BufFile    *file;
 	File		pfile;
 
-	pfile = OpenTemporaryFile();
+	pfile = OpenTemporaryFile(interTxn);
 	Assert(pfile >= 0);
 
 	file = makeBufFile(pfile);
 	file->isTemp = true;
+	file->isInterTxn = interTxn;
 
 	return file;
 }

src/backend/storage/file/fd.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/storage/file/fd.c,v 1.95 2002/09/02 06:11:42 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/storage/file/fd.c,v 1.96 2003/03/27 16:51:29 momjian Exp $
  *
  * NOTES:
  *
@@ -112,14 +112,14 @@ int			max_files_per_process = 1000;
 
 #define FileUnknownPos (-1L)
 
+/* these are the assigned bits in fdstate below: */
+#define FD_TEMPORARY		(1 << 0)
+#define FD_TXN_TEMPORARY	(1 << 1)
+
 typedef struct vfd
 {
 	signed short fd;			/* current FD, or VFD_CLOSED if none */
 	unsigned short fdstate;		/* bitflags for VFD's state */
-
-/* these are the assigned bits in fdstate: */
-#define FD_TEMPORARY	(1 << 0)	/* should be unlinked when closed */
-
 	File		nextFree;		/* link to next free VFD, if in freelist */
 	File		lruMoreRecently;	/* doubly linked recency-of-use list */
 	File		lruLessRecently;
@@ -750,9 +750,15 @@ PathNameOpenFile(FileName fileName, int fileFlags, int fileMode)
  * This routine takes care of generating an appropriate tempfile name.
  * There's no need to pass in fileFlags or fileMode either, since only
  * one setting makes any sense for a temp file.
+ *
+ * keepOverTxn: if true, don't close the file at end-of-transaction. In
+ * most cases, you don't want temporary files to outlive the transaction
+ * that created them, so this should be false -- but if you need
+ * "somewhat" temporary storage, this might be useful. In either case,
+ * the file is removed when the File is explicitely closed.
  */
 File
-OpenTemporaryFile(void)
+OpenTemporaryFile(bool keepOverTxn)
 {
 	char		tempfilepath[128];
 	File		file;
@@ -795,9 +801,13 @@ OpenTemporaryFile(void)
 			elog(ERROR, "Failed to create temporary file %s", tempfilepath);
 	}
 
-	/* Mark it for deletion at close or EOXact */
+	/* Mark it for deletion at close */
 	VfdCache[file].fdstate |= FD_TEMPORARY;
 
+	/* Mark it for deletion at EOXact */
+	if (!keepOverTxn)
+		VfdCache[file].fdstate |= FD_TXN_TEMPORARY;
+
 	return file;
 }
 
@@ -1114,6 +1124,7 @@ AtEOXact_Files(void)
 		for (i = 1; i < SizeVfdCache; i++)
 		{
 			if ((VfdCache[i].fdstate & FD_TEMPORARY) &&
+				(VfdCache[i].fdstate & FD_TXN_TEMPORARY) &&
 				VfdCache[i].fileName != NULL)
 				FileClose(i);
 		}

src/backend/tcop/dest.c

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/tcop/dest.c,v 1.50 2003/01/21 22:06:12 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/tcop/dest.c,v 1.51 2003/03/27 16:51:29 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -29,6 +29,7 @@
 #include "postgres.h"
 
 #include "access/printtup.h"
+#include "executor/tstoreReceiver.h"
 #include "libpq/libpq.h"
 #include "libpq/pqformat.h"
 
@@ -60,9 +61,11 @@ donothingCleanup(DestReceiver *self)
 static DestReceiver donothingDR = {
 	donothingReceive, donothingSetup, donothingCleanup
 };
+
 static DestReceiver debugtupDR = {
 	debugtup, debugSetup, donothingCleanup
 };
+
 static DestReceiver spi_printtupDR = {
 	spi_printtup, spi_dest_setup, donothingCleanup
 };
@@ -98,6 +101,9 @@ DestToFunction(CommandDest dest)
 		case SPI:
 			return &spi_printtupDR;
 
+		case Tuplestore:
+			return tstoreReceiverCreateDR();
+
 		case None:
 			return &donothingDR;
 	}
@@ -122,6 +128,7 @@ EndCommand(const char *commandTag, CommandDest dest)
 
 		case None:
 		case Debug:
+		case Tuplestore:
 		case SPI:
 			break;
 	}
@@ -183,6 +190,7 @@ NullCommand(CommandDest dest)
 			break;
 
 		case Debug:
+		case Tuplestore:
 		case None:
 		default:
 			break;
@@ -213,6 +221,7 @@ ReadyForQuery(CommandDest dest)
 			break;
 
 		case Debug:
+		case Tuplestore:
 		case None:
 		default:
 			break;

src/backend/utils/mmgr/mcxt.c

@@ -14,7 +14,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/mmgr/mcxt.c,v 1.38 2002/12/16 16:22:46 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/mmgr/mcxt.c,v 1.39 2003/03/27 16:51:29 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -36,7 +36,8 @@
 MemoryContext CurrentMemoryContext = NULL;
 
 /*
- * Standard top-level contexts
+ * Standard top-level contexts. For a description of the purpose of each
+ * of these contexts, refer to src/backend/utils/mmgr/README
  */
 MemoryContext TopMemoryContext = NULL;
 MemoryContext ErrorContext = NULL;

src/backend/utils/mmgr/portalmem.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/mmgr/portalmem.c,v 1.53 2003/03/11 19:40:23 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/mmgr/portalmem.c,v 1.54 2003/03/27 16:51:29 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -20,8 +20,8 @@
  *		"PortalData" structure, plans the query and then stores the query
  *		in the portal without executing it.  Later, when the backend
  *		sees a
- *				fetch 1 from FOO
- *		the system looks up the portal named "FOO" in the portal table,
+ *				fetch 1 from foo
+ *		the system looks up the portal named "foo" in the portal table,
  *		gets the planned query and then calls the executor with a count
  *		of 1.  The executor then runs the query and returns a single
  *		tuple.	The problem is that we have to hold onto the state of the
@@ -38,7 +38,6 @@
 #include "utils/memutils.h"
 #include "utils/portal.h"
 
-
 /*
  * estimate of the maximum number of open portals a user would have,
  * used in initially sizing the PortalHashTable in EnablePortalManager()
@@ -131,8 +130,8 @@ EnablePortalManager(void)
 	ctl.entrysize = sizeof(PortalHashEnt);
 
 	/*
-	 * use PORTALS_PER_USER, defined in utils/portal.h as a guess of how
-	 * many hash table entries to create, initially
+	 * use PORTALS_PER_USER as a guess of how many hash table entries to
+	 * create, initially
 	 */
 	PortalHashTable = hash_create("Portal hash", PORTALS_PER_USER,
 								  &ctl, HASH_ELEM);
@@ -157,7 +156,9 @@ GetPortalByName(const char *name)
 
 /*
  * PortalSetQuery
- *		Attaches a "query" to portal.
+ *		Attaches a "query" to the specified portal. Note that in the
+ *		case of DECLARE CURSOR, some Portal options have already been
+ *		set based upon the parsetree of the original DECLARE statement.
  */
 void
 PortalSetQuery(Portal portal,
@@ -166,9 +167,25 @@ PortalSetQuery(Portal portal,
 {
 	AssertArg(PortalIsValid(portal));
 
+	/*
+	 * If the user didn't specify a SCROLL type, allow or disallow
+	 * scrolling based on whether it would require any additional
+	 * runtime overhead to do so.
+	 */
+	if (portal->scrollType == DEFAULT_SCROLL)
+	{
+		bool backwardPlan;
+
+		backwardPlan = ExecSupportsBackwardScan(queryDesc->plantree);
+
+		if (backwardPlan)
+			portal->scrollType = ENABLE_SCROLL;
+		else
+			portal->scrollType = DISABLE_SCROLL;
+	}
+
 	portal->queryDesc = queryDesc;
 	portal->cleanup = cleanup;
-	portal->backwardOK = ExecSupportsBackwardScan(queryDesc->plantree);
 	portal->atStart = true;
 	portal->atEnd = false;		/* allow fetches */
 	portal->portalPos = 0;
@@ -179,10 +196,8 @@ PortalSetQuery(Portal portal,
  * CreatePortal
  *		Returns a new portal given a name.
  *
- * Exceptions:
- *		BadState if called when disabled.
- *		BadArg if portal name is invalid.
- *		"WARNING" if portal name is in use (existing portal is returned!)
+ *		An elog(WARNING) is emitted if portal name is in use (existing
+ *		portal is returned!)
  */
 Portal
 CreatePortal(const char *name)
@@ -214,7 +229,11 @@ CreatePortal(const char *name)
 	/* initialize portal query */
 	portal->queryDesc = NULL;
 	portal->cleanup = NULL;
-	portal->backwardOK = false;
+	portal->scrollType = DEFAULT_SCROLL;
+	portal->holdOpen = false;
+	portal->holdStore = NULL;
+	portal->holdContext = NULL;
+	portal->createXact = GetCurrentTransactionId();
 	portal->atStart = true;
 	portal->atEnd = true;		/* disallow fetches until query is set */
 	portal->portalPos = 0;
@@ -228,17 +247,47 @@ CreatePortal(const char *name)
 
 /*
  * PortalDrop
- *		Destroys portal.
+ *		Destroy the portal.
  *
- * Exceptions:
- *		BadState if called when disabled.
- *		BadArg if portal is invalid.
+ *		keepHoldable: if true, holdable portals should not be removed by
+ *		this function. More specifically, invoking this function with
+ *		keepHoldable = true on a holdable portal prepares the portal for
+ *		access outside of its creating transaction.
  */
 void
-PortalDrop(Portal portal)
+PortalDrop(Portal portal, bool persistHoldable)
 {
 	AssertArg(PortalIsValid(portal));
 
+	if (portal->holdOpen && persistHoldable)
+	{
+		/*
+		 * We're "dropping" a holdable portal, but what we really need
+		 * to do is prepare the portal for access outside of its
+		 * creating transaction.
+		 */
+
+		/*
+		 * Create the memory context that is used for storage of
+		 * long-term (cross transaction) data needed by the holdable
+		 * portal.
+		 */
+		portal->holdContext =
+			AllocSetContextCreate(PortalMemory,
+								  "PortalHeapMemory",
+								  ALLOCSET_DEFAULT_MINSIZE,
+								  ALLOCSET_DEFAULT_INITSIZE,
+								  ALLOCSET_DEFAULT_MAXSIZE);
+
+		/*
+		 * Note that PersistHoldablePortal() releases any resources used
+		 * by the portal that are local to the creating txn.
+		 */
+		PersistHoldablePortal(portal);
+
+		return;
+	}
+
 	/* remove portal from hash table */
 	PortalHashTableDelete(portal);
 
@@ -246,8 +295,20 @@ PortalDrop(Portal portal)
 	if (PointerIsValid(portal->cleanup))
 		(*portal->cleanup) (portal);
 
-	/* release subsidiary storage */
-	MemoryContextDelete(PortalGetHeapMemory(portal));
+	/*
+	 * delete short-term memory context; in the case of a holdable
+	 * portal, this has already been done
+	 */
+	if (PortalGetHeapMemory(portal))
+		MemoryContextDelete(PortalGetHeapMemory(portal));
+
+	/*
+	 * delete long-term memory context; in the case of a non-holdable
+	 * portal, this context has never been created, so we don't need to
+	 * do anything
+	 */
+	if (portal->holdContext)
+		MemoryContextDelete(portal->holdContext);
 
 	/* release name and portal data (both are in PortalMemory) */
 	pfree(portal->name);
@@ -255,7 +316,12 @@ PortalDrop(Portal portal)
 }
 
 /*
- * Destroy all portals created in the current transaction (ie, all of them).
+ * Cleanup the portals created in the current transaction. If the
+ * transaction was aborted, all the portals created in this transaction
+ * should be removed. If the transaction was successfully committed, any
+ * holdable cursors created in this transaction need to be kept
+ * open. Only cursors created in the current transaction should be
+ * removed in this fashion.
  *
  * XXX This assumes that portals can be deleted in a random order, ie,
  * no portal has a reference to any other (at least not one that will be
@@ -264,13 +330,17 @@ PortalDrop(Portal portal)
  * references...
  */
 void
-AtEOXact_portals(void)
+AtEOXact_portals(bool isCommit)
 {
 	HASH_SEQ_STATUS status;
 	PortalHashEnt *hentry;
+	TransactionId xact = GetCurrentTransactionId();
 
 	hash_seq_init(&status, PortalHashTable);
 
 	while ((hentry = (PortalHashEnt *) hash_seq_search(&status)) != NULL)
-		PortalDrop(hentry->portal);
+	{
+		if (hentry->portal->createXact == xact)
+			PortalDrop(hentry->portal, isCommit);
+	}
 }

src/backend/utils/sort/logtape.c

@@ -64,7 +64,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/sort/logtape.c,v 1.8 2002/06/20 20:29:40 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/sort/logtape.c,v 1.9 2003/03/27 16:51:29 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -470,7 +470,7 @@ LogicalTapeSetCreate(int ntapes)
 	Assert(ntapes > 0);
 	lts = (LogicalTapeSet *) palloc(sizeof(LogicalTapeSet) +
 									(ntapes - 1) *sizeof(LogicalTape *));
-	lts->pfile = BufFileCreateTemp();
+	lts->pfile = BufFileCreateTemp(false);
 	lts->nFileBlocks = 0L;
 	lts->freeBlocksLen = 32;	/* reasonable initial guess */
 	lts->freeBlocks = (long *) palloc(lts->freeBlocksLen * sizeof(long));

src/backend/utils/sort/tuplestore.c

@@ -36,7 +36,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/sort/tuplestore.c,v 1.11 2003/03/09 02:19:13 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/sort/tuplestore.c,v 1.12 2003/03/27 16:51:29 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -65,6 +65,7 @@ struct Tuplestorestate
 {
 	TupStoreStatus status;		/* enumerated value as shown above */
 	bool		randomAccess;	/* did caller request random access? */
+	bool		interTxn;		/* keep open through transactions? */
 	long		availMem;		/* remaining memory available, in bytes */
 	BufFile    *myfile;			/* underlying file, or NULL if none */
 
@@ -190,7 +191,8 @@ struct Tuplestorestate
 
 
 static Tuplestorestate *tuplestore_begin_common(bool randomAccess,
-						int maxKBytes);
+												bool interTxn,
+												int maxKBytes);
 static void dumptuples(Tuplestorestate *state);
 static unsigned int getlen(Tuplestorestate *state, bool eofOK);
 static void *copytup_heap(Tuplestorestate *state, void *tup);
@@ -205,7 +207,7 @@ static void *readtup_heap(Tuplestorestate *state, unsigned int len);
  */
 
 static Tuplestorestate *
-tuplestore_begin_common(bool randomAccess, int maxKBytes)
+tuplestore_begin_common(bool randomAccess, bool interTxn, int maxKBytes)
 {
 	Tuplestorestate *state;
 
@@ -213,6 +215,7 @@ tuplestore_begin_common(bool randomAccess, int maxKBytes)
 
 	state->status = TSS_INMEM;
 	state->randomAccess = randomAccess;
+	state->interTxn = interTxn;
 	state->availMem = maxKBytes * 1024L;
 	state->myfile = NULL;
 
@@ -231,10 +234,27 @@ tuplestore_begin_common(bool randomAccess, int maxKBytes)
 	return state;
 }
 
+/*
+ * tuplestore_begin_heap
+ *
+ * Create a new tuplestore; other types of tuple stores (other than
+ * "heap" tuple stores, for heap tuples) are possible, but not presently
+ * implemented.
+ *
+ * randomAccess: if true, both forward and backward accesses to the
+ * tuple store are allowed.
+ *
+ * interTxn: if true, the files used by on-disk storage persist beyond
+ * the end of the current transaction.
+ *
+ * maxKBytes: how much data to store in memory (any data beyond this
+ * amount is paged to disk).
+ */
 Tuplestorestate *
-tuplestore_begin_heap(bool randomAccess, int maxKBytes)
+tuplestore_begin_heap(bool randomAccess, bool interTxn, int maxKBytes)
 {
-	Tuplestorestate *state = tuplestore_begin_common(randomAccess, maxKBytes);
+	Tuplestorestate *state = tuplestore_begin_common(randomAccess,
+													 interTxn, maxKBytes);
 
 	state->copytup = copytup_heap;
 	state->writetup = writetup_heap;
@@ -321,7 +341,7 @@ tuplestore_puttuple(Tuplestorestate *state, void *tuple)
 			/*
 			 * Nope; time to switch to tape-based operation.
 			 */
-			state->myfile = BufFileCreateTemp();
+			state->myfile = BufFileCreateTemp(state->interTxn);
 			state->status = TSS_WRITEFILE;
 			dumptuples(state);
 			break;