Preserve relfilenodes:

Add support to pg_dump --binary-upgrade to preserve all relfilenodes,
for use by pg_migrator.

src/backend/catalog/heap.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/catalog/heap.c,v 1.364 2010/01/02 16:57:36 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/catalog/heap.c,v 1.365 2010/01/06 03:03:58 momjian Exp $
  *
  *
  * INTERFACE ROUTINES
@@ -96,6 +96,9 @@ static Node *cookConstraint(ParseState *pstate,
 			   char *relname);
 static List *insert_ordered_unique_oid(List *list, Oid datum);
 
+Oid binary_upgrade_next_heap_relfilenode = InvalidOid;
+Oid binary_upgrade_next_toast_relfilenode = InvalidOid;
+
 
 /* ----------------------------------------------------------------
  *				XXX UGLY HARD CODED BADNESS FOLLOWS XXX
@@ -942,15 +945,29 @@ heap_create_with_catalog(const char *relname,
 					 errmsg("only shared relations can be placed in pg_global tablespace")));
 	}
 
-	/*
+	if ((relkind == RELKIND_RELATION || relkind == RELKIND_SEQUENCE) &&
-	 * Allocate an OID for the relation, unless we were told what to use.
+		OidIsValid(binary_upgrade_next_heap_relfilenode))
-	 *
+	{
-	 * The OID will be the relfilenode as well, so make sure it doesn't
+		relid = binary_upgrade_next_heap_relfilenode;
-	 * collide with either pg_class OIDs or existing physical files.
+		binary_upgrade_next_heap_relfilenode = InvalidOid;
-	 */
+	}
-	if (!OidIsValid(relid))
+	else if (relkind == RELKIND_TOASTVALUE &&
+		OidIsValid(binary_upgrade_next_toast_relfilenode))
+	{
+		relid = binary_upgrade_next_toast_relfilenode;
+		binary_upgrade_next_toast_relfilenode = InvalidOid;
+	}
+	else if (!OidIsValid(relid))
+	{
+		/*
+		 * Allocate an OID for the relation, unless we were told what to use.
+		 *
+		 * The OID will be the relfilenode as well, so make sure it doesn't
+		 * collide with either pg_class OIDs or existing physical files.
+		 */
 		relid = GetNewRelFileNode(reltablespace, shared_relation,
 								  pg_class_desc);
+	}
 
 	/*
 	 * Determine the relation's initial permissions.

src/backend/catalog/index.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.328 2010/01/02 16:57:36 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.329 2010/01/06 03:03:58 momjian Exp $
  *
  *
  * INTERFACE ROUTINES
@@ -79,6 +79,9 @@ typedef struct
 				tups_inserted;
 } v_i_state;
 
+/* For simple relation creation, this is the toast index relfilenode */
+Oid binary_upgrade_next_index_relfilenode = InvalidOid;
+
 /* non-export function prototypes */
 static TupleDesc ConstructTupleDescriptor(Relation heapRelation,
 						 IndexInfo *indexInfo,
@@ -640,15 +643,22 @@ index_create(Oid heapRelationId,
 											accessMethodObjectId,
 											classObjectId);
 
-	/*
+	if (OidIsValid(binary_upgrade_next_index_relfilenode))
-	 * Allocate an OID for the index, unless we were told what to use.
+	{
-	 *
+		indexRelationId = binary_upgrade_next_index_relfilenode;
-	 * The OID will be the relfilenode as well, so make sure it doesn't
+		binary_upgrade_next_index_relfilenode = InvalidOid;
-	 * collide with either pg_class OIDs or existing physical files.
+	}
-	 */
+	else if (!OidIsValid(indexRelationId))
-	if (!OidIsValid(indexRelationId))
+	{
+		/*
+		 * Allocate an OID for the index, unless we were told what to use.
+		 *
+		 * The OID will be the relfilenode as well, so make sure it doesn't
+		 * collide with either pg_class OIDs or existing physical files.
+		 */
 		indexRelationId = GetNewRelFileNode(tableSpaceId, shared_relation,
 											pg_class);
+	}
 
 	/*
 	 * create the index relation's relcache entry and physical disk file. (If

src/backend/catalog/toasting.c

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/catalog/toasting.c,v 1.26 2010/01/02 16:57:36 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/catalog/toasting.c,v 1.27 2010/01/06 03:03:58 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -32,22 +32,17 @@
 #include "utils/syscache.h"
 
 Oid binary_upgrade_next_pg_type_toast_oid = InvalidOid;
+extern Oid binary_upgrade_next_toast_relfilenode;
 
 static bool create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid,
-				   Datum reloptions, bool force);
+				   Datum reloptions);
 static bool needs_toast_table(Relation rel);
 
 
 /*
  * AlterTableCreateToastTable
  *		If the table needs a toast table, and doesn't already have one,
- *		then create a toast table for it.  (With the force option, make
+ *		then create a toast table for it.
- *		a toast table even if it appears unnecessary.)
- *
- * The caller can also specify the OID to be used for the toast table.
- * Usually, toastOid should be InvalidOid to allow a free OID to be assigned.
- * (This option, as well as the force option, is not used by core Postgres,
- * but is provided to support pg_migrator.)
  *
  * reloptions for the toast table can be passed, too.  Pass (Datum) 0
  * for default reloptions.
@@ -57,8 +52,7 @@ static bool needs_toast_table(Relation rel);
  * to end with CommandCounterIncrement if it makes any changes.
  */
 void
-AlterTableCreateToastTable(Oid relOid, Oid toastOid,
+AlterTableCreateToastTable(Oid relOid, Datum reloptions)
-						   Datum reloptions, bool force)
 {
 	Relation	rel;
 
@@ -70,7 +64,7 @@ AlterTableCreateToastTable(Oid relOid, Oid toastOid,
 	rel = heap_open(relOid, AccessExclusiveLock);
 
 	/* create_toast_table does all the work */
-	(void) create_toast_table(rel, toastOid, InvalidOid, reloptions, force);
+	(void) create_toast_table(rel, InvalidOid, InvalidOid, reloptions);
 
 	heap_close(rel, NoLock);
 }
@@ -96,7 +90,7 @@ BootstrapToastTable(char *relName, Oid toastOid, Oid toastIndexOid)
 						relName)));
 
 	/* create_toast_table does all the work */
-	if (!create_toast_table(rel, toastOid, toastIndexOid, (Datum) 0, false))
+	if (!create_toast_table(rel, toastOid, toastIndexOid, (Datum) 0))
 		elog(ERROR, "\"%s\" does not require a toast table",
 			 relName);
 
@@ -108,12 +102,11 @@ BootstrapToastTable(char *relName, Oid toastOid, Oid toastIndexOid)
  * create_toast_table --- internal workhorse
  *
  * rel is already opened and exclusive-locked
- * toastOid and toastIndexOid are normally InvalidOid, but
+ * toastOid and toastIndexOid are normally InvalidOid, but during
- * either or both can be nonzero to specify caller-assigned OIDs
+ * bootstrap they can be nonzero to specify hand-assigned OIDs
  */
 static bool
-create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid,
+create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, Datum reloptions)
-				   Datum reloptions, bool force)
 {
 	Oid			relOid = RelationGetRelid(rel);
 	HeapTuple	reltup;
@@ -152,12 +145,10 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid,
 
 	/*
 	 * Check to see whether the table actually needs a TOAST table.
-	 *
+	 * If the relfilenode is specified, force toast file creation.
-	 * Caller can optionally override this check.  (Note: at present no
-	 * callers in core Postgres do so, but this option is needed by
-	 * pg_migrator.)
 	 */
-	if (!force && !needs_toast_table(rel))
+	if (!needs_toast_table(rel) &&
+		!OidIsValid(binary_upgrade_next_toast_relfilenode))
 		return false;
 
 	/*

src/backend/commands/cluster.c

@@ -11,7 +11,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/commands/cluster.c,v 1.189 2010/01/02 16:57:37 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/commands/cluster.c,v 1.190 2010/01/06 03:04:00 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -743,7 +743,7 @@ make_new_heap(Oid OIDOldHeap, const char *NewName, Oid NewTableSpace)
 		if (isNull)
 			reloptions = (Datum) 0;
 	}
-	AlterTableCreateToastTable(OIDNewHeap, InvalidOid, reloptions, false);
+	AlterTableCreateToastTable(OIDNewHeap, reloptions);
 
 	if (OidIsValid(toastid))
 		ReleaseSysCache(tuple);

src/backend/commands/tablecmds.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/commands/tablecmds.c,v 1.313 2010/01/02 16:57:37 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/commands/tablecmds.c,v 1.314 2010/01/06 03:04:00 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -2614,8 +2614,7 @@ ATRewriteCatalogs(List **wqueue)
 			(tab->subcmds[AT_PASS_ADD_COL] ||
 			 tab->subcmds[AT_PASS_ALTER_TYPE] ||
 			 tab->subcmds[AT_PASS_COL_ATTRS]))
-			AlterTableCreateToastTable(tab->relid, InvalidOid,
+			AlterTableCreateToastTable(tab->relid, (Datum) 0);
-									   (Datum) 0, false);
 	}
 }
 

src/backend/executor/execMain.c

@@ -26,7 +26,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/executor/execMain.c,v 1.339 2010/01/02 16:57:40 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/executor/execMain.c,v 1.340 2010/01/06 03:04:01 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -2194,7 +2194,7 @@ OpenIntoRel(QueryDesc *queryDesc)
 
 	(void) heap_reloptions(RELKIND_TOASTVALUE, reloptions, true);
 
-	AlterTableCreateToastTable(intoRelationId, InvalidOid, reloptions, false);
+	AlterTableCreateToastTable(intoRelationId, reloptions);
 
 	/*
 	 * And open the constructed table for writing.

src/backend/tcop/utility.c

@@ -10,7 +10,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/tcop/utility.c,v 1.327 2010/01/05 21:53:58 rhaas Exp $
+ *	  $PostgreSQL: pgsql/src/backend/tcop/utility.c,v 1.328 2010/01/06 03:04:01 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -492,14 +492,10 @@ standard_ProcessUtility(Node *parsetree,
 															"toast",
 															validnsps,
 															true, false);
-						(void) heap_reloptions(RELKIND_TOASTVALUE,
+						(void) heap_reloptions(RELKIND_TOASTVALUE, toast_options,
-											   toast_options,
 											   true);
 
-						AlterTableCreateToastTable(relOid,
+						AlterTableCreateToastTable(relOid, toast_options);
-												   InvalidOid,
-												   toast_options,
-												   false);
 					}
 					else
 					{

src/bin/pg_dump/pg_dump.c

@@ -12,7 +12,7 @@
  *	by PostgreSQL
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/bin/pg_dump/pg_dump.c,v 1.564 2010/01/02 16:57:59 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/bin/pg_dump/pg_dump.c,v 1.565 2010/01/06 03:04:02 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -200,7 +200,8 @@ static void binary_upgrade_set_type_oids_by_type_oid(
 					PQExpBuffer upgrade_buffer, Oid pg_type_oid);
 static bool binary_upgrade_set_type_oids_by_rel_oid(
 					PQExpBuffer upgrade_buffer, Oid pg_rel_oid);
-static void binary_upgrade_clear_pg_type_toast_oid(PQExpBuffer upgrade_buffer);
+static void binary_upgrade_set_relfilenodes(PQExpBuffer upgrade_buffer,
+					Oid pg_class_oid, bool is_index);
 static const char *getAttrName(int attrnum, TableInfo *tblInfo);
 static const char *fmtCopyColumnList(const TableInfo *ti);
 static void do_sql_command(PGconn *conn, const char *query);
@@ -2289,21 +2290,78 @@ binary_upgrade_set_type_oids_by_rel_oid(PQExpBuffer upgrade_buffer,
 }
 
 static void
-binary_upgrade_clear_pg_type_toast_oid(PQExpBuffer upgrade_buffer)
+binary_upgrade_set_relfilenodes(PQExpBuffer upgrade_buffer, Oid pg_class_oid,
+								bool is_index)
 {
-	/*
+	PQExpBuffer upgrade_query = createPQExpBuffer();
-	 *	One complexity is that while the heap might now have a TOAST table,
+	int			ntups;
-	 *	the TOAST table might have been created long after creation when
+	PGresult   *upgrade_res;
-	 *	the table was loaded with wide data.  For that reason, we clear
+	Oid			pg_class_relfilenode;
-	 *	binary_upgrade_set_next_pg_type_toast_oid so it is not reused
+	Oid			pg_class_reltoastrelid;
-	 *	by a later table.  Logically any later creation that needs a TOAST
+	Oid			pg_class_reltoastidxid;
-	 *	table should have its own TOAST pg_type oid, but we are cautious.
+
-	 */
+	appendPQExpBuffer(upgrade_query,
+					  "SELECT c.relfilenode, c.reltoastrelid, t.reltoastidxid "
+					  "FROM pg_catalog.pg_class c LEFT JOIN "
+					  "pg_catalog.pg_class t ON (c.reltoastrelid = t.oid) "
+					  "WHERE c.oid = '%u'::pg_catalog.oid;",
+					  pg_class_oid);
+
+	upgrade_res = PQexec(g_conn, upgrade_query->data);
+	check_sql_result(upgrade_res, g_conn, upgrade_query->data, PGRES_TUPLES_OK);
+
+	/* Expecting a single result only */
+	ntups = PQntuples(upgrade_res);
+	if (ntups != 1)
+	{
+		write_msg(NULL, ngettext("query returned %d row instead of one: %s\n",
+							   "query returned %d rows instead of one: %s\n",
+								 ntups),
+				  ntups, upgrade_query->data);
+		exit_nicely();
+	}
+
+	pg_class_relfilenode = atooid(PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "relfilenode")));
+	pg_class_reltoastrelid = atooid(PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "reltoastrelid")));
+	pg_class_reltoastidxid = atooid(PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "reltoastidxid")));
+
 	appendPQExpBuffer(upgrade_buffer,
-		"\n-- For binary upgrade, clear toast oid because it might not have been needed\n");
+						"\n-- For binary upgrade, must preserve relfilenodes\n");
-	appendPQExpBuffer(upgrade_buffer,
+
-		"SELECT binary_upgrade.set_next_pg_type_oid('%u'::pg_catalog.oid);\n\n",
+	if (!is_index)
-		InvalidOid);
+		appendPQExpBuffer(upgrade_buffer,
+			"SELECT binary_upgrade.set_next_heap_relfilenode('%u'::pg_catalog.oid);\n",
+			pg_class_relfilenode);
+	else
+		appendPQExpBuffer(upgrade_buffer,
+			"SELECT binary_upgrade.set_next_index_relfilenode('%u'::pg_catalog.oid);\n",
+			pg_class_relfilenode);
+	
+	if (OidIsValid(pg_class_reltoastrelid))
+	{
+		/*
+		 *  One complexity is that the table definition might not require
+		 *	the creation of a TOAST table, and the TOAST table might have
+		 *	been created long after table creation, when the table was
+		 *	loaded with wide data.  By setting the TOAST relfilenode we
+		 *	force creation of the TOAST heap and TOAST index by the
+		 *	backend so we can cleanly migrate the files during binary
+		 *	migration.
+		 */
+
+		appendPQExpBuffer(upgrade_buffer,
+			"SELECT binary_upgrade.set_next_toast_relfilenode('%u'::pg_catalog.oid);\n",
+			pg_class_reltoastrelid);
+
+		/* every toast table has an index */
+		appendPQExpBuffer(upgrade_buffer,
+			"SELECT binary_upgrade.set_next_index_relfilenode('%u'::pg_catalog.oid);\n",
+			pg_class_reltoastidxid);
+	}
+	appendPQExpBuffer(upgrade_buffer, "\n");
+
+	PQclear(upgrade_res);
+	destroyPQExpBuffer(upgrade_query);
 }
 
 /*
@@ -10480,6 +10538,9 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo)
 		appendPQExpBuffer(delq, "%s;\n",
 						  fmtId(tbinfo->dobj.name));
 
+		if (binary_upgrade)
+			binary_upgrade_set_relfilenodes(q, tbinfo->dobj.catId.oid, false);
+
 		appendPQExpBuffer(q, "CREATE TABLE %s (",
 						  fmtId(tbinfo->dobj.name));
 		actual_atts = 0;
@@ -10781,9 +10842,6 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo)
 		}
 	}
 
-	if (binary_upgrade && toast_set)
-		binary_upgrade_clear_pg_type_toast_oid(q);
-
 	ArchiveEntry(fout, tbinfo->dobj.catId, tbinfo->dobj.dumpId,
 				 tbinfo->dobj.name,
 				 tbinfo->dobj.namespace->dobj.name,
@@ -10926,6 +10984,9 @@ dumpIndex(Archive *fout, IndxInfo *indxinfo)
 	 */
 	if (indxinfo->indexconstraint == 0)
 	{
+		if (binary_upgrade)
+			binary_upgrade_set_relfilenodes(q, indxinfo->dobj.catId.oid, true);
+
 		/* Plain secondary index */
 		appendPQExpBuffer(q, "%s;\n", indxinfo->indexdef);
 
@@ -11006,6 +11067,9 @@ dumpConstraint(Archive *fout, ConstraintInfo *coninfo)
 			exit_nicely();
 		}
 
+		if (binary_upgrade && !coninfo->condef)
+			binary_upgrade_set_relfilenodes(q, indxinfo->dobj.catId.oid, true);
+
 		appendPQExpBuffer(q, "ALTER TABLE ONLY %s\n",
 						  fmtId(tbinfo->dobj.name));
 		appendPQExpBuffer(q, "    ADD CONSTRAINT %s ",
@@ -11416,7 +11480,10 @@ dumpSequence(Archive *fout, TableInfo *tbinfo)
 		resetPQExpBuffer(query);
 
 		if (binary_upgrade)
+		{
+			binary_upgrade_set_relfilenodes(query, tbinfo->dobj.catId.oid, false);
 			binary_upgrade_set_type_oids_by_rel_oid(query, tbinfo->dobj.catId.oid);
+		}
 
 		appendPQExpBuffer(query,
 						  "CREATE SEQUENCE %s\n",

src/include/catalog/toasting.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/catalog/toasting.h,v 1.12 2010/01/05 01:06:57 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/toasting.h,v 1.13 2010/01/06 03:04:03 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -17,8 +17,7 @@
 /*
  * toasting.c prototypes
  */
-extern void AlterTableCreateToastTable(Oid relOid, Oid toastOid,
+extern void AlterTableCreateToastTable(Oid relOid, Datum reloptions);
-						   Datum reloptions, bool force);
 extern void BootstrapToastTable(char *relName,
 					Oid toastOid, Oid toastIndexOid);