A bunch of changes aimed at reducing backend startup time...

Improve 'pg_internal.init' relcache entry preload mechanism so that it is
safe to use for all system catalogs, and arrange to preload a realistic
set of system-catalog entries instead of only the three nailed-in-cache
indexes that were formerly loaded this way.  Fix mechanism for deleting
out-of-date pg_internal.init files: this must be synchronized with transaction
commit, not just done at random times within transactions.  Drive it off
relcache invalidation mechanism so that no special-case tests are needed.

Cache additional information in relcache entries for indexes (their pg_index
tuples and index-operator OIDs) to eliminate repeated lookups.  Also cache
index opclass info at the per-opclass level to avoid repeated lookups during
relcache load.

Generalize 'systable scan' utilities originally developed by Hiroshi,
move them into genam.c, use in a number of places where there was formerly
ugly code for choosing either heap or index scan.  In particular this allows
simplification of the logic that prevents infinite recursion between syscache
and relcache during startup: we can easily switch to heapscans in relcache.c
when and where needed to avoid recursion, so IndexScanOK becomes simpler and
does not need any expensive initialization.

Eliminate useless opening of a heapscan data structure while doing an indexscan
(this saves an mdnblocks call and thus at least one kernel call).

src/backend/utils/cache/catcache.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/cache/catcache.c,v 1.86 2001/11/05 17:46:30 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/cache/catcache.c,v 1.87 2002/02/19 20:11:17 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -24,9 +24,13 @@
 #include "catalog/catname.h"
 #include "catalog/indexing.h"
 #include "miscadmin.h"
+#ifdef CATCACHE_STATS
+#include "storage/ipc.h"		/* for on_proc_exit */
+#endif
 #include "utils/builtins.h"
 #include "utils/fmgroids.h"
 #include "utils/catcache.h"
+#include "utils/relcache.h"
 #include "utils/syscache.h"
 
 
@@ -94,6 +98,9 @@ static Index CatalogCacheComputeTupleHashIndex(CatCache *cache,
 								  HeapTuple tuple);
 static void CatalogCacheInitializeCache(CatCache *cache);
 static Datum cc_hashname(PG_FUNCTION_ARGS);
+#ifdef CATCACHE_STATS
+static void CatCachePrintStats(void);
+#endif
 
 
 /*
@@ -147,6 +154,46 @@ cc_hashname(PG_FUNCTION_ARGS)
 }
 
 
+#ifdef CATCACHE_STATS
+
+static void
+CatCachePrintStats(void)
+{
+	CatCache   *cache;
+	long		cc_searches = 0;
+	long		cc_hits = 0;
+	long		cc_newloads = 0;
+
+	elog(DEBUG, "Catcache stats dump: %d/%d tuples in catcaches",
+		 CacheHdr->ch_ntup, CacheHdr->ch_maxtup);
+
+	for (cache = CacheHdr->ch_caches; cache; cache = cache->cc_next)
+	{
+		if (cache->cc_ntup == 0 && cache->cc_searches == 0)
+			continue;			/* don't print unused caches */
+		elog(DEBUG, "Catcache %s/%s: %d tup, %ld srch, %ld hits, %ld loads, %ld not found",
+			 cache->cc_relname,
+			 cache->cc_indname,
+			 cache->cc_ntup,
+			 cache->cc_searches,
+			 cache->cc_hits,
+			 cache->cc_newloads,
+			 cache->cc_searches - cache->cc_hits - cache->cc_newloads);
+		cc_searches += cache->cc_searches;
+		cc_hits += cache->cc_hits;
+		cc_newloads += cache->cc_newloads;
+	}
+	elog(DEBUG, "Catcache totals: %d tup, %ld srch, %ld hits, %ld loads, %ld not found",
+		 CacheHdr->ch_ntup,
+		 cc_searches,
+		 cc_hits,
+		 cc_newloads,
+		 cc_searches - cc_hits - cc_newloads);
+}
+
+#endif /* CATCACHE_STATS */
+
+
 /*
  * Standard routine for creating cache context if it doesn't exist yet
  *
@@ -289,6 +336,30 @@ CatalogCacheInitializeCache(CatCache *cache)
 	cache->cc_tupdesc = tupdesc;
 }
 
+/*
+ * InitCatCachePhase2 -- external interface for CatalogCacheInitializeCache
+ *
+ * The only reason to call this routine is to ensure that the relcache
+ * has created entries for all the catalogs and indexes referenced by
+ * catcaches.  Therefore, open the index too.  An exception is the indexes
+ * on pg_am, which we don't use (cf. IndexScanOK).
+ */
+void
+InitCatCachePhase2(CatCache *cache)
+{
+	if (cache->cc_tupdesc == NULL)
+		CatalogCacheInitializeCache(cache);
+
+	if (cache->id != AMOID &&
+		cache->id != AMNAME)
+	{
+		Relation	idesc;
+
+		idesc = index_openr(cache->cc_indname);
+		index_close(idesc);
+	}
+}
+
 /*
  *		CatalogCacheComputeHashIndex
  */
@@ -644,11 +715,11 @@ CatalogCacheFlushRelation(Oid relId)
 				{
 					bool		isNull;
 
-					tupRelid = DatumGetObjectId(
-												fastgetattr(&ct->tuple,
-													cache->cc_reloidattr,
-													   cache->cc_tupdesc,
-															&isNull));
+					tupRelid =
+						DatumGetObjectId(fastgetattr(&ct->tuple,
+													 cache->cc_reloidattr,
+													 cache->cc_tupdesc,
+													 &isNull));
 					Assert(!isNull);
 				}
 
@@ -717,20 +788,19 @@ InitCatCache(int id,
 		CacheHdr->ch_ntup = 0;
 		CacheHdr->ch_maxtup = MAXCCTUPLES;
 		DLInitList(&CacheHdr->ch_lrulist);
+#ifdef CATCACHE_STATS
+		on_proc_exit(CatCachePrintStats, 0);
+#endif
 	}
 
 	/*
 	 * allocate a new cache structure
+	 *
+	 * Note: we assume zeroing initializes the bucket headers correctly
 	 */
 	cp = (CatCache *) palloc(sizeof(CatCache) + NCCBUCKETS * sizeof(Dllist));
 	MemSet((char *) cp, 0, sizeof(CatCache) + NCCBUCKETS * sizeof(Dllist));
 
-	/*
-	 * initialize the cache buckets (each bucket is a list header)
-	 */
-	for (i = 0; i < NCCBUCKETS; ++i)
-		DLInitList(&cp->cc_bucket[i]);
-
 	/*
 	 * initialize the cache's relation information for the relation
 	 * corresponding to this cache, and initialize some of the new cache's
@@ -777,57 +847,44 @@ InitCatCache(int id,
  *		certain system indexes that support critical syscaches.
  *		We can't use an indexscan to fetch these, else we'll get into
  *		infinite recursion.  A plain heap scan will work, however.
+ *
+ *		Once we have completed relcache initialization (signaled by
+ *		criticalRelcachesBuilt), we don't have to worry anymore.
  */
 static bool
 IndexScanOK(CatCache *cache, ScanKey cur_skey)
 {
 	if (cache->id == INDEXRELID)
 	{
-		static Oid	indexSelfOid = InvalidOid;
-
-		/* One-time lookup of the OID of pg_index_indexrelid_index */
-		if (!OidIsValid(indexSelfOid))
-		{
-			Relation	rel;
-			ScanKeyData key;
-			HeapScanDesc sd;
-			HeapTuple	ntp;
-
-			rel = heap_openr(RelationRelationName, AccessShareLock);
-			ScanKeyEntryInitialize(&key, 0, Anum_pg_class_relname,
-								   F_NAMEEQ,
-								   PointerGetDatum(IndexRelidIndex));
-			sd = heap_beginscan(rel, false, SnapshotNow, 1, &key);
-			ntp = heap_getnext(sd, 0);
-			if (!HeapTupleIsValid(ntp))
-				elog(ERROR, "IndexScanOK: %s not found in %s",
-					 IndexRelidIndex, RelationRelationName);
-			indexSelfOid = ntp->t_data->t_oid;
-			heap_endscan(sd);
-			heap_close(rel, AccessShareLock);
-		}
-
-		/* Looking for pg_index_indexrelid_index? */
-		if (DatumGetObjectId(cur_skey[0].sk_argument) == indexSelfOid)
+		/*
+		 * Since the OIDs of indexes aren't hardwired, it's painful to
+		 * figure out which is which.  Just force all pg_index searches
+		 * to be heap scans while building the relcaches.
+		 */
+		if (!criticalRelcachesBuilt)
 			return false;
 	}
-	else if (cache->id == AMOPSTRATEGY ||
-			 cache->id == AMPROCNUM)
+	else if (cache->id == AMOID ||
+			 cache->id == AMNAME)
 	{
-		/* Looking for an OID or INT2 btree operator or function? */
-		Oid			lookup_oid = DatumGetObjectId(cur_skey[0].sk_argument);
-
-		if (lookup_oid == OID_BTREE_OPS_OID ||
-			lookup_oid == INT2_BTREE_OPS_OID)
-			return false;
+		/*
+		 * Always do heap scans in pg_am, because it's so small there's
+		 * not much point in an indexscan anyway.  We *must* do this when
+		 * initially building critical relcache entries, but we might as
+		 * well just always do it.
+		 */
+		return false;
 	}
 	else if (cache->id == OPEROID)
 	{
-		/* Looking for an OID comparison function? */
-		Oid			lookup_oid = DatumGetObjectId(cur_skey[0].sk_argument);
+		if (!criticalRelcachesBuilt)
+		{
+			/* Looking for an OID comparison function? */
+			Oid		lookup_oid = DatumGetObjectId(cur_skey[0].sk_argument);
 
-		if (lookup_oid >= MIN_OIDCMP && lookup_oid <= MAX_OIDCMP)
-			return false;
+			if (lookup_oid >= MIN_OIDCMP && lookup_oid <= MAX_OIDCMP)
+				return false;
+		}
 	}
 
 	/* Normal case, allow index scan */
@@ -861,6 +918,10 @@ SearchCatCache(CatCache *cache,
 	if (cache->cc_tupdesc == NULL)
 		CatalogCacheInitializeCache(cache);
 
+#ifdef CATCACHE_STATS
+	cache->cc_searches++;
+#endif
+
 	/*
 	 * initialize the search key information
 	 */
@@ -919,6 +980,10 @@ SearchCatCache(CatCache *cache,
 					cache->cc_relname, hash);
 #endif   /* CACHEDEBUG */
 
+#ifdef CATCACHE_STATS
+		cache->cc_hits++;
+#endif
+
 		return &ct->tuple;
 	}
 
@@ -1046,6 +1111,10 @@ SearchCatCache(CatCache *cache,
 	DLAddHead(&CacheHdr->ch_lrulist, &ct->lrulist_elem);
 	DLAddHead(&cache->cc_bucket[hash], &ct->cache_elem);
 
+#ifdef CATCACHE_STATS
+	cache->cc_newloads++;
+#endif
+
 	/*
 	 * If we've exceeded the desired size of the caches, try to throw away
 	 * the least recently used entry.

src/backend/utils/cache/inval.c

@@ -48,6 +48,12 @@
  *	(XXX is it worth testing likewise for duplicate catcache flush entries?
  *	Probably not.)
  *
+ *	If a relcache flush is issued for a system relation that we preload
+ *	from the relcache init file, we must also delete the init file so that
+ *	it will be rebuilt during the next backend restart.  The actual work of
+ *	manipulating the init file is in relcache.c, but we keep track of the
+ *	need for it here.
+ *
  *	All the request lists are kept in TopTransactionContext memory, since
  *	they need not live beyond the end of the current transaction.
  *
@@ -56,7 +62,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/cache/inval.c,v 1.47 2001/11/16 23:30:35 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/cache/inval.c,v 1.48 2002/02/19 20:11:17 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -107,6 +113,8 @@ typedef struct InvalidationListHeader
  */
 static InvalidationListHeader GlobalInvalidMsgs;
 
+static bool RelcacheInitFileInval; /* init file must be invalidated? */
+
 /*
  * head of invalidation message list for the current command
  * eaten by AtCommit_LocalCache() in CommandCounterIncrement()
@@ -339,6 +347,13 @@ RegisterRelcacheInvalidation(Oid dbId, Oid relId)
 								   dbId, relId);
 	AddRelcacheInvalidationMessage(&LocalInvalidMsgs,
 								   dbId, relId);
+
+	/*
+	 * If the relation being invalidated is one of those cached in the
+	 * relcache init file, mark that we need to zap that file at commit.
+	 */
+	if (RelationIdIsInInitFile(relId))
+		RelcacheInitFileInval = true;
 }
 
 /*
@@ -418,8 +433,8 @@ InvalidateSystemCaches(void)
 
 /*
  * PrepareForTupleInvalidation
- *		Invoke functions for the tuple which register invalidation
- *		of catalog/relation cache.
+ *		Detect whether invalidation of this tuple implies invalidation
+ *		of catalog/relation cache entries; if so, register inval events.
  */
 static void
 PrepareForTupleInvalidation(Relation relation, HeapTuple tuple,
@@ -525,8 +540,19 @@ AtEOXactInvalidationMessages(bool isCommit)
 {
 	if (isCommit)
 	{
+		/*
+		 * Relcache init file invalidation requires processing both
+		 * before and after we send the SI messages.  However, we need
+		 * not do anything unless we committed.
+		 */
+		if (RelcacheInitFileInval)
+			RelationCacheInitFileInvalidate(true);
+
 		ProcessInvalidationMessages(&GlobalInvalidMsgs,
 									SendSharedInvalidMessage);
+
+		if (RelcacheInitFileInval)
+			RelationCacheInitFileInvalidate(false);
 	}
 	else
 	{
@@ -534,6 +560,8 @@ AtEOXactInvalidationMessages(bool isCommit)
 									LocalExecuteInvalidationMessage);
 	}
 
+	RelcacheInitFileInval = false;
+
 	DiscardInvalidationMessages(&GlobalInvalidMsgs, false);
 	DiscardInvalidationMessages(&LocalInvalidMsgs, false);
 	DiscardInvalidationMessages(&RollbackMsgs, false);

src/backend/utils/cache/syscache.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/cache/syscache.c,v 1.66 2001/10/25 05:49:46 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/cache/syscache.c,v 1.67 2002/02/19 20:11:18 tgl Exp $
  *
  * NOTES
  *	  These routines allow the parser/planner/executor to perform
@@ -113,6 +113,16 @@ static struct cachedesc cacheinfo[] = {
 			0,
 			0
 	}},
+	{AccessMethodRelationName,	/* AMOID */
+		AmOidIndex,
+		0,
+		1,
+		{
+			ObjectIdAttributeNumber,
+			0,
+			0,
+			0
+	}},
 	{AccessMethodOperatorRelationName,	/* AMOPOPID */
 		AccessMethodOperatorIndex,
 		0,
@@ -365,8 +375,7 @@ static struct cachedesc cacheinfo[] = {
 	}}
 };
 
-static CatCache *SysCache[
-						  lengthof(cacheinfo)];
+static CatCache *SysCache[lengthof(cacheinfo)];
 static int	SysCacheSize = lengthof(cacheinfo);
 static bool CacheInitialized = false;
 
@@ -383,7 +392,7 @@ IsCacheInitialized(void)
  *
  * Note that no database access is done here; we only allocate memory
  * and initialize the cache structure.	Interrogation of the database
- * to complete initialization of a cache happens only upon first use
+ * to complete initialization of a cache happens upon first use
  * of that cache.
  */
 void
@@ -411,6 +420,32 @@ InitCatalogCache(void)
 }
 
 
+/*
+ * InitCatalogCachePhase2 - finish initializing the caches
+ *
+ * Finish initializing all the caches, including necessary database
+ * access.
+ *
+ * This is *not* essential; normally we allow syscaches to be initialized
+ * on first use.  However, it is useful as a mechanism to preload the
+ * relcache with entries for the most-commonly-used system catalogs.
+ * Therefore, we invoke this routine when we need to write a new relcache
+ * init file.
+ */
+void
+InitCatalogCachePhase2(void)
+{
+	int			cacheId;
+
+	Assert(CacheInitialized);
+
+	for (cacheId = 0; cacheId < SysCacheSize; cacheId++)
+	{
+		InitCatCachePhase2(SysCache[cacheId]);
+	}
+}
+
+
 /*
  * SearchSysCache
  *