Don't to predicate lock for analyze scans, refactor scan option passing.

Before this commit, when ANALYZE was run on a table and serializable
was used (either by virtue of an explicit BEGIN TRANSACTION ISOLATION
LEVEL SERIALIZABLE, or default_transaction_isolation being set to
serializable) a null pointer dereference lead to a crash.

The analyze scan doesn't need a snapshot (nor predicate locking), but
before this commit a scan only contained information about being a
bitmap or sample scan.

Refactor the option passing to the scan_begin callback to use a
bitmask instead. Alternatively we could have added a new boolean
parameter, but that seems harder to read. Even before this issue
various people (Heikki, Tom, Robert) suggested doing so.

These changes don't change the scan APIs outside of tableam. The flags
argument could be exposed, it's not necessary to fix this
problem. Also the wrapper table_beginscan* functions encapsulate most
of that complexity.

After these changes fixing the bug is trivial, just don't acquire
predicate lock for analyze style scans. That was already done for
bitmap heap scans.  Add an assert that a snapshot is passed when
acquiring the predicate lock, so this kind of bug doesn't require
running with serializable.

Also add a comment about sample scans currently requiring predicate
locking the entire relation, that previously wasn't remarked upon.

Reported-By: Joe Wildish
Author: Andres Freund
Discussion:
    https://postgr.es/m/4EA80A20-E9BF-49F1-9F01-5B66CAB21453@elusive.cx
    https://postgr.es/m/20190411164947.nkii4gaeilt4bui7@alap3.anarazel.de
    https://postgr.es/m/20190518203102.g7peu2fianukjuxm@alap3.anarazel.de

src/include/access/heapam.h

@@ -110,12 +110,7 @@ typedef enum
 extern TableScanDesc heap_beginscan(Relation relation, Snapshot snapshot,
 			   int nkeys, ScanKey key,
 			   ParallelTableScanDesc parallel_scan,
-			   bool allow_strat,
-			   bool allow_sync,
-			   bool allow_pagemode,
-			   bool is_bitmapscan,
-			   bool is_samplescan,
-			   bool temp_snap);
+			   uint32 flags);
 extern void heap_setscanlimits(TableScanDesc scan, BlockNumber startBlk,
 				   BlockNumber endBlk);
 extern void heapgetpage(TableScanDesc scan, BlockNumber page);

src/include/access/relscan.h

@@ -35,13 +35,12 @@ typedef struct TableScanDescData
 	struct SnapshotData *rs_snapshot;	/* snapshot to see */
 	int			rs_nkeys;		/* number of scan keys */
 	struct ScanKeyData *rs_key; /* array of scan key descriptors */
-	bool		rs_bitmapscan;	/* true if this is really a bitmap scan */
-	bool		rs_samplescan;	/* true if this is really a sample scan */
-	bool		rs_pageatatime; /* verify visibility page-at-a-time? */
-	bool		rs_allow_strat; /* allow or disallow use of access strategy */
-	bool		rs_allow_sync;	/* allow or disallow use of syncscan */
-	bool		rs_temp_snap;	/* unregister snapshot at scan end? */
-	bool		rs_syncscan;	/* report location to syncscan logic? */
+
+	/*
+	 * Information about type and behaviour of the scan, a bitmask of members
+	 * of the ScanOptions enum (see tableam.h).
+	 */
+	uint32		rs_flags;
 
 	struct ParallelTableScanDescData *rs_parallel;	/* parallel scan
 													 * information */

src/include/access/tableam.h

@@ -39,6 +39,28 @@ struct TBMIterateResult;
 struct VacuumParams;
 struct ValidateIndexState;
 
+/*
+ * Bitmask values for the flags argument to the scan_begin callback.
+ */
+typedef enum ScanOptions
+{
+	/* one of SO_TYPE_* may be specified */
+	SO_TYPE_SEQSCAN		= 1 << 0,
+	SO_TYPE_BITMAPSCAN	= 1 << 1,
+	SO_TYPE_SAMPLESCAN	= 1 << 2,
+	SO_TYPE_ANALYZE		= 1 << 3,
+
+	/* several of SO_ALLOW_* may be specified */
+	/* allow or disallow use of access strategy */
+	SO_ALLOW_STRAT		= 1 << 4,
+	/* report location to syncscan logic? */
+	SO_ALLOW_SYNC		= 1 << 5,
+	/* verify visibility page-at-a-time? */
+	SO_ALLOW_PAGEMODE	= 1 << 6,
+
+	/* unregister snapshot at scan end? */
+	SO_TEMP_SNAPSHOT	= 1 << 7
+} ScanOptions;
 
 /*
  * Result codes for table_{update,delete,lock_tuple}, and for visibility
@@ -78,7 +100,6 @@ typedef enum TM_Result
 	TM_WouldBlock
 } TM_Result;
 
-
 /*
  * When table_update, table_delete, or table_lock_tuple fail because the target
  * tuple is already outdated, they fill in this struct to provide information
@@ -170,26 +191,17 @@ typedef struct TableAmRoutine
 	 * parallelscan_initialize(), and has to be for the same relation. Will
 	 * only be set coming from table_beginscan_parallel().
 	 *
-	 * allow_{strat, sync, pagemode} specify whether a scan strategy,
-	 * synchronized scans, or page mode may be used (although not every AM
-	 * will support those).
-	 *
-	 * is_{bitmapscan, samplescan} specify whether the scan is intended to
-	 * support those types of scans.
-	 *
-	 * if temp_snap is true, the snapshot will need to be deallocated at
-	 * scan_end.
+	 * `flags` is a bitmask indicating the type of scan (ScanOptions's
+	 * SO_TYPE_*, currently only one may be specified), options controlling
+	 * the scan's behaviour (ScanOptions's SO_ALLOW_*, several may be
+	 * specified, an AM may ignore unsupported ones) and whether the snapshot
+	 * needs to be deallocated at scan_end (ScanOptions's SO_TEMP_SNAPSHOT).
 	 */
 	TableScanDesc (*scan_begin) (Relation rel,
 								 Snapshot snapshot,
 								 int nkeys, struct ScanKeyData *key,
 								 ParallelTableScanDesc pscan,
-								 bool allow_strat,
-								 bool allow_sync,
-								 bool allow_pagemode,
-								 bool is_bitmapscan,
-								 bool is_samplescan,
-								 bool temp_snap);
+								 uint32 flags);
 
 	/*
 	 * Release resources and deallocate scan. If TableScanDesc.temp_snap,
@@ -715,8 +727,10 @@ static inline TableScanDesc
 table_beginscan(Relation rel, Snapshot snapshot,
 				int nkeys, struct ScanKeyData *key)
 {
-	return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL,
-									   true, true, true, false, false, false);
+	uint32		flags = SO_TYPE_SEQSCAN |
+		SO_ALLOW_STRAT | SO_ALLOW_SYNC | SO_ALLOW_PAGEMODE;
+
+	return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags);
 }
 
 /*
@@ -738,9 +752,14 @@ table_beginscan_strat(Relation rel, Snapshot snapshot,
 					  int nkeys, struct ScanKeyData *key,
 					  bool allow_strat, bool allow_sync)
 {
-	return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL,
-									   allow_strat, allow_sync, true,
-									   false, false, false);
+	uint32		flags = SO_TYPE_SEQSCAN | SO_ALLOW_PAGEMODE;
+
+	if (allow_strat)
+		flags |= SO_ALLOW_STRAT;
+	if (allow_sync)
+		flags |= SO_ALLOW_SYNC;
+
+	return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags);
 }
 
 /*
@@ -753,8 +772,9 @@ static inline TableScanDesc
 table_beginscan_bm(Relation rel, Snapshot snapshot,
 				   int nkeys, struct ScanKeyData *key)
 {
-	return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL,
-									   false, false, true, true, false, false);
+	uint32		flags = SO_TYPE_BITMAPSCAN | SO_ALLOW_PAGEMODE;
+
+	return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags);
 }
 
 /*
@@ -770,9 +790,16 @@ table_beginscan_sampling(Relation rel, Snapshot snapshot,
 						 bool allow_strat, bool allow_sync,
 						 bool allow_pagemode)
 {
-	return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL,
-									   allow_strat, allow_sync, allow_pagemode,
-									   false, true, false);
+	uint32		flags = SO_TYPE_SAMPLESCAN;
+
+	if (allow_strat)
+		flags |= SO_ALLOW_STRAT;
+	if (allow_sync)
+		flags |= SO_ALLOW_SYNC;
+	if (allow_pagemode)
+		flags |= SO_ALLOW_PAGEMODE;
+
+	return rel->rd_tableam->scan_begin(rel, snapshot, nkeys, key, NULL, flags);
 }
 
 /*
@@ -783,9 +810,9 @@ table_beginscan_sampling(Relation rel, Snapshot snapshot,
 static inline TableScanDesc
 table_beginscan_analyze(Relation rel)
 {
-	return rel->rd_tableam->scan_begin(rel, NULL, 0, NULL, NULL,
-									   true, false, true,
-									   false, true, false);
+	uint32		flags = SO_TYPE_ANALYZE;
+
+	return rel->rd_tableam->scan_begin(rel, NULL, 0, NULL, NULL, flags);
 }
 
 /*