diff --git a/src/backend/access/gin/ginfast.c b/src/backend/access/gin/ginfast.c
index 761bff70807..36256d6d1f5 100644
--- a/src/backend/access/gin/ginfast.c
+++ b/src/backend/access/gin/ginfast.c
@@ -897,8 +897,6 @@ ginInsertCleanup(GinState *ginstate, bool must_empty_list,
 		 */
 		processPendingPage(&accum, &datums, page, FirstOffsetNumber);
 
-		vacuum_delay_point();
-
 		/*
 		 * Is it time to flush memory to disk?	Flush if we are at the end of
 		 * the pending list, or if we have a full row and memory is getting
@@ -1004,10 +1002,12 @@ ginInsertCleanup(GinState *ginstate, bool must_empty_list,
 			UnlockReleaseBuffer(buffer);
 		}
 
+		/* call vacuum_delay_point while not holding any buffer lock */
+		vacuum_delay_point();
+
 		/*
 		 * Read next page in pending list
 		 */
-		vacuum_delay_point();
 		buffer = ReadBuffer(index, blkno);
 		LockBuffer(buffer, GIN_SHARE);
 		page = BufferGetPage(buffer);
diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c
index 0752fb38a92..6de540e72e5 100644
--- a/src/backend/access/hash/hash.c
+++ b/src/backend/access/hash/hash.c
@@ -499,6 +499,9 @@ loop_top:
 		Page		page;
 		bool		split_cleanup = false;
 
+		/* call vacuum_delay_point while not holding any buffer lock */
+		vacuum_delay_point();
+
 		/* Get address of bucket's start page */
 		bucket_blkno = BUCKET_TO_BLKNO(cachedmetap, cur_bucket);
 
@@ -712,8 +715,6 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf,
 		bool		retain_pin = false;
 		bool		clear_dead_marking = false;
 
-		vacuum_delay_point();
-
 		page = BufferGetPage(buf);
 		opaque = (HashPageOpaque) PageGetSpecialPointer(page);
 
diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index 6322d2c53b8..c27fea7cbfb 100644
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -2206,6 +2206,14 @@ vacuum_delay_point(void)
 {
 	double		msec = 0;
 
+	/*
+	 * If we're holding locks or holding interrupts for some other reason,
+	 * don't sleep, because we don't want to hold locks any longer than
+	 * necessary.
+	 */
+	if (!INTERRUPTS_CAN_BE_PROCESSED())
+		return;
+
 	/* Always check for interrupts */
 	CHECK_FOR_INTERRUPTS();