diff --git a/src/hnsw.h b/src/hnsw.h
index d9bbb2f..f715a82 100644
--- a/src/hnsw.h
+++ b/src/hnsw.h
@@ -294,6 +294,7 @@ typedef struct HnswTypeInfo
 	Datum		(*dimensions) (PG_FUNCTION_ARGS);
 	Datum		(*normalize) (PG_FUNCTION_ARGS);
 	void		(*checkValue) (Pointer v);
+	bool		useFsm;
 }			HnswTypeInfo;
 
 typedef struct HnswSupport
diff --git a/src/hnswinsert.c b/src/hnswinsert.c
index 10bd3fd..b092d36 100644
--- a/src/hnswinsert.c
+++ b/src/hnswinsert.c
@@ -163,7 +163,8 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 	BlockNumber newInsertPage = InvalidBlockNumber;
 	uint8		tupleVersion;
 	char	   *base = NULL;
-	bool		tryFsm = !building;
+	bool		useFsm = !building && HnswGetTypeInfo(index)->useFsm;
+	bool		tryFsm = useFsm;
 	int			fsmTries = 0;
 
 	/* Calculate sizes */
@@ -346,7 +347,7 @@ AddElementOnDisk(Relation index, HnswElement e, int m, BlockNumber insertPage, B
 	}
 
 	/* Update free space map */
-	if (!building)
+	if (useFsm)
 		HnswRecordPageWithFreeSpace(index, e->blkno, page);
 
 	/* Commit */
diff --git a/src/hnswutils.c b/src/hnswutils.c
index 8e9c4b5..4cde2b8 100644
--- a/src/hnswutils.c
+++ b/src/hnswutils.c
@@ -1440,7 +1440,8 @@ HnswGetTypeInfo(Relation index)
 			.maxDimensions = HNSW_MAX_DIM,
 			.dimensions = vector_dims,
 			.normalize = l2_normalize,
-			.checkValue = NULL
+			.checkValue = NULL,
+			.useFsm = false
 		};
 
 		return (&typeInfo);
@@ -1457,7 +1458,8 @@ hnsw_halfvec_support(PG_FUNCTION_ARGS)
 		.maxDimensions = HNSW_MAX_DIM * 2,
 		.dimensions = halfvec_vector_dims,
 		.normalize = halfvec_l2_normalize,
-		.checkValue = NULL
+		.checkValue = NULL,
+		.useFsm = false
 	};
 
 	PG_RETURN_POINTER(&typeInfo);
@@ -1471,7 +1473,8 @@ hnsw_bit_support(PG_FUNCTION_ARGS)
 		.maxDimensions = HNSW_MAX_DIM * 32,
 		.dimensions = bitlength,
 		.normalize = NULL,
-		.checkValue = NULL
+		.checkValue = NULL,
+		.useFsm = false
 	};
 
 	PG_RETURN_POINTER(&typeInfo);
@@ -1485,7 +1488,8 @@ hnsw_sparsevec_support(PG_FUNCTION_ARGS)
 		.maxDimensions = SPARSEVEC_MAX_DIM,
 		.dimensions = sparsevec_vector_dims,
 		.normalize = sparsevec_l2_normalize,
-		.checkValue = SparsevecCheckValue
+		.checkValue = SparsevecCheckValue,
+		.useFsm = true
 	};
 
 	PG_RETURN_POINTER(&typeInfo);
diff --git a/src/hnswvacuum.c b/src/hnswvacuum.c
index cdfe1db..a1c5760 100644
--- a/src/hnswvacuum.c
+++ b/src/hnswvacuum.c
@@ -599,6 +599,7 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 	BlockNumber insertPage = InvalidBlockNumber;
 	Relation	index = vacuumstate->index;
 	BufferAccessStrategy bas = vacuumstate->bas;
+	bool		useFsm = HnswGetTypeInfo(index)->useFsm;
 
 	/*
 	 * Wait for inserts and index scans to complete. Inserts and scans before
@@ -710,15 +711,23 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 			if (nbuf != buf)
 				UnlockReleaseBuffer(nbuf);
 
+			/* Set to first free page */
+			if (!useFsm && !BlockNumberIsValid(insertPage))
+				insertPage = blkno;
+
 			/* Prepare new xlog */
 			state = GenericXLogStart(index);
 			page = GenericXLogRegisterBuffer(state, buf, 0);
 		}
 
-		HnswRecordPageWithFreeSpace(index, blkno, page);
+		if (useFsm)
+		{
+			/* Update free space map */
+			HnswRecordPageWithFreeSpace(index, blkno, page);
 
-		/* Set to last page */
-		insertPage = blkno;
+			/* Set to last page */
+			insertPage = blkno;
+		}
 
 		blkno = HnswPageGetOpaque(page)->nextblkno;
 
@@ -726,7 +735,8 @@ MarkDeleted(HnswVacuumState * vacuumstate)
 		UnlockReleaseBuffer(buf);
 	}
 
-	FreeSpaceMapVacuum(index);
+	if (useFsm)
+		FreeSpaceMapVacuum(index);
 
 	/* Update insert page last, after everything has been marked as deleted */
 	HnswUpdateMetaPage(index, 0, NULL, insertPage, MAIN_FORKNUM, false);