diff --git a/tests/test_modeling_utils.py b/tests/test_modeling_utils.py
index ea94f48608..697a377f8c 100755
--- a/tests/test_modeling_utils.py
+++ b/tests/test_modeling_utils.py
@@ -701,7 +701,7 @@ class PipelineTesterMixin(unittest.TestCase):
 
         prompt = "A painting of a squirrel eating a burger"
         generator = torch.manual_seed(0)
-        image = ldm([prompt], generator=generator, num_inference_steps=20)
+        image = ldm([prompt], generator=generator, num_inference_steps=1)
 
         image_slice = image[0, -1, -3:, -3:].cpu()