diff --git a/utils/fetch_torch_cuda_pipeline_test_matrix.py b/utils/fetch_torch_cuda_pipeline_test_matrix.py
index e6a9c4b6a3..227a60bc59 100644
--- a/utils/fetch_torch_cuda_pipeline_test_matrix.py
+++ b/utils/fetch_torch_cuda_pipeline_test_matrix.py
@@ -16,12 +16,8 @@ ALWAYS_TEST_PIPELINE_MODULES = [
     "stable_diffusion_2",
     "stable_diffusion_xl",
     "stable_diffusion_adapter",
-    "deepfloyd_if",
     "ip_adapters",
-    "kandinsky",
     "kandinsky2_2",
-    "text_to_video_synthesis",
-    "wuerstchen",
 ]
 PIPELINE_USAGE_CUTOFF = int(os.getenv("PIPELINE_USAGE_CUTOFF", 50000))