diff --git a/tests/server/torture_server.c b/tests/server/torture_server.c
index 3f31c391..25742645 100644
--- a/tests/server/torture_server.c
+++ b/tests/server/torture_server.c
@@ -258,6 +258,9 @@ static int session_setup(void **state)
 
     assert_non_null(tss);
 
+    /* Make sure we do not test the agent */
+    unsetenv("SSH_AUTH_SOCK");
+
     cwd = torture_get_current_working_dir();
     assert_non_null(cwd);