diff --git a/integration-cli/docker_api_containers_test.go b/integration-cli/docker_api_containers_test.go
index 2a528612b7..c02c48f981 100644
--- a/integration-cli/docker_api_containers_test.go
+++ b/integration-cli/docker_api_containers_test.go
@@ -1291,7 +1291,7 @@ func (s *DockerSuite) TestPutContainerArchiveErrSymlinkInVolumeToReadOnlyRootfs(
 		readOnly: true,
 		volumes:  defaultVolumes(testVol), // Our bind mount is at /vol2
 	})
-	defer deleteContainer(cID)
+	defer deleteContainer(false, cID)
 
 	// Attempt to extract to a symlink in the volume which points to a
 	// directory outside the volume. This should cause an error because the
diff --git a/integration-cli/docker_cli_by_digest_test.go b/integration-cli/docker_cli_by_digest_test.go
index 3b306f10b9..66a37e8cf4 100644
--- a/integration-cli/docker_cli_by_digest_test.go
+++ b/integration-cli/docker_cli_by_digest_test.go
@@ -39,7 +39,7 @@ func setupImageWithTag(c *check.C, tag string) (digest.Digest, error) {
 	c.Assert(err, checker.IsNil, check.Commentf("image tagging failed: %s", out))
 
 	// delete the container as we don't need it any more
-	err = deleteContainer(containerName)
+	err = deleteContainer(false, containerName)
 	c.Assert(err, checker.IsNil)
 
 	// push the image
diff --git a/integration-cli/docker_cli_run_test.go b/integration-cli/docker_cli_run_test.go
index c4e4b44cfe..0fb9e970e3 100644
--- a/integration-cli/docker_cli_run_test.go
+++ b/integration-cli/docker_cli_run_test.go
@@ -2117,7 +2117,7 @@ func (s *DockerSuite) TestRunDeallocatePortOnMissingIptablesRule(c *check.C) {
 	if err != nil {
 		c.Fatal(err, out)
 	}
-	if err := deleteContainer(id); err != nil {
+	if err := deleteContainer(false, id); err != nil {
 		c.Fatal(err)
 	}
 
diff --git a/integration-cli/docker_utils_test.go b/integration-cli/docker_utils_test.go
index 79f71e744d..f4d08931cc 100644
--- a/integration-cli/docker_utils_test.go
+++ b/integration-cli/docker_utils_test.go
@@ -118,8 +118,16 @@ func newRequestClient(method, endpoint string, data io.Reader, ct, daemon string
 	return req, client, nil
 }
 
-func deleteContainer(container ...string) error {
+// FIXME(vdemeester) move this away are remove ignoreNoSuchContainer bool
+func deleteContainer(ignoreNoSuchContainer bool, container ...string) error {
 	result := icmd.RunCommand(dockerBinary, append([]string{"rm", "-fv"}, container...)...)
+	if ignoreNoSuchContainer && result.Error != nil {
+		// If the error is "No such container: ..." this means the container doesn't exists anymore,
+		// we can safely ignore that one.
+		if strings.Contains(result.Error.Error(), "No such container") {
+			return nil
+		}
+	}
 	return result.Compare(icmd.Success)
 }
 
@@ -138,7 +146,7 @@ func deleteAllContainers(c *check.C) {
 	c.Assert(err, checker.IsNil, check.Commentf("containers: %v", containers))
 
 	if containers != "" {
-		err = deleteContainer(strings.Split(strings.TrimSpace(containers), "\n")...)
+		err = deleteContainer(true, strings.Split(strings.TrimSpace(containers), "\n")...)
 		c.Assert(err, checker.IsNil)
 	}
 }
@@ -596,7 +604,7 @@ func (f *remoteFileServer) Close() error {
 	if f.container == "" {
 		return nil
 	}
-	return deleteContainer(f.container)
+	return deleteContainer(false, f.container)
 }
 
 func newRemoteFileServer(ctx *FakeContext) (*remoteFileServer, error) {