diff --git a/components/cli/plugin_install.go b/components/cli/plugin_install.go
index d0a3d517fc..407f1cddf2 100644
--- a/components/cli/plugin_install.go
+++ b/components/cli/plugin_install.go
@@ -10,7 +10,7 @@ import (
 )
 
 // PluginInstall installs a plugin
-func (cli *Client) PluginInstall(ctx context.Context, name string, options types.PluginInstallOptions) error {
+func (cli *Client) PluginInstall(ctx context.Context, name string, options types.PluginInstallOptions) (err error) {
 	// FIXME(vdemeester) name is a ref, we might want to parse/validate it here.
 	query := url.Values{}
 	query.Set("name", name)
@@ -27,6 +27,14 @@ func (cli *Client) PluginInstall(ctx context.Context, name string, options types
 		ensureReaderClosed(resp)
 		return err
 	}
+
+	defer func() {
+		if err != nil {
+			delResp, _ := cli.delete(ctx, "/plugins/"+name, nil, nil)
+			ensureReaderClosed(delResp)
+		}
+	}()
+
 	var privileges types.PluginPrivileges
 	if err := json.NewDecoder(resp.body).Decode(&privileges); err != nil {
 		ensureReaderClosed(resp)
@@ -40,8 +48,6 @@ func (cli *Client) PluginInstall(ctx context.Context, name string, options types
 			return err
 		}
 		if !accept {
-			resp, _ := cli.delete(ctx, "/plugins/"+name, nil, nil)
-			ensureReaderClosed(resp)
 			return pluginPermissionDenied{name}
 		}
 	}