diff --git a/libavcodec/nvdec.c b/libavcodec/nvdec.c
index 6420483f38..a986e9be16 100644
--- a/libavcodec/nvdec.c
+++ b/libavcodec/nvdec.c
@@ -627,8 +627,7 @@ int ff_nvdec_start_frame_sep_ref(AVCodecContext *avctx, AVFrame *frame, int has_
             cf->ref_idx_ref = av_refstruct_pool_get(ctx->decoder_pool);
             if (!cf->ref_idx_ref) {
                 av_log(avctx, AV_LOG_ERROR, "No decoder surfaces left\n");
-                ret = AVERROR(ENOMEM);
-                goto fail;
+                return AVERROR(ENOMEM);
             }
         }
         cf->ref_idx = *cf->ref_idx_ref;
@@ -638,9 +637,6 @@ int ff_nvdec_start_frame_sep_ref(AVCodecContext *avctx, AVFrame *frame, int has_
     }
 
     return 0;
-fail:
-    nvdec_fdd_priv_free(cf);
-    return ret;
 }
 
 int ff_nvdec_end_frame(AVCodecContext *avctx)