diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c
index 14fdadc7cc..f5f0cf7f10 100644
--- a/libavcodec/hevc/hevcdec.c
+++ b/libavcodec/hevc/hevcdec.c
@@ -768,6 +768,8 @@ static int set_sps(HEVCContext *s, HEVCLayerContext *l, const HEVCSPS *sps)
     if (!sps)
         return 0;
 
+    av_refstruct_unref(&s->vps);
+
     ret = pic_arrays_init(l, sps);
     if (ret < 0)
         goto fail;