From f29e8dd79eed86a73f6fce67e1d9776633c1db07 Mon Sep 17 00:00:00 2001 From: zhang xingxing <2388969553@qq.com> Date: Tue, 13 Oct 2026 06:39:23 +0200 Subject: [PATCH] avformat/rtmpdh: fail bn_hex2bn() when bn_new() fails with GMP and mbedTLS This is not a security issue. Fixes: d9RoImPBG3x8 Fixes: swarm-NULL-q1-passed-to-bn_modexp-when-GMP-MbedTLS-alloc-fails-c4db13 Regression since: 78efc69e7c (GMP), 62f5c9d68b (mbedTLS) NULL bignum passed to bn_modexp() Replicated through Modified FFmpeg with ASAN (GMP and mbedTLS builds, the q1 allocation in ff_dh_generate_public_key() made to fail) Found-by: zhang xingxing <2388969553@qq.com> Signed-off-by: Michael Niedermayer --- libavformat/rtmpdh.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavformat/rtmpdh.c b/libavformat/rtmpdh.c index 6a6c2ccd87..fb7aaf07c0 100644 --- a/libavformat/rtmpdh.c +++ b/libavformat/rtmpdh.c @@ -95,7 +95,7 @@ if (bn) \ ret = (mpz_set_str(bn, buf, 16) == 0); \ else \ - ret = 1; \ + ret = 0; \ } while (0) #define bn_random(bn, num_bits) \ do { \ @@ -198,7 +198,7 @@ static int bn_modexp(FFBigNum bn, FFBigNum y, FFBigNum q, FFBigNum p) if (bn) \ ret = (mbedtls_mpi_read_string(bn, 16, buf) == 0); \ else \ - ret = 1; \ + ret = 0; \ } while (0) #define bn_random(bn, num_bits) \ do { \