reallocarray: check for ptrdiff_t overflow

* doc/glibc-functions/reallocarray.texi (reallocarray):
Mention ptrdiff_t overflow.
* lib/reallocarray.c (reallocarray): Reindent as per usual GNU.
* lib/stdlib.in.h (reallocarray): Allow reallocarray to be replaced.
* m4/reallocarray.m4 (gl_FUNC_REALLOCARRAY):
Check for ptrdiff_t overflow.
* m4/stdlib_h.m4 (gl_STDLIB_H_DEFAULTS): Set up REPLACE_REALLOCARRAY.
* modules/reallocarray (Files): Add malloc.m4.
(configure.ac): Also test REPLACE_REALLOCARRAY.
* modules/reallocarray-tests (Depends-on): Add stdint.
* modules/stdlib (stdlib.h): Substitute REPLACE_REALLOCARRAY.
* tests/test-reallocarray.c: Include stdint.h.
(main): Check for ptrdiff_t overflow.

ChangeLog

@@ -1,5 +1,20 @@
 2021-04-24  Paul Eggert  <eggert@cs.ucla.edu>
 
+	reallocarray: check for ptrdiff_t overflow
+	* doc/glibc-functions/reallocarray.texi (reallocarray):
+	Mention ptrdiff_t overflow.
+	* lib/reallocarray.c (reallocarray): Reindent as per usual GNU.
+	* lib/stdlib.in.h (reallocarray): Allow reallocarray to be replaced.
+	* m4/reallocarray.m4 (gl_FUNC_REALLOCARRAY):
+	Check for ptrdiff_t overflow.
+	* m4/stdlib_h.m4 (gl_STDLIB_H_DEFAULTS): Set up REPLACE_REALLOCARRAY.
+	* modules/reallocarray (Files): Add malloc.m4.
+	(configure.ac): Also test REPLACE_REALLOCARRAY.
+	* modules/reallocarray-tests (Depends-on): Add stdint.
+	* modules/stdlib (stdlib.h): Substitute REPLACE_REALLOCARRAY.
+	* tests/test-reallocarray.c: Include stdint.h.
+	(main): Check for ptrdiff_t overflow.
+
 	calloc-gnu-tests: add overflow tests
 	* tests/test-calloc-gnu.c (identity): New function, replacing ‘eight’.
 	(main): Do 2 * log2(SIZE_MAX) tests instead of just two tests.

doc/glibc-functions/reallocarray.texi

@@ -26,6 +26,12 @@ Portability problems fixed by Gnulib:
 @item
 This function is missing on many platforms:
 glibc 2.25, macOS 11.1, FreeBSD 10, NetBSD 7, OpenBSD 5.5, Minix 3.3.0, AIX 7.2, HP-UX 11, IRIX, Solaris 11.4, Cygwin 1.7.x, mingw, MSVC 14, Android 9.0.
+
+@item
+On some platforms, @code{reallocarray (n, s)} can succeed even if
+multiplying @code{n} by @code{s} would exceed @code{PTRDIFF_MAX},
+which can lead to undefined behavior later:
+FreeBSD 13, NetBSD 9, OpenBSD 6, musl 1.2.
 @end itemize
 
 Portability problems not fixed by Gnulib:

lib/stdlib.in.h

@@ -1032,12 +1032,23 @@ _GL_WARN_ON_USE (realloc, "realloc is not POSIX compliant everywhere - "
 
 
 #if @GNULIB_REALLOCARRAY@
+# if @REPLACE_REALLOCARRAY@
+#  if !(defined __cplusplus && defined GNULIB_NAMESPACE)
+#   undef reallocarray
+#   define reallocarray rpl_reallocarray
+#  endif
+_GL_FUNCDECL_RPL (reallocarray, void *,
+                  (void *ptr, size_t nmemb, size_t size));
+_GL_CXXALIAS_RPL (reallocarray, void *,
+                  (void *ptr, size_t nmemb, size_t size));
+# else
 #  if ! @HAVE_REALLOCARRAY@
 _GL_FUNCDECL_SYS (reallocarray, void *,
                   (void *ptr, size_t nmemb, size_t size));
 #  endif
 _GL_CXXALIAS_SYS (reallocarray, void *,
                   (void *ptr, size_t nmemb, size_t size));
+# endif
 _GL_CXXALIASWARN (reallocarray);
 #elif defined GNULIB_POSIXCHECK
 # undef reallocarray

m4/reallocarray.m4

@@ -1,4 +1,4 @@
-# reallocarray.m4 serial 2
+# reallocarray.m4 serial 3
 dnl Copyright (C) 2017-2021 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -10,9 +10,12 @@ AC_DEFUN([gl_FUNC_REALLOCARRAY],
   AC_REQUIRE([gl_USE_SYSTEM_EXTENSIONS])
 
   AC_REQUIRE([gl_STDLIB_H_DEFAULTS])
+  AC_REQUIRE([gl_CHECK_MALLOC_PTRDIFF])
   AC_CHECK_FUNCS([reallocarray])
   if test "$ac_cv_func_reallocarray" = no; then
     HAVE_REALLOCARRAY=0
+  elif test "$gl_cv_malloc_ptrdiff" = no; then
+    REPLACE_REALLOCARRAY=1
   fi
 ])
 

m4/stdlib_h.m4

@@ -1,4 +1,4 @@
-# stdlib_h.m4 serial 62
+# stdlib_h.m4 serial 63
 dnl Copyright (C) 2007-2021 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -179,6 +179,7 @@ AC_DEFUN([gl_STDLIB_H_DEFAULTS],
   REPLACE_RANDOM=0;          AC_SUBST([REPLACE_RANDOM])
   REPLACE_RANDOM_R=0;        AC_SUBST([REPLACE_RANDOM_R])
   REPLACE_REALLOC=0;         AC_SUBST([REPLACE_REALLOC])
+  REPLACE_REALLOCARRAY=0;    AC_SUBST([REPLACE_REALLOCARRAY])
   REPLACE_REALPATH=0;        AC_SUBST([REPLACE_REALPATH])
   REPLACE_SETENV=0;          AC_SUBST([REPLACE_SETENV])
   REPLACE_SETSTATE=0;        AC_SUBST([REPLACE_SETSTATE])

modules/reallocarray

@@ -3,6 +3,7 @@ reallocarray function that is glibc compatible.
 
 Files:
 lib/reallocarray.c
+m4/malloc.m4
 m4/reallocarray.m4
 
 Depends-on:
@@ -13,7 +14,7 @@ stdlib
 
 configure.ac:
 gl_FUNC_REALLOCARRAY
-if test $HAVE_REALLOCARRAY = 0; then
+if test $HAVE_REALLOCARRAY = 0 || test $REPLACE_REALLOCARRAY = 1; then
   AC_LIBOBJ([reallocarray])
   gl_PREREQ_REALLOCARRAY
 fi

modules/reallocarray-tests

@@ -3,6 +3,7 @@ tests/test-reallocarray.c
 tests/signature.h
 
 Depends-on:
+stdint
 
 configure.ac:
 

modules/stdlib

@@ -137,6 +137,7 @@ stdlib.h: stdlib.in.h $(top_builddir)/config.status $(CXXDEFS_H) \
 	      -e 's|@''REPLACE_RANDOM''@|$(REPLACE_RANDOM)|g' \
 	      -e 's|@''REPLACE_RANDOM_R''@|$(REPLACE_RANDOM_R)|g' \
 	      -e 's|@''REPLACE_REALLOC''@|$(REPLACE_REALLOC)|g' \
+	      -e 's|@''REPLACE_REALLOCARRAY''@|$(REPLACE_REALLOCARRAY)|g' \
 	      -e 's|@''REPLACE_REALPATH''@|$(REPLACE_REALPATH)|g' \
 	      -e 's|@''REPLACE_SETENV''@|$(REPLACE_SETENV)|g' \
 	      -e 's|@''REPLACE_SETSTATE''@|$(REPLACE_SETSTATE)|g' \

tests/test-reallocarray.c

@@ -18,6 +18,7 @@
 
 #include <stdlib.h>
 #include <errno.h>
+#include <stdint.h>
 
 #include "signature.h"
 SIGNATURE_CHECK (reallocarray, void *, (void *, size_t, size_t));
@@ -25,19 +26,23 @@ SIGNATURE_CHECK (reallocarray, void *, (void *, size_t, size_t));
 int
 main ()
 {
-   size_t n;
+  void *volatile p = NULL;
 
   /* Check that reallocarray fails when requested to allocate a block
-      of memory larger than SIZE_MAX bytes.  */
-   for (n = 2; n != 0; n <<= 1)
+     of memory larger than PTRDIFF_MAX or SIZE_MAX bytes.  */
+  for (size_t n = 2; n != 0; n <<= 1)
     {
-       void *volatile p = reallocarray (NULL, (size_t) -1 / n + 1, n);
+      p = reallocarray (p, PTRDIFF_MAX / n + 1, n);
       if (p)
         return 1;
-
-       /* Ensure that errno is correctly set.  */
       if (errno != ENOMEM)
-         return 1;
+        return 2;
+
+      p = reallocarray (p, SIZE_MAX / n + 1, n);
+      if (p)
+        return 3;
+      if (errno != ENOMEM)
+        return 4;
     }
 
   return 0;