mcheck: Wean away from malloc hooks [BZ #23489]

Split the mcheck implementation into the debugging hooks and API so
that the API can be replicated in libc and libc_malloc_debug.so.  The
libc APIs always result in failure.

The mcheck implementation has also been moved entirely into
libc_malloc_debug.so and with it, all of the hook initialization code
can now be moved into the debug library.  Now the initialization can
be done independently of libc internals.

With this patch, libc_malloc_debug.so can no longer be used with older
libcs, which is not its goal anyway.  tst-vfork3 breaks due to this
since it spawns shell scripts, which in turn execute using the system
glibc.  Move the test to tests-container so that only the built glibc
is used.

This move also fixes bugs in the mcheck version of memalign and
realloc, thus allowing removal of the tests from tests-mcheck
exclusion list.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>
Tested-by: Carlos O'Donell <carlos@redhat.com>

malloc/malloc-debug.c

@@ -43,14 +43,96 @@ DEBUG_FN(valloc);
 DEBUG_FN(pvalloc);
 DEBUG_FN(calloc);
 
-extern void (*__free_hook) (void *, const void *);
-compat_symbol_reference (libc, __free_hook, __free_hook, GLIBC_2_0);
-extern void * (*__malloc_hook) (size_t, const void *);
-compat_symbol_reference (libc, __malloc_hook, __malloc_hook, GLIBC_2_0);
-extern void * (*__realloc_hook) (void *, size_t, const void *);
-compat_symbol_reference (libc, __realloc_hook, __realloc_hook, GLIBC_2_0);
-extern void * (*__memalign_hook) (size_t, size_t, const void *);
-compat_symbol_reference (libc, __memalign_hook, __memalign_hook, GLIBC_2_0);
+static int debug_initialized = -1;
+
+enum malloc_debug_hooks
+{
+  MALLOC_NONE_HOOK = 0,
+  MALLOC_MCHECK_HOOK = 1 << 0, /* mcheck()  */
+};
+static unsigned __malloc_debugging_hooks;
+
+static __always_inline bool
+__is_malloc_debug_enabled (enum malloc_debug_hooks flag)
+{
+  return __malloc_debugging_hooks & flag;
+}
+
+static __always_inline void
+__malloc_debug_enable (enum malloc_debug_hooks flag)
+{
+  __malloc_debugging_hooks |= flag;
+}
+
+static __always_inline void
+__malloc_debug_disable (enum malloc_debug_hooks flag)
+{
+  __malloc_debugging_hooks &= ~flag;
+}
+
+#include "mcheck.c"
+
+extern void (*__malloc_initialize_hook) (void);
+compat_symbol_reference (libc, __malloc_initialize_hook,
+			 __malloc_initialize_hook, GLIBC_2_0);
+
+static void *malloc_hook_ini (size_t, const void *) __THROW;
+static void *realloc_hook_ini (void *, size_t, const void *) __THROW;
+static void *memalign_hook_ini (size_t, size_t, const void *) __THROW;
+
+void (*__free_hook) (void *, const void *) = NULL;
+void *(*__malloc_hook) (size_t, const void *) = malloc_hook_ini;
+void *(*__realloc_hook) (void *, size_t, const void *) = realloc_hook_ini;
+void *(*__memalign_hook) (size_t, size_t, const void *) = memalign_hook_ini;
+
+/* Hooks for debugging versions.  The initial hooks just call the
+   initialization routine, then do the normal work. */
+
+/* These hooks will get executed only through the interposed allocator
+   functions in libc_malloc_debug.so.  This means that the calls to malloc,
+   realloc, etc. will lead back into the interposed functions, which is what we
+   want.
+
+   These initial hooks are assumed to be called in a single-threaded context,
+   so it is safe to reset all hooks at once upon initialization.  */
+
+static void
+generic_hook_ini (void)
+{
+  debug_initialized = 0;
+  __malloc_hook = NULL;
+  __realloc_hook = NULL;
+  __memalign_hook = NULL;
+  /* The compiler does not know that these functions are allocators, so it will
+     not try to optimize it away.  */
+  __libc_free (__libc_malloc (0));
+
+  void (*hook) (void) = __malloc_initialize_hook;
+  if (hook != NULL)
+    (*hook)();
+  debug_initialized = 1;
+}
+
+static void *
+malloc_hook_ini (size_t sz, const void *caller)
+{
+  generic_hook_ini ();
+  return __debug_malloc (sz);
+}
+
+static void *
+realloc_hook_ini (void *ptr, size_t sz, const void *caller)
+{
+  generic_hook_ini ();
+  return __debug_realloc (ptr, sz);
+}
+
+static void *
+memalign_hook_ini (size_t alignment, size_t sz, const void *caller)
+{
+  generic_hook_ini ();
+  return __debug_memalign (alignment, sz);
+}
 
 static size_t pagesize;
 
@@ -63,7 +145,17 @@ __debug_malloc (size_t bytes)
   if (__builtin_expect (hook != NULL, 0))
     return (*hook)(bytes, RETURN_ADDRESS (0));
 
-  return __libc_malloc (bytes);
+  void *victim = NULL;
+  size_t orig_bytes = bytes;
+  if (!__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK)
+      || !malloc_mcheck_before (&bytes, &victim))
+    {
+      victim = __libc_malloc (bytes);
+    }
+  if (__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK) && victim != NULL)
+    victim = malloc_mcheck_after (victim, orig_bytes);
+
+  return victim;
 }
 strong_alias (__debug_malloc, malloc)
 
@@ -76,6 +168,10 @@ __debug_free (void *mem)
       (*hook)(mem, RETURN_ADDRESS (0));
       return;
     }
+
+  if (__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK))
+    mem = free_mcheck (mem);
+
   __libc_free (mem);
 }
 strong_alias (__debug_free, free)
@@ -88,7 +184,19 @@ __debug_realloc (void *oldmem, size_t bytes)
   if (__builtin_expect (hook != NULL, 0))
     return (*hook)(oldmem, bytes, RETURN_ADDRESS (0));
 
-  return __libc_realloc (oldmem, bytes);
+  size_t orig_bytes = bytes, oldsize = 0;
+  void *victim = NULL;
+
+  if (!__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK)
+      || !realloc_mcheck_before (&oldmem, &bytes, &oldsize, &victim))
+    {
+      victim = __libc_realloc (oldmem, bytes);
+    }
+  if (__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK) && victim != NULL)
+    victim = realloc_mcheck_after (victim, oldmem, orig_bytes,
+				   oldsize);
+
+  return victim;
 }
 strong_alias (__debug_realloc, realloc)
 
@@ -100,7 +208,18 @@ _debug_mid_memalign (size_t alignment, size_t bytes, const void *address)
   if (__builtin_expect (hook != NULL, 0))
     return (*hook)(alignment, bytes, address);
 
-  return __libc_memalign (alignment, bytes);
+  void *victim = NULL;
+  size_t orig_bytes = bytes;
+
+  if (!__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK)
+      || !memalign_mcheck_before (alignment, &bytes, &victim))
+    {
+      victim = __libc_memalign (alignment, bytes);
+    }
+  if (__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK) && victim != NULL)
+    victim = memalign_mcheck_after (victim, alignment, orig_bytes);
+
+  return victim;
 }
 
 static void *
@@ -165,17 +284,17 @@ strong_alias (__debug_posix_memalign, posix_memalign)
 static void *
 __debug_calloc (size_t nmemb, size_t size)
 {
+  size_t bytes;
+
+  if (__glibc_unlikely (__builtin_mul_overflow (nmemb, size, &bytes)))
+    {
+      errno = ENOMEM;
+      return NULL;
+    }
+
   void *(*hook) (size_t, const void *) = atomic_forced_read (__malloc_hook);
   if (__builtin_expect (hook != NULL, 0))
     {
-      size_t bytes;
-
-      if (__glibc_unlikely (__builtin_mul_overflow (nmemb, size, &bytes)))
-	{
-	  errno = ENOMEM;
-	  return NULL;
-	}
-
       void *mem = (*hook)(bytes, RETURN_ADDRESS (0));
 
       if (mem != NULL)
@@ -184,6 +303,20 @@ __debug_calloc (size_t nmemb, size_t size)
       return mem;
     }
 
-  return __libc_calloc (nmemb, size);
+  size_t orig_bytes = bytes;
+  void *victim = NULL;
+
+  if (!__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK)
+      || !malloc_mcheck_before (&bytes, &victim))
+    {
+      victim = __libc_malloc (bytes);
+    }
+  if (victim != NULL)
+    {
+      if (__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK))
+	victim = malloc_mcheck_after (victim, orig_bytes);
+      memset (victim, 0, orig_bytes);
+    }
+  return victim;
 }
 strong_alias (__debug_calloc, calloc)