lib/glibc
1
0
Fork 0
mirror of https://sourceware.org/git/glibc.git synced 2025-07-16 07:21:57 +03:00
Code Activity

malloc: Use tailcalls in __libc_free

Browse Source 
Use tailcalls to avoid the overhead of a frame on the free fastpath.
Move tcache initialization to _int_free_chunk().  Add malloc_printerr_tail()
which can be tailcalled without forcing a frame like no-return functions.
Change tcache_double_free_verify() to retry via __libc_free() after clearing
the key.

Reviewed-by: Florian Weimer  <fweimer@redhat.com>
Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>
This commit is contained in:
Wilco Dijkstra
2025-03-31 12:19:06 +00:00
parent 393b1a6e50
commit c968fe5062
1 changed files with 24 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
malloc/malloc.c
View File

@ -1099,6 +1099,9 @@ static void* _int_memalign(mstate, size_t, size_t);
static void* _mid_memalign(size_t, size_t, void *); static void* _mid_memalign(size_t, size_t, void *);
#endif #endif
#if USE_TCACHE
static void malloc_printerr_tail(const char *str);
#endif
static void malloc_printerr(const char *str) __attribute__ ((noreturn)); static void malloc_printerr(const char *str) __attribute__ ((noreturn));
static void munmap_chunk(mchunkptr p); static void munmap_chunk(mchunkptr p);
@ -3238,9 +3241,12 @@ tcache_double_free_verify (tcache_entry *e, size_t tc_idx)
malloc_printerr ("free(): unaligned chunk detected in tcache 2"); malloc_printerr ("free(): unaligned chunk detected in tcache 2");
if (tmp == e) if (tmp == e)
malloc_printerr ("free(): double free detected in tcache 2"); malloc_printerr ("free(): double free detected in tcache 2");
/* If we get here, it was a coincidence. We've wasted a
few cycles, but don't abort. */
} }
/* No double free detected - it might be in a tcache of another thread,
or user data that happens to match the key. Since we are not sure,
clear the key and retry freeing it. */
e->key = 0;
__libc_free (e);
} }
static void static void
@ -3433,15 +3439,13 @@ __libc_free (void *mem)
p = mem2chunk (mem); p = mem2chunk (mem);
MAYBE_INIT_TCACHE ();
/* Mark the chunk as belonging to the library again. */ /* Mark the chunk as belonging to the library again. */
tag_region (chunk2mem (p), memsize (p)); tag_region (chunk2mem (p), memsize (p));
INTERNAL_SIZE_T size = chunksize (p); INTERNAL_SIZE_T size = chunksize (p);
if (__glibc_unlikely (misaligned_chunk (p))) if (__glibc_unlikely (misaligned_chunk (p)))
malloc_printerr ("free(): invalid pointer"); return malloc_printerr_tail ("free(): invalid pointer");
check_inuse_chunk (arena_for_chunk (p), p); check_inuse_chunk (arena_for_chunk (p), p);
@ -3455,7 +3459,7 @@ __libc_free (void *mem)
/* Check for double free - verify if the key matches. */ /* Check for double free - verify if the key matches. */
if (__glibc_unlikely (e->key == tcache_key)) if (__glibc_unlikely (e->key == tcache_key))
tcache_double_free_verify (e, tc_idx); return tcache_double_free_verify (e, tc_idx);
if (__glibc_likely (tcache->counts[tc_idx] < mp_.tcache_count)) if (__glibc_likely (tcache->counts[tc_idx] < mp_.tcache_count))
return tcache_put (p, tc_idx); return tcache_put (p, tc_idx);
@ -3465,7 +3469,7 @@ __libc_free (void *mem)
/* Check size >= MINSIZE and p + size does not overflow. */ /* Check size >= MINSIZE and p + size does not overflow. */
if (__glibc_unlikely (__builtin_add_overflow_p ((uintptr_t) p, size - MINSIZE, if (__glibc_unlikely (__builtin_add_overflow_p ((uintptr_t) p, size - MINSIZE,
(uintptr_t) 0))) (uintptr_t) 0)))
malloc_printerr ("free(): invalid size"); return malloc_printerr_tail ("free(): invalid size");
_int_free_chunk (arena_for_chunk (p), p, size, 0); _int_free_chunk (arena_for_chunk (p), p, size, 0);
} }
@ -4552,6 +4556,8 @@ _int_free_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size, int have_lock)
{ {
mfastbinptr *fb; /* associated fastbin */ mfastbinptr *fb; /* associated fastbin */
MAYBE_INIT_TCACHE ();
/* /*
If eligible, place chunk on a fastbin so it can be found If eligible, place chunk on a fastbin so it can be found
and used quickly in malloc. and used quickly in malloc.
@ -5813,6 +5819,17 @@ malloc_printerr (const char *str)
__builtin_unreachable (); __builtin_unreachable ();
} }
#if USE_TCACHE
static __attribute_noinline__ void
malloc_printerr_tail (const char *str)
{
/* Ensure this cannot be a no-return function. */
if (!__malloc_initialized)
return;
malloc_printerr (str);
}
#endif
#if IS_IN (libc) #if IS_IN (libc)
/* We need a wrapper function for one of the additions of POSIX. */ /* We need a wrapper function for one of the additions of POSIX. */
int int