lib/glibc
1
0
Fork 0
mirror of https://sourceware.org/git/glibc.git synced 2025-07-28 00:21:52 +03:00
Code Activity

malloc: Move mmap code out of __libc_free hotpath

Browse Source 
Currently __libc_free checks for a freed mmap chunk in the fast path.
Also errno is always saved and restored to preserve it.  Since mmap chunks
are larger than the largest tcache chunk, it is safe to delay this and
handle tcache, smallbin and medium bin blocks first.  Move saving of errno
to cases that actually need it.  Remove a safety check that fails on mmap
chunks and a check that mmap chunks cannot be added to tcache.

Performance of bench-malloc-thread improves by 9.2% for 1 thread and
6.9% for 32 threads on Neoverse V2.

Reviewed-by: DJ Delorie  <dj@redhat.com>
Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>
This commit is contained in:
Wilco Dijkstra
2025-03-24 18:23:37 +00:00
parent 9df92958c1
commit b0cb99bef5
1 changed files with 40 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
malloc/malloc.c
View File

@ -3316,6 +3316,14 @@ tcache_init(void)
if (tcache_shutting_down)
return;
/* Check minimum mmap chunk is larger than max tcache size. This means
mmap chunks with their different layout are never added to tcache. */
if (MAX_TCACHE_SIZE >= GLRO (dl_pagesize) / 2)
malloc_printerr ("max tcache size too large");
/* Preserve errno when called from free() - _int_malloc may corrupt it. */
int err = errno;
arena_get (ar_ptr, bytes);
victim = _int_malloc (ar_ptr, bytes);
if (!victim && ar_ptr != NULL)
@ -3328,6 +3336,8 @@ tcache_init(void)
if (ar_ptr != NULL)
__libc_lock_unlock (ar_ptr->mutex);
__set_errno (err);
/* In a low memory situation, we may not be able to allocate memory
- in which case, we just keep trying later. However, we
typically do this very early, so either there is sufficient
@ -3453,37 +3463,15 @@ __libc_free (void *mem)
if (__glibc_unlikely (mtag_enabled))
*(volatile char *)mem;
int err = errno;
p = mem2chunk (mem);
if (chunk_is_mmapped (p)) /* release mmapped memory. */
{
/* See if the dynamic brk/mmap threshold needs adjusting.
Dumped fake mmapped chunks do not affect the threshold. */
if (!mp_.no_dyn_threshold
&& chunksize_nomask (p) > mp_.mmap_threshold
&& chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX)
{
mp_.mmap_threshold = chunksize (p);
mp_.trim_threshold = 2 * mp_.mmap_threshold;
LIBC_PROBE (memory_mallopt_free_dyn_thresholds, 2,
mp_.mmap_threshold, mp_.trim_threshold);
}
munmap_chunk (p);
}
else
{
MAYBE_INIT_TCACHE ();
MAYBE_INIT_TCACHE ();
/* Mark the chunk as belonging to the library again. */
(void)tag_region (chunk2mem (p), memsize (p));
/* Mark the chunk as belonging to the library again. */
tag_region (chunk2mem (p), memsize (p));
ar_ptr = arena_for_chunk (p);
_int_free (ar_ptr, p, 0);
}
__set_errno (err);
ar_ptr = arena_for_chunk (p);
_int_free (ar_ptr, p, 0);
}
libc_hidden_def (__libc_free)
@ -4570,9 +4558,8 @@ _int_free_check (mstate av, mchunkptr p, INTERNAL_SIZE_T size)
if (__builtin_expect ((uintptr_t) p > (uintptr_t) -size, 0)
|| __builtin_expect (misaligned_chunk (p), 0))
malloc_printerr ("free(): invalid pointer");
/* We know that each chunk is at least MINSIZE bytes in size or a
multiple of MALLOC_ALIGNMENT. */
if (__glibc_unlikely (size < MINSIZE || !aligned_OK (size)))
/* We know that each chunk is at least MINSIZE bytes. */
if (__glibc_unlikely (size < MINSIZE))
malloc_printerr ("free(): invalid size");
check_inuse_chunk (av, p);
@ -4669,6 +4656,9 @@ _int_free_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size, int have_lock)
else if (!chunk_is_mmapped(p)) {
/* Preserve errno in case block merging results in munmap. */
int err = errno;
/* If we're single-threaded, don't lock the arena. */
if (SINGLE_THREAD_P)
have_lock = true;
@ -4680,13 +4670,33 @@ _int_free_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size, int have_lock)
if (!have_lock)
__libc_lock_unlock (av->mutex);
__set_errno (err);
}
/*
If the chunk was allocated via mmap, release via munmap().
*/
else {
/* Preserve errno in case munmap sets it. */
int err = errno;
/* See if the dynamic brk/mmap threshold needs adjusting.
Dumped fake mmapped chunks do not affect the threshold. */
if (!mp_.no_dyn_threshold
&& chunksize_nomask (p) > mp_.mmap_threshold
&& chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX)
{
mp_.mmap_threshold = chunksize (p);
mp_.trim_threshold = 2 * mp_.mmap_threshold;
LIBC_PROBE (memory_mallopt_free_dyn_thresholds, 2,
mp_.mmap_threshold, mp_.trim_threshold);
}
munmap_chunk (p);
__set_errno (err);
}
}