lib/glibc
1
0
Fork 0
mirror of https://sourceware.org/git/glibc.git synced 2025-08-01 10:06:57 +03:00
Code Activity

malloc: Add madvise support for Transparent Huge Pages

Browse Source 
Linux Transparent Huge Pages (THP) current supports three different
states: 'never', 'madvise', and 'always'.  The 'never' is
self-explanatory and 'always' will enable THP for all anonymous
pages.  However, 'madvise' is still the default for some system and
for such case THP will be only used if the memory range is explicity
advertise by the program through a madvise(MADV_HUGEPAGE) call.

To enable it a new tunable is provided, 'glibc.malloc.hugetlb',
where setting to a value diffent than 0 enables the madvise call.

This patch issues the madvise(MADV_HUGEPAGE) call after a successful
mmap() call at sysmalloc() with sizes larger than the default huge
page size.  The madvise() call is disable is system does not support
THP or if it has the mode set to "never" and on Linux only support
one page size for THP, even if the architecture supports multiple
sizes.

To test is a new rule is added tests-malloc-hugetlb1, which run the
addes tests with the required GLIBC_TUNABLE setting.

Checked on x86_64-linux-gnu.

Reviewed-by: DJ Delorie <dj@redhat.com>
This commit is contained in:
Adhemerval Zanella
2021-08-13 08:36:29 -03:00
parent cb976fba4c
commit 5f6d8d97c6
13 changed files with 259 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
malloc/malloc.c
View File

@ -1880,6 +1880,11 @@ struct malloc_par
INTERNAL_SIZE_T arena_test;
INTERNAL_SIZE_T arena_max;
#if HAVE_TUNABLES
/* Transparent Large Page support. */
INTERNAL_SIZE_T thp_pagesize;
#endif
/* Memory map support */
int n_mmaps;
int n_mmaps_max;
@ -2008,6 +2013,20 @@ free_perturb (char *p, size_t n)
#include <stap-probe.h>
/* ----------- Routines dealing with transparent huge pages ----------- */
static inline void
madvise_thp (void *p, INTERNAL_SIZE_T size)
{
#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
/* Do not consider areas smaller than a huge page or if the tunable is
not active. */
if (mp_.thp_pagesize == 0 || size < mp_.thp_pagesize)
return;
__madvise (p, size, MADV_HUGEPAGE);
#endif
}
/* ------------------- Support for multiple arenas -------------------- */
#include "arena.c"
@ -2445,6 +2464,8 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av)
if (mm != MAP_FAILED)
{
madvise_thp (mm, size);
/*
The offset to the start of the mmapped region is stored
in the prev_size field of the chunk. This allows us to adjust
@ -2606,6 +2627,8 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av)
if (size > 0)
{
brk = (char *) (MORECORE (size));
if (brk != (char *) (MORECORE_FAILURE))
madvise_thp (brk, size);
LIBC_PROBE (memory_sbrk_more, 2, brk, size);
}
@ -2637,6 +2660,8 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av)
if (mbrk != MAP_FAILED)
{
madvise_thp (mbrk, size);
/* We do not need, and cannot use, another sbrk call to find end */
brk = mbrk;
snd_brk = brk + size;
@ -2748,6 +2773,8 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av)
correction = 0;
snd_brk = (char *) (MORECORE (0));
}
else
madvise_thp (snd_brk, correction);
}
/* handle non-contiguous cases */
@ -2988,6 +3015,8 @@ mremap_chunk (mchunkptr p, size_t new_size)
if (cp == MAP_FAILED)
return 0;
madvise_thp (cp, new_size);
p = (mchunkptr) (cp + offset);
assert (aligned_OK (chunk2mem (p)));
@ -5316,6 +5345,24 @@ do_set_mxfast (size_t value)
return 0;
}
#if HAVE_TUNABLES
static __always_inline int
do_set_hugetlb (int32_t value)
{
if (value == 1)
{
enum malloc_thp_mode_t thp_mode = __malloc_thp_mode ();
/*
Only enable THP madvise usage if system does support it and
has 'madvise' mode. Otherwise the madvise() call is wasteful.
*/
if (thp_mode == malloc_thp_mode_madvise)
mp_.thp_pagesize = __malloc_default_thp_pagesize ();
}
return 0;
}
#endif
int
__libc_mallopt (int param_number, int value)
{