diff --git a/elf/dl-tunables.c b/elf/dl-tunables.c index 1440d3fa6a5..5c65e8b4583 100644 --- a/elf/dl-tunables.c +++ b/elf/dl-tunables.c @@ -33,7 +33,7 @@ #include #include #include -#include +#include #define TUNABLES_INTERNAL 1 #include "dl-tunables.h" @@ -297,8 +297,9 @@ __tunables_init (char **envp) char *envval = NULL; char **prev_envp = envp; - /* Default to glibc.malloc.hugetlb=1 if DEFAULT_THP_PAGESIZE is non-zero. */ - if (DEFAULT_THP_PAGESIZE > 0) + /* Default to glibc.malloc.hugetlb=1 if MALLOC_DEFAULT_THP_PAGESIZE + is non-zero. */ + if (MALLOC_DEFAULT_THP_PAGESIZE > 0) TUNABLE_SET (glibc, malloc, hugetlb, 1); /* Ignore tunables for AT_SECURE programs. */ diff --git a/malloc/malloc-internal.h b/malloc/malloc-internal.h index dd326af1ea0..a6340bfd888 100644 --- a/malloc/malloc-internal.h +++ b/malloc/malloc-internal.h @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include /* Called in the parent process before a fork. */ diff --git a/malloc/malloc.c b/malloc/malloc.c index 6a888b0eb7d..febd7b516f4 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -1759,7 +1759,7 @@ struct malloc_par INTERNAL_SIZE_T arena_max; /* Transparent Large Page support. */ - enum malloc_thp_mode_t thp_mode; + enum thp_mode_t thp_mode; INTERNAL_SIZE_T thp_pagesize; /* A value different than 0 means to align mmap allocation to hp_pagesize add hp_flags on flags. */ @@ -1814,7 +1814,7 @@ static struct malloc_par mp_ = .trim_threshold = DEFAULT_TRIM_THRESHOLD, #define NARENAS_FROM_NCORES(n) ((n) * (sizeof (long) == 4 ? 2 : 8)) .arena_test = NARENAS_FROM_NCORES (1), - .thp_mode = malloc_thp_mode_not_supported + .thp_mode = thp_mode_not_supported #if USE_TCACHE , .tcache_count = TCACHE_FILL_COUNT, @@ -1896,7 +1896,7 @@ madvise_thp (void *p, INTERNAL_SIZE_T size) /* Only use __madvise if the system is using 'madvise' mode and the size is at least a huge page, otherwise the call is wasteful. */ - if (mp_.thp_mode != malloc_thp_mode_madvise || size < mp_.thp_pagesize) + if (mp_.thp_mode != thp_mode_madvise || size < mp_.thp_pagesize) return; /* Linux requires the input address to be page-aligned, and unaligned @@ -5017,33 +5017,32 @@ do_set_mxfast (size_t value) static __always_inline int do_set_hugetlb (size_t value) { - /* Enable THP if DEFAULT_THP_PAGESIZE is non-zero. */ - if (DEFAULT_THP_PAGESIZE > 0) + /* Enable THP if MALLOC_DEFAULT_THP_PAGESIZE is non-zero. */ + if (MALLOC_DEFAULT_THP_PAGESIZE > 0) { - mp_.thp_mode = malloc_thp_mode_madvise; - mp_.thp_pagesize = DEFAULT_THP_PAGESIZE; + mp_.thp_mode = thp_mode_madvise; + mp_.thp_pagesize = MALLOC_DEFAULT_THP_PAGESIZE; } if (value == 0) { /* Turn off THP support completely. */ - mp_.thp_mode = malloc_thp_mode_never; + mp_.thp_mode = thp_mode_never; mp_.thp_pagesize = 0; } else if (value == 1) { /* Avoid querying the THP page size/mode since accessing /sys/kernel/mm is relatively slow and might not be accessible in containers. */ - if (DEFAULT_THP_PAGESIZE > 0) + if (MALLOC_DEFAULT_THP_PAGESIZE > 0) return 0; - mp_.thp_mode = __malloc_thp_mode (); - if (mp_.thp_mode == malloc_thp_mode_madvise - || mp_.thp_mode == malloc_thp_mode_always) - mp_.thp_pagesize = __malloc_default_thp_pagesize (); + mp_.thp_mode = __get_thp_mode (); + if (mp_.thp_mode == thp_mode_madvise || mp_.thp_mode == thp_mode_always) + mp_.thp_pagesize = __get_thp_size (); } else if (value >= 2) - __malloc_hugepage_config (value == 2 ? 0 : value, &mp_.hp_pagesize, + __get_hugepage_config (value == 2 ? 0 : value, &mp_.hp_pagesize, &mp_.hp_flags); return 0; } diff --git a/sysdeps/generic/Makefile b/sysdeps/generic/Makefile index 0c25592f35b..53c4efa63e5 100644 --- a/sysdeps/generic/Makefile +++ b/sysdeps/generic/Makefile @@ -32,9 +32,9 @@ endif endif ifeq ($(subdir),malloc) -sysdep_malloc_debug_routines += malloc-hugepages +sysdep_malloc_debug_routines += hugepages endif ifeq ($(subdir),misc) -sysdep_routines += malloc-hugepages +sysdep_routines += hugepages endif diff --git a/sysdeps/generic/malloc-hugepages.c b/sysdeps/generic/hugepages.c similarity index 76% rename from sysdeps/generic/malloc-hugepages.c rename to sysdeps/generic/hugepages.c index 57b82a2438d..28bab662b13 100644 --- a/sysdeps/generic/malloc-hugepages.c +++ b/sysdeps/generic/hugepages.c @@ -16,23 +16,22 @@ License along with the GNU C Library; see the file COPYING.LIB. If not, see . */ -#include +#include unsigned long int -__malloc_default_thp_pagesize (void) +__get_thp_size (void) { return 0; } -enum malloc_thp_mode_t -__malloc_thp_mode (void) +enum thp_mode_t +__get_thp_mode (void) { - return malloc_thp_mode_not_supported; + return thp_mode_not_supported; } -/* Return the default transparent huge page size. */ void -__malloc_hugepage_config (size_t requested, size_t *pagesize, int *flags) +__get_hugepage_config (size_t requested, size_t *pagesize, int *flags) { *pagesize = 0; *flags = 0; diff --git a/sysdeps/generic/malloc-hugepages.h b/sysdeps/generic/hugepages.h similarity index 68% rename from sysdeps/generic/malloc-hugepages.h rename to sysdeps/generic/hugepages.h index ab2690f7b8a..5fc9b5c8dea 100644 --- a/sysdeps/generic/malloc-hugepages.h +++ b/sysdeps/generic/hugepages.h @@ -1,4 +1,4 @@ -/* Malloc huge page support. Generic implementation. +/* Huge page support. Generic implementation. Copyright (C) 2021-2026 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,33 +16,37 @@ License along with the GNU C Library; see the file COPYING.LIB. If not, see . */ -#ifndef _MALLOC_HUGEPAGES_H -#define _MALLOC_HUGEPAGES_H +#ifndef _HUGEPAGES_H +#define _HUGEPAGES_H #include /* Return the default transparent huge page size. */ -unsigned long int __malloc_default_thp_pagesize (void) attribute_hidden; +unsigned long int __get_thp_size (void) attribute_hidden; -enum malloc_thp_mode_t +enum thp_mode_t { - malloc_thp_mode_always, - malloc_thp_mode_madvise, - malloc_thp_mode_never, - malloc_thp_mode_not_supported + thp_mode_always, + thp_mode_madvise, + thp_mode_never, + thp_mode_not_supported }; -enum malloc_thp_mode_t __malloc_thp_mode (void) attribute_hidden; +enum thp_mode_t __get_thp_mode (void) attribute_hidden; /* Return the supported huge page size from the REQUESTED sizes on PAGESIZE along with the required extra mmap flags on FLAGS, Requesting the value of 0 returns the default huge page size, otherwise the value will be matched against the sizes supported by the system. */ -void __malloc_hugepage_config (size_t requested, size_t *pagesize, int *flags) +void __get_hugepage_config (size_t requested, size_t *pagesize, int *flags) attribute_hidden; -#ifndef DEFAULT_THP_PAGESIZE -# define DEFAULT_THP_PAGESIZE 0 +#ifndef MALLOC_DEFAULT_THP_PAGESIZE +# define MALLOC_DEFAULT_THP_PAGESIZE 0 #endif -#endif /* _MALLOC_HUGEPAGES_H */ +#ifndef MAX_THP_PAGESIZE +# define MAX_THP_PAGESIZE (32 * 1024 * 1024) +#endif + +#endif /* _HUGEPAGES_H */ diff --git a/sysdeps/unix/sysv/linux/aarch64/malloc-hugepages.h b/sysdeps/unix/sysv/linux/aarch64/hugepages.h similarity index 91% rename from sysdeps/unix/sysv/linux/aarch64/malloc-hugepages.h rename to sysdeps/unix/sysv/linux/aarch64/hugepages.h index 2204112e4e5..4f9de8a3949 100644 --- a/sysdeps/unix/sysv/linux/aarch64/malloc-hugepages.h +++ b/sysdeps/unix/sysv/linux/aarch64/hugepages.h @@ -16,6 +16,6 @@ License along with the GNU C Library; see the file COPYING.LIB. If not, see . */ -#define DEFAULT_THP_PAGESIZE (1UL << 21) +#define MALLOC_DEFAULT_THP_PAGESIZE (1UL << 21) -#include_next +#include_next diff --git a/sysdeps/unix/sysv/linux/malloc-hugepages.c b/sysdeps/unix/sysv/linux/hugepages.c similarity index 89% rename from sysdeps/unix/sysv/linux/malloc-hugepages.c rename to sysdeps/unix/sysv/linux/hugepages.c index 77b49374e60..707a75c2a29 100644 --- a/sysdeps/unix/sysv/linux/malloc-hugepages.c +++ b/sysdeps/unix/sysv/linux/hugepages.c @@ -18,16 +18,13 @@ #include #include -#include +#include #include #include unsigned long int -__malloc_default_thp_pagesize (void) +__get_thp_size (void) { - if (DEFAULT_THP_PAGESIZE != 0) - return DEFAULT_THP_PAGESIZE; - int fd = __open64_nocancel ( "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", O_RDONLY); if (fd == -1) @@ -50,13 +47,13 @@ __malloc_default_thp_pagesize (void) return r; } -enum malloc_thp_mode_t -__malloc_thp_mode (void) +enum thp_mode_t +__get_thp_mode (void) { int fd = __open64_nocancel ("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY); if (fd == -1) - return malloc_thp_mode_not_supported; + return thp_mode_not_supported; static const char mode_always[] = "[always] madvise never\n"; static const char mode_madvise[] = "always [madvise] never\n"; @@ -65,24 +62,26 @@ __malloc_thp_mode (void) char str[sizeof(mode_always)]; ssize_t s = __read_nocancel (fd, str, sizeof (str)); if (s >= sizeof str || s < 0) - return malloc_thp_mode_not_supported; + return thp_mode_not_supported; str[s] = '\0'; __close_nocancel (fd); if (s == sizeof (mode_always) - 1) { if (strcmp (str, mode_always) == 0) - return malloc_thp_mode_always; + return thp_mode_always; else if (strcmp (str, mode_madvise) == 0) - return malloc_thp_mode_madvise; + return thp_mode_madvise; else if (strcmp (str, mode_never) == 0) - return malloc_thp_mode_never; + return thp_mode_never; } - return malloc_thp_mode_not_supported; + return thp_mode_not_supported; } +#if !IS_IN (rtld) + static size_t -malloc_default_hugepage_size (void) +get_default_hugepage_size (void) { int fd = __open64_nocancel ("/proc/meminfo", O_RDONLY); if (fd == -1) @@ -136,14 +135,14 @@ hugepage_flags (size_t pagesize) } void -__malloc_hugepage_config (size_t requested, size_t *pagesize, int *flags) +__get_hugepage_config (size_t requested, size_t *pagesize, int *flags) { *pagesize = 0; *flags = 0; if (requested == 0) { - *pagesize = malloc_default_hugepage_size (); + *pagesize = get_default_hugepage_size (); if (*pagesize != 0) *flags = hugepage_flags (*pagesize); return; @@ -205,3 +204,5 @@ __malloc_hugepage_config (size_t requested, size_t *pagesize, int *flags) __close_nocancel (dirfd); } + +#endif /* !IS_IN(rtld) */