diff mbox series

[v5,5/7] malloc: Add huge page support to arenas

Message ID 20211214185806.4109231-6-adhemerval.zanella@linaro.org
State Committed
Commit c1beb51d08d3d7ec935b0a2419b4c6fad91d1969
Headers show
Series malloc: Improve Huge Page support | expand

Checks

Context Check Description
dj/TryBot-apply_patch success Patch applied to master at the time it was sent

Commit Message

Adhemerval Zanella Dec. 14, 2021, 6:58 p.m. UTC
It is enabled as default for glibc.malloc.hugetlb set to 2 or higher.
It also uses a non configurable minimum value and maximum value,
currently set respectively to 1 and 4 selected huge page size.

The arena allocation with huge pages does not use MAP_NORESERVE.  As
indicate by kernel internal documentation [1], the flag might trigger
a SIGBUS on soft page faults if at memory access there is no left
pages in the pool.

On systems without a reserved huge pages pool, is just stress the
mmap(MAP_HUGETLB) allocation failure.  To improve test coverage it is
required to create a pool with some allocated pages.

Checked on x86_64-linux-gnu with no reserved pages, 10 reserved pages
(which trigger mmap(MAP_HUGETBL) failures) and with 256 reserved pages
(which does not trigger mmap(MAP_HUGETLB) failures).

[1] https://www.kernel.org/doc/html/v4.18/vm/hugetlbfs_reserv.html#resv-map-modifications
---
 malloc/Makefile |   7 ++-
 malloc/arena.c  | 134 +++++++++++++++++++++++++++++++++---------------
 malloc/malloc.c |   2 +-
 3 files changed, 99 insertions(+), 44 deletions(-)

Comments

DJ Delorie Dec. 15, 2021, 4:51 a.m. UTC | #1
Have you run any benchmarks against this series?  We're replacing
compile-time constants with variables, which may have a tiny affect on
performance.

LGTM
Reviewed-by: DJ Delorie <dj@redhat.com>

Adhemerval Zanella via Libc-alpha <libc-alpha@sourceware.org> writes:
>  	tst-malloc-usable \
>  	tst-malloc-usable-tunables \
>  	tst-mallocstate
> +# The tst-free-errno relies on the used malloc page size to mmap an
> +# overlapping region.
> +tests-exclude-hugetlb2 = \
> +	$(tests-exclude-hugetlb1) \
> +	tst-free-errno

Ok.

>  tests-malloc-hugetlb2 = \
> -	$(filter-out $(tests-exclude-hugetlb1), $(tests))
> +	$(filter-out $(tests-exclude-hugetlb2), $(tests))

Ok.

> diff --git a/malloc/arena.c b/malloc/arena.c
> +/* When huge pages are used to create new arenas, the maximum and minumum
> +   size are based on the runtime defined huge page size.  */
> +
> +static inline size_t
> +heap_min_size (void)
> +{
> +#if HAVE_TUNABLES
> +  return mp_.hp_pagesize == 0 ? HEAP_MIN_SIZE : mp_.hp_pagesize;
> +#else
> +  return HEAP_MIN_SIZE;
> +#endif
> +}
> +
> +static inline size_t
> +heap_max_size (void)
> +{
> +#if HAVE_TUNABLES
> +  return mp_.hp_pagesize == 0 ? HEAP_MAX_SIZE : mp_.hp_pagesize * 4;
> +#else
> +  return HEAP_MAX_SIZE;
> +#endif
> +}

Ok.

> @@ -56,10 +79,11 @@ typedef struct _heap_info
>    size_t size;   /* Current size in bytes. */
>    size_t mprotect_size; /* Size in bytes that has been mprotected
>                             PROT_READ|PROT_WRITE.  */
> +  size_t pagesize; /* Page size used when allocating the arena.  */
>    /* Make sure the following data is properly aligned, particularly
>       that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of
>       MALLOC_ALIGNMENT. */
> -  char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK];
> +  char pad[-3 * SIZE_SZ & MALLOC_ALIGN_MASK];
>  } heap_info;

Ok.

> @@ -125,10 +149,18 @@ static bool __malloc_initialized = false;
>  
>  /* find the heap and corresponding arena for a given ptr */
>  
> -#define heap_for_ptr(ptr) \
> -  ((heap_info *) ((unsigned long) (ptr) & ~(HEAP_MAX_SIZE - 1)))
> -#define arena_for_chunk(ptr) \
> -  (chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr)
> +static inline heap_info *
> +heap_for_ptr (void *ptr)
> +{
> +  size_t max_size = heap_max_size ();
> +  return PTR_ALIGN_DOWN (ptr, max_size);
> +}
> +
> +static inline struct malloc_state *
> +arena_for_chunk (mchunkptr ptr)
> +{
> +  return chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr;
> +}

Ok.


> @@ -443,71 +475,72 @@ static char *aligned_heap_area;
>     of the page size. */
>  
>  static heap_info *
> -new_heap (size_t size, size_t top_pad)
> +alloc_new_heap  (size_t size, size_t top_pad, size_t pagesize,
> +		 int mmap_flags)
>  {
> -  size_t pagesize = GLRO (dl_pagesize);

Moved to args, ok.

>    char *p1, *p2;
>    unsigned long ul;
>    heap_info *h;
> +  size_t min_size = heap_min_size ();
> +  size_t max_size = heap_max_size ();

Ok.

> -  if (size + top_pad < HEAP_MIN_SIZE)
> -    size = HEAP_MIN_SIZE;
> -  else if (size + top_pad <= HEAP_MAX_SIZE)
> +  if (size + top_pad < min_size)
> +    size = min_size;
> +  else if (size + top_pad <= max_size)
>      size += top_pad;
> -  else if (size > HEAP_MAX_SIZE)
> +  else if (size > max_size)
>      return 0;
>    else
> -    size = HEAP_MAX_SIZE;
> +    size = max_size;
>    size = ALIGN_UP (size, pagesize);

Ok.

> -  /* A memory region aligned to a multiple of HEAP_MAX_SIZE is needed.
> +  /* A memory region aligned to a multiple of max_size is needed.
>       No swap space needs to be reserved for the following large
>       mapping (on Linux, this is the case for all non-writable mappings
>       anyway). */

Ok.

>    p2 = MAP_FAILED;
>    if (aligned_heap_area)
>      {
> -      p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE,
> -                          MAP_NORESERVE);
> +      p2 = (char *) MMAP (aligned_heap_area, max_size, PROT_NONE, mmap_flags);
>        aligned_heap_area = NULL;
> -      if (p2 != MAP_FAILED && ((unsigned long) p2 & (HEAP_MAX_SIZE - 1)))
> +      if (p2 != MAP_FAILED && ((unsigned long) p2 & (max_size - 1)))
>          {
> -          __munmap (p2, HEAP_MAX_SIZE);
> +          __munmap (p2, max_size);
>            p2 = MAP_FAILED;
>          }

Ok.

>    if (p2 == MAP_FAILED)
>      {
> -      p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, MAP_NORESERVE);
> +      p1 = (char *) MMAP (0, max_size << 1, PROT_NONE, mmap_flags);
>        if (p1 != MAP_FAILED)
>          {
> -          p2 = (char *) (((unsigned long) p1 + (HEAP_MAX_SIZE - 1))
> -                         & ~(HEAP_MAX_SIZE - 1));
> +          p2 = (char *) (((unsigned long) p1 + (max_size - 1))
> +                         & ~(max_size - 1));
>            ul = p2 - p1;
>            if (ul)
>              __munmap (p1, ul);
>            else
> -            aligned_heap_area = p2 + HEAP_MAX_SIZE;
> -          __munmap (p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul);
> +            aligned_heap_area = p2 + max_size;
> +          __munmap (p2 + max_size, max_size - ul);
>          }

Ok.

>        else
>          {
> -          /* Try to take the chance that an allocation of only HEAP_MAX_SIZE
> +          /* Try to take the chance that an allocation of only max_size
>               is already aligned. */
> -          p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE);
> +          p2 = (char *) MMAP (0, max_size, PROT_NONE, mmap_flags);
>            if (p2 == MAP_FAILED)
>              return 0;
>  
> -          if ((unsigned long) p2 & (HEAP_MAX_SIZE - 1))
> +          if ((unsigned long) p2 & (max_size - 1))
>              {
> -              __munmap (p2, HEAP_MAX_SIZE);
> +              __munmap (p2, max_size);
>                return 0;
>              }
>          }
>      }
>    if (__mprotect (p2, size, mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0)
>      {
> -      __munmap (p2, HEAP_MAX_SIZE);
> +      __munmap (p2, max_size);
>        return 0;
>      }

Ok.

> @@ -516,22 +549,42 @@ new_heap (size_t size, size_t top_pad)
> +  h->pagesize = pagesize;

Ok.

> +static heap_info *
> +new_heap (size_t size, size_t top_pad)
> +{
> +#if HAVE_TUNABLES
> +  if (__glibc_unlikely (mp_.hp_pagesize != 0))
> +    {
> +      /* MAP_NORESERVE is not used for huge pages because some kernel may
> +	 not reserve the mmap region and a subsequent access may trigger
> +	 a SIGBUS if there is no free pages in the pool.  */
> +      heap_info *h = alloc_new_heap (size, top_pad, mp_.hp_pagesize,
> +				     mp_.hp_flags);
> +      if (h != NULL)
> +	return h;
> +    }
> +#endif
> +  return alloc_new_heap (size, top_pad, GLRO (dl_pagesize), MAP_NORESERVE);
> +}
> +

Ok.

>  /* Grow a heap.  size is automatically rounded up to a
>     multiple of the page size. */
>  
>  static int
>  grow_heap (heap_info *h, long diff)
>  {
> -  size_t pagesize = GLRO (dl_pagesize);
> +  size_t pagesize = h->pagesize;
> +  size_t max_size = heap_max_size ();
>    long new_size;
>  
>    diff = ALIGN_UP (diff, pagesize);
>    new_size = (long) h->size + diff;
> -  if ((unsigned long) new_size > (unsigned long) HEAP_MAX_SIZE)
> +  if ((unsigned long) new_size > (unsigned long) max_size)
>      return -1;

Ok.

> @@ -581,21 +634,14 @@ shrink_heap (heap_info *h, long diff)
>  
>  /* Delete a heap. */
>  
> -#define delete_heap(heap) \
> -  do {									      \
> -      if ((char *) (heap) + HEAP_MAX_SIZE == aligned_heap_area)		      \
> -        aligned_heap_area = NULL;					      \
> -      __munmap ((char *) (heap), HEAP_MAX_SIZE);			      \
> -    } while (0)
> -
>  static int
>  heap_trim (heap_info *heap, size_t pad)
>  {
>    mstate ar_ptr = heap->ar_ptr;
> -  unsigned long pagesz = GLRO (dl_pagesize);
>    mchunkptr top_chunk = top (ar_ptr), p;
>    heap_info *prev_heap;
>    long new_size, top_size, top_area, extra, prev_size, misalign;
> +  size_t max_size = heap_max_size ();
>  
>    /* Can this heap go away completely? */
>    while (top_chunk == chunk_at_offset (heap, sizeof (*heap)))
> @@ -612,19 +658,23 @@ heap_trim (heap_info *heap, size_t pad)
>        assert (new_size > 0 && new_size < (long) (2 * MINSIZE));
>        if (!prev_inuse (p))
>          new_size += prev_size (p);
> -      assert (new_size > 0 && new_size < HEAP_MAX_SIZE);
> -      if (new_size + (HEAP_MAX_SIZE - prev_heap->size) < pad + MINSIZE + pagesz)
> +      assert (new_size > 0 && new_size < max_size);
> +      if (new_size + (max_size - prev_heap->size) < pad + MINSIZE
> +						    + heap->pagesize)
>          break;

Ok.

>        ar_ptr->system_mem -= heap->size;
>        LIBC_PROBE (memory_heap_free, 2, heap, heap->size);
> -      delete_heap (heap);
> +      if ((char *) heap + max_size == aligned_heap_area)
> +	aligned_heap_area = NULL;
> +      __munmap (heap, max_size);

Ok.

> -      assert (((unsigned long) ((char *) p + new_size) & (pagesz - 1)) == 0);
> +      assert (((unsigned long) ((char *) p + new_size) & (heap->pagesize - 1))
> +	      == 0);

Ok.

> @@ -644,7 +694,7 @@ heap_trim (heap_info *heap, size_t pad)
>      return 0;
>  
>    /* Release in pagesize units and round down to the nearest page.  */
> -  extra = ALIGN_DOWN(top_area - pad, pagesz);
> +  extra = ALIGN_DOWN(top_area - pad, heap->pagesize);
>    if (extra == 0)
>      return 0;

Ok.

> diff --git a/malloc/malloc.c b/malloc/malloc.c
> index 3e2f427d94..37fbd7e51d 100644
> --- a/malloc/malloc.c
> +++ b/malloc/malloc.c
> @@ -5301,7 +5301,7 @@ static __always_inline int
>  do_set_mmap_threshold (size_t value)
>  {
>    /* Forbid setting the threshold too high.  */
> -  if (value <= HEAP_MAX_SIZE / 2)
> +  if (value <= heap_max_size () / 2)

Ok.
Adhemerval Zanella Dec. 15, 2021, 3:06 p.m. UTC | #2
On 15/12/2021 01:51, DJ Delorie wrote:
> 
> Have you run any benchmarks against this series?  We're replacing
> compile-time constants with variables, which may have a tiny affect on
> performance.

With our own benchtests, the bench-malloc-simple shows a lot of noise,
with some values better other worse (which in the end equalizes, so
I think it is pretty much the same).

The bench-malloc-thread shows:

                                            master  patched  diff
   1 time_per_iteration                   41.5724  43.1116  3.70
   8 time_per_iteration                   48.5594  48.0252 -1.10
  16 time_per_iteration                   62.5156  63.1405  1.00
  24 time_per_iteration                   84.7967  85.5267  0.86

So I think it might also be just noise or some OS jitter.

> 
> LGTM
> Reviewed-by: DJ Delorie <dj@redhat.com>
> 
> Adhemerval Zanella via Libc-alpha <libc-alpha@sourceware.org> writes:
>>  	tst-malloc-usable \
>>  	tst-malloc-usable-tunables \
>>  	tst-mallocstate
>> +# The tst-free-errno relies on the used malloc page size to mmap an
>> +# overlapping region.
>> +tests-exclude-hugetlb2 = \
>> +	$(tests-exclude-hugetlb1) \
>> +	tst-free-errno
> 
> Ok.
> 
>>  tests-malloc-hugetlb2 = \
>> -	$(filter-out $(tests-exclude-hugetlb1), $(tests))
>> +	$(filter-out $(tests-exclude-hugetlb2), $(tests))
> 
> Ok.
> 
>> diff --git a/malloc/arena.c b/malloc/arena.c
>> +/* When huge pages are used to create new arenas, the maximum and minumum
>> +   size are based on the runtime defined huge page size.  */
>> +
>> +static inline size_t
>> +heap_min_size (void)
>> +{
>> +#if HAVE_TUNABLES
>> +  return mp_.hp_pagesize == 0 ? HEAP_MIN_SIZE : mp_.hp_pagesize;
>> +#else
>> +  return HEAP_MIN_SIZE;
>> +#endif
>> +}
>> +
>> +static inline size_t
>> +heap_max_size (void)
>> +{
>> +#if HAVE_TUNABLES
>> +  return mp_.hp_pagesize == 0 ? HEAP_MAX_SIZE : mp_.hp_pagesize * 4;
>> +#else
>> +  return HEAP_MAX_SIZE;
>> +#endif
>> +}
> 
> Ok.
> 
>> @@ -56,10 +79,11 @@ typedef struct _heap_info
>>    size_t size;   /* Current size in bytes. */
>>    size_t mprotect_size; /* Size in bytes that has been mprotected
>>                             PROT_READ|PROT_WRITE.  */
>> +  size_t pagesize; /* Page size used when allocating the arena.  */
>>    /* Make sure the following data is properly aligned, particularly
>>       that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of
>>       MALLOC_ALIGNMENT. */
>> -  char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK];
>> +  char pad[-3 * SIZE_SZ & MALLOC_ALIGN_MASK];
>>  } heap_info;
> 
> Ok.
> 
>> @@ -125,10 +149,18 @@ static bool __malloc_initialized = false;
>>  
>>  /* find the heap and corresponding arena for a given ptr */
>>  
>> -#define heap_for_ptr(ptr) \
>> -  ((heap_info *) ((unsigned long) (ptr) & ~(HEAP_MAX_SIZE - 1)))
>> -#define arena_for_chunk(ptr) \
>> -  (chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr)
>> +static inline heap_info *
>> +heap_for_ptr (void *ptr)
>> +{
>> +  size_t max_size = heap_max_size ();
>> +  return PTR_ALIGN_DOWN (ptr, max_size);
>> +}
>> +
>> +static inline struct malloc_state *
>> +arena_for_chunk (mchunkptr ptr)
>> +{
>> +  return chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr;
>> +}
> 
> Ok.
> 
> 
>> @@ -443,71 +475,72 @@ static char *aligned_heap_area;
>>     of the page size. */
>>  
>>  static heap_info *
>> -new_heap (size_t size, size_t top_pad)
>> +alloc_new_heap  (size_t size, size_t top_pad, size_t pagesize,
>> +		 int mmap_flags)
>>  {
>> -  size_t pagesize = GLRO (dl_pagesize);
> 
> Moved to args, ok.
> 
>>    char *p1, *p2;
>>    unsigned long ul;
>>    heap_info *h;
>> +  size_t min_size = heap_min_size ();
>> +  size_t max_size = heap_max_size ();
> 
> Ok.
> 
>> -  if (size + top_pad < HEAP_MIN_SIZE)
>> -    size = HEAP_MIN_SIZE;
>> -  else if (size + top_pad <= HEAP_MAX_SIZE)
>> +  if (size + top_pad < min_size)
>> +    size = min_size;
>> +  else if (size + top_pad <= max_size)
>>      size += top_pad;
>> -  else if (size > HEAP_MAX_SIZE)
>> +  else if (size > max_size)
>>      return 0;
>>    else
>> -    size = HEAP_MAX_SIZE;
>> +    size = max_size;
>>    size = ALIGN_UP (size, pagesize);
> 
> Ok.
> 
>> -  /* A memory region aligned to a multiple of HEAP_MAX_SIZE is needed.
>> +  /* A memory region aligned to a multiple of max_size is needed.
>>       No swap space needs to be reserved for the following large
>>       mapping (on Linux, this is the case for all non-writable mappings
>>       anyway). */
> 
> Ok.
> 
>>    p2 = MAP_FAILED;
>>    if (aligned_heap_area)
>>      {
>> -      p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE,
>> -                          MAP_NORESERVE);
>> +      p2 = (char *) MMAP (aligned_heap_area, max_size, PROT_NONE, mmap_flags);
>>        aligned_heap_area = NULL;
>> -      if (p2 != MAP_FAILED && ((unsigned long) p2 & (HEAP_MAX_SIZE - 1)))
>> +      if (p2 != MAP_FAILED && ((unsigned long) p2 & (max_size - 1)))
>>          {
>> -          __munmap (p2, HEAP_MAX_SIZE);
>> +          __munmap (p2, max_size);
>>            p2 = MAP_FAILED;
>>          }
> 
> Ok.
> 
>>    if (p2 == MAP_FAILED)
>>      {
>> -      p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, MAP_NORESERVE);
>> +      p1 = (char *) MMAP (0, max_size << 1, PROT_NONE, mmap_flags);
>>        if (p1 != MAP_FAILED)
>>          {
>> -          p2 = (char *) (((unsigned long) p1 + (HEAP_MAX_SIZE - 1))
>> -                         & ~(HEAP_MAX_SIZE - 1));
>> +          p2 = (char *) (((unsigned long) p1 + (max_size - 1))
>> +                         & ~(max_size - 1));
>>            ul = p2 - p1;
>>            if (ul)
>>              __munmap (p1, ul);
>>            else
>> -            aligned_heap_area = p2 + HEAP_MAX_SIZE;
>> -          __munmap (p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul);
>> +            aligned_heap_area = p2 + max_size;
>> +          __munmap (p2 + max_size, max_size - ul);
>>          }
> 
> Ok.
> 
>>        else
>>          {
>> -          /* Try to take the chance that an allocation of only HEAP_MAX_SIZE
>> +          /* Try to take the chance that an allocation of only max_size
>>               is already aligned. */
>> -          p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE);
>> +          p2 = (char *) MMAP (0, max_size, PROT_NONE, mmap_flags);
>>            if (p2 == MAP_FAILED)
>>              return 0;
>>  
>> -          if ((unsigned long) p2 & (HEAP_MAX_SIZE - 1))
>> +          if ((unsigned long) p2 & (max_size - 1))
>>              {
>> -              __munmap (p2, HEAP_MAX_SIZE);
>> +              __munmap (p2, max_size);
>>                return 0;
>>              }
>>          }
>>      }
>>    if (__mprotect (p2, size, mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0)
>>      {
>> -      __munmap (p2, HEAP_MAX_SIZE);
>> +      __munmap (p2, max_size);
>>        return 0;
>>      }
> 
> Ok.
> 
>> @@ -516,22 +549,42 @@ new_heap (size_t size, size_t top_pad)
>> +  h->pagesize = pagesize;
> 
> Ok.
> 
>> +static heap_info *
>> +new_heap (size_t size, size_t top_pad)
>> +{
>> +#if HAVE_TUNABLES
>> +  if (__glibc_unlikely (mp_.hp_pagesize != 0))
>> +    {
>> +      /* MAP_NORESERVE is not used for huge pages because some kernel may
>> +	 not reserve the mmap region and a subsequent access may trigger
>> +	 a SIGBUS if there is no free pages in the pool.  */
>> +      heap_info *h = alloc_new_heap (size, top_pad, mp_.hp_pagesize,
>> +				     mp_.hp_flags);
>> +      if (h != NULL)
>> +	return h;
>> +    }
>> +#endif
>> +  return alloc_new_heap (size, top_pad, GLRO (dl_pagesize), MAP_NORESERVE);
>> +}
>> +
> 
> Ok.
> 
>>  /* Grow a heap.  size is automatically rounded up to a
>>     multiple of the page size. */
>>  
>>  static int
>>  grow_heap (heap_info *h, long diff)
>>  {
>> -  size_t pagesize = GLRO (dl_pagesize);
>> +  size_t pagesize = h->pagesize;
>> +  size_t max_size = heap_max_size ();
>>    long new_size;
>>  
>>    diff = ALIGN_UP (diff, pagesize);
>>    new_size = (long) h->size + diff;
>> -  if ((unsigned long) new_size > (unsigned long) HEAP_MAX_SIZE)
>> +  if ((unsigned long) new_size > (unsigned long) max_size)
>>      return -1;
> 
> Ok.
> 
>> @@ -581,21 +634,14 @@ shrink_heap (heap_info *h, long diff)
>>  
>>  /* Delete a heap. */
>>  
>> -#define delete_heap(heap) \
>> -  do {									      \
>> -      if ((char *) (heap) + HEAP_MAX_SIZE == aligned_heap_area)		      \
>> -        aligned_heap_area = NULL;					      \
>> -      __munmap ((char *) (heap), HEAP_MAX_SIZE);			      \
>> -    } while (0)
>> -
>>  static int
>>  heap_trim (heap_info *heap, size_t pad)
>>  {
>>    mstate ar_ptr = heap->ar_ptr;
>> -  unsigned long pagesz = GLRO (dl_pagesize);
>>    mchunkptr top_chunk = top (ar_ptr), p;
>>    heap_info *prev_heap;
>>    long new_size, top_size, top_area, extra, prev_size, misalign;
>> +  size_t max_size = heap_max_size ();
>>  
>>    /* Can this heap go away completely? */
>>    while (top_chunk == chunk_at_offset (heap, sizeof (*heap)))
>> @@ -612,19 +658,23 @@ heap_trim (heap_info *heap, size_t pad)
>>        assert (new_size > 0 && new_size < (long) (2 * MINSIZE));
>>        if (!prev_inuse (p))
>>          new_size += prev_size (p);
>> -      assert (new_size > 0 && new_size < HEAP_MAX_SIZE);
>> -      if (new_size + (HEAP_MAX_SIZE - prev_heap->size) < pad + MINSIZE + pagesz)
>> +      assert (new_size > 0 && new_size < max_size);
>> +      if (new_size + (max_size - prev_heap->size) < pad + MINSIZE
>> +						    + heap->pagesize)
>>          break;
> 
> Ok.
> 
>>        ar_ptr->system_mem -= heap->size;
>>        LIBC_PROBE (memory_heap_free, 2, heap, heap->size);
>> -      delete_heap (heap);
>> +      if ((char *) heap + max_size == aligned_heap_area)
>> +	aligned_heap_area = NULL;
>> +      __munmap (heap, max_size);
> 
> Ok.
> 
>> -      assert (((unsigned long) ((char *) p + new_size) & (pagesz - 1)) == 0);
>> +      assert (((unsigned long) ((char *) p + new_size) & (heap->pagesize - 1))
>> +	      == 0);
> 
> Ok.
> 
>> @@ -644,7 +694,7 @@ heap_trim (heap_info *heap, size_t pad)
>>      return 0;
>>  
>>    /* Release in pagesize units and round down to the nearest page.  */
>> -  extra = ALIGN_DOWN(top_area - pad, pagesz);
>> +  extra = ALIGN_DOWN(top_area - pad, heap->pagesize);
>>    if (extra == 0)
>>      return 0;
> 
> Ok.
> 
>> diff --git a/malloc/malloc.c b/malloc/malloc.c
>> index 3e2f427d94..37fbd7e51d 100644
>> --- a/malloc/malloc.c
>> +++ b/malloc/malloc.c
>> @@ -5301,7 +5301,7 @@ static __always_inline int
>>  do_set_mmap_threshold (size_t value)
>>  {
>>    /* Forbid setting the threshold too high.  */
>> -  if (value <= HEAP_MAX_SIZE / 2)
>> +  if (value <= heap_max_size () / 2)
> 
> Ok.
>
diff mbox series

Patch

diff --git a/malloc/Makefile b/malloc/Makefile
index 83de7f2a35..abd0f811d4 100644
--- a/malloc/Makefile
+++ b/malloc/Makefile
@@ -91,10 +91,15 @@  tests-exclude-hugetlb1 = \
 	tst-malloc-usable \
 	tst-malloc-usable-tunables \
 	tst-mallocstate
+# The tst-free-errno relies on the used malloc page size to mmap an
+# overlapping region.
+tests-exclude-hugetlb2 = \
+	$(tests-exclude-hugetlb1) \
+	tst-free-errno
 tests-malloc-hugetlb1 = \
 	$(filter-out $(tests-exclude-hugetlb1), $(tests))
 tests-malloc-hugetlb2 = \
-	$(filter-out $(tests-exclude-hugetlb1), $(tests))
+	$(filter-out $(tests-exclude-hugetlb2), $(tests))
 
 # -lmcheck needs __malloc_initialize_hook, which was deprecated in 2.24.
 ifeq ($(have-GLIBC_2.23)$(build-shared),yesyes)
diff --git a/malloc/arena.c b/malloc/arena.c
index cd00c7bef4..c6d811ff3b 100644
--- a/malloc/arena.c
+++ b/malloc/arena.c
@@ -41,6 +41,29 @@ 
    mmap threshold, so that requests with a size just below that
    threshold can be fulfilled without creating too many heaps.  */
 
+/* When huge pages are used to create new arenas, the maximum and minumum
+   size are based on the runtime defined huge page size.  */
+
+static inline size_t
+heap_min_size (void)
+{
+#if HAVE_TUNABLES
+  return mp_.hp_pagesize == 0 ? HEAP_MIN_SIZE : mp_.hp_pagesize;
+#else
+  return HEAP_MIN_SIZE;
+#endif
+}
+
+static inline size_t
+heap_max_size (void)
+{
+#if HAVE_TUNABLES
+  return mp_.hp_pagesize == 0 ? HEAP_MAX_SIZE : mp_.hp_pagesize * 4;
+#else
+  return HEAP_MAX_SIZE;
+#endif
+}
+
 /***************************************************************************/
 
 #define top(ar_ptr) ((ar_ptr)->top)
@@ -56,10 +79,11 @@  typedef struct _heap_info
   size_t size;   /* Current size in bytes. */
   size_t mprotect_size; /* Size in bytes that has been mprotected
                            PROT_READ|PROT_WRITE.  */
+  size_t pagesize; /* Page size used when allocating the arena.  */
   /* Make sure the following data is properly aligned, particularly
      that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of
      MALLOC_ALIGNMENT. */
-  char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK];
+  char pad[-3 * SIZE_SZ & MALLOC_ALIGN_MASK];
 } heap_info;
 
 /* Get a compile-time error if the heap_info padding is not correct
@@ -125,10 +149,18 @@  static bool __malloc_initialized = false;
 
 /* find the heap and corresponding arena for a given ptr */
 
-#define heap_for_ptr(ptr) \
-  ((heap_info *) ((unsigned long) (ptr) & ~(HEAP_MAX_SIZE - 1)))
-#define arena_for_chunk(ptr) \
-  (chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr)
+static inline heap_info *
+heap_for_ptr (void *ptr)
+{
+  size_t max_size = heap_max_size ();
+  return PTR_ALIGN_DOWN (ptr, max_size);
+}
+
+static inline struct malloc_state *
+arena_for_chunk (mchunkptr ptr)
+{
+  return chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr;
+}
 
 
 /**************************************************************************/
@@ -443,71 +475,72 @@  static char *aligned_heap_area;
    of the page size. */
 
 static heap_info *
-new_heap (size_t size, size_t top_pad)
+alloc_new_heap  (size_t size, size_t top_pad, size_t pagesize,
+		 int mmap_flags)
 {
-  size_t pagesize = GLRO (dl_pagesize);
   char *p1, *p2;
   unsigned long ul;
   heap_info *h;
+  size_t min_size = heap_min_size ();
+  size_t max_size = heap_max_size ();
 
-  if (size + top_pad < HEAP_MIN_SIZE)
-    size = HEAP_MIN_SIZE;
-  else if (size + top_pad <= HEAP_MAX_SIZE)
+  if (size + top_pad < min_size)
+    size = min_size;
+  else if (size + top_pad <= max_size)
     size += top_pad;
-  else if (size > HEAP_MAX_SIZE)
+  else if (size > max_size)
     return 0;
   else
-    size = HEAP_MAX_SIZE;
+    size = max_size;
   size = ALIGN_UP (size, pagesize);
 
-  /* A memory region aligned to a multiple of HEAP_MAX_SIZE is needed.
+  /* A memory region aligned to a multiple of max_size is needed.
      No swap space needs to be reserved for the following large
      mapping (on Linux, this is the case for all non-writable mappings
      anyway). */
   p2 = MAP_FAILED;
   if (aligned_heap_area)
     {
-      p2 = (char *) MMAP (aligned_heap_area, HEAP_MAX_SIZE, PROT_NONE,
-                          MAP_NORESERVE);
+      p2 = (char *) MMAP (aligned_heap_area, max_size, PROT_NONE, mmap_flags);
       aligned_heap_area = NULL;
-      if (p2 != MAP_FAILED && ((unsigned long) p2 & (HEAP_MAX_SIZE - 1)))
+      if (p2 != MAP_FAILED && ((unsigned long) p2 & (max_size - 1)))
         {
-          __munmap (p2, HEAP_MAX_SIZE);
+          __munmap (p2, max_size);
           p2 = MAP_FAILED;
         }
     }
   if (p2 == MAP_FAILED)
     {
-      p1 = (char *) MMAP (0, HEAP_MAX_SIZE << 1, PROT_NONE, MAP_NORESERVE);
+      p1 = (char *) MMAP (0, max_size << 1, PROT_NONE, mmap_flags);
       if (p1 != MAP_FAILED)
         {
-          p2 = (char *) (((unsigned long) p1 + (HEAP_MAX_SIZE - 1))
-                         & ~(HEAP_MAX_SIZE - 1));
+          p2 = (char *) (((unsigned long) p1 + (max_size - 1))
+                         & ~(max_size - 1));
           ul = p2 - p1;
           if (ul)
             __munmap (p1, ul);
           else
-            aligned_heap_area = p2 + HEAP_MAX_SIZE;
-          __munmap (p2 + HEAP_MAX_SIZE, HEAP_MAX_SIZE - ul);
+            aligned_heap_area = p2 + max_size;
+          __munmap (p2 + max_size, max_size - ul);
         }
       else
         {
-          /* Try to take the chance that an allocation of only HEAP_MAX_SIZE
+          /* Try to take the chance that an allocation of only max_size
              is already aligned. */
-          p2 = (char *) MMAP (0, HEAP_MAX_SIZE, PROT_NONE, MAP_NORESERVE);
+          p2 = (char *) MMAP (0, max_size, PROT_NONE, mmap_flags);
           if (p2 == MAP_FAILED)
             return 0;
 
-          if ((unsigned long) p2 & (HEAP_MAX_SIZE - 1))
+          if ((unsigned long) p2 & (max_size - 1))
             {
-              __munmap (p2, HEAP_MAX_SIZE);
+              __munmap (p2, max_size);
               return 0;
             }
         }
     }
   if (__mprotect (p2, size, mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0)
     {
-      __munmap (p2, HEAP_MAX_SIZE);
+      __munmap (p2, max_size);
       return 0;
     }
 
@@ -516,22 +549,42 @@  new_heap (size_t size, size_t top_pad)
   h = (heap_info *) p2;
   h->size = size;
   h->mprotect_size = size;
+  h->pagesize = pagesize;
   LIBC_PROBE (memory_heap_new, 2, h, h->size);
   return h;
 }
 
+static heap_info *
+new_heap (size_t size, size_t top_pad)
+{
+#if HAVE_TUNABLES
+  if (__glibc_unlikely (mp_.hp_pagesize != 0))
+    {
+      /* MAP_NORESERVE is not used for huge pages because some kernel may
+	 not reserve the mmap region and a subsequent access may trigger
+	 a SIGBUS if there is no free pages in the pool.  */
+      heap_info *h = alloc_new_heap (size, top_pad, mp_.hp_pagesize,
+				     mp_.hp_flags);
+      if (h != NULL)
+	return h;
+    }
+#endif
+  return alloc_new_heap (size, top_pad, GLRO (dl_pagesize), MAP_NORESERVE);
+}
+
 /* Grow a heap.  size is automatically rounded up to a
    multiple of the page size. */
 
 static int
 grow_heap (heap_info *h, long diff)
 {
-  size_t pagesize = GLRO (dl_pagesize);
+  size_t pagesize = h->pagesize;
+  size_t max_size = heap_max_size ();
   long new_size;
 
   diff = ALIGN_UP (diff, pagesize);
   new_size = (long) h->size + diff;
-  if ((unsigned long) new_size > (unsigned long) HEAP_MAX_SIZE)
+  if ((unsigned long) new_size > (unsigned long) max_size)
     return -1;
 
   if ((unsigned long) new_size > h->mprotect_size)
@@ -581,21 +634,14 @@  shrink_heap (heap_info *h, long diff)
 
 /* Delete a heap. */
 
-#define delete_heap(heap) \
-  do {									      \
-      if ((char *) (heap) + HEAP_MAX_SIZE == aligned_heap_area)		      \
-        aligned_heap_area = NULL;					      \
-      __munmap ((char *) (heap), HEAP_MAX_SIZE);			      \
-    } while (0)
-
 static int
 heap_trim (heap_info *heap, size_t pad)
 {
   mstate ar_ptr = heap->ar_ptr;
-  unsigned long pagesz = GLRO (dl_pagesize);
   mchunkptr top_chunk = top (ar_ptr), p;
   heap_info *prev_heap;
   long new_size, top_size, top_area, extra, prev_size, misalign;
+  size_t max_size = heap_max_size ();
 
   /* Can this heap go away completely? */
   while (top_chunk == chunk_at_offset (heap, sizeof (*heap)))
@@ -612,19 +658,23 @@  heap_trim (heap_info *heap, size_t pad)
       assert (new_size > 0 && new_size < (long) (2 * MINSIZE));
       if (!prev_inuse (p))
         new_size += prev_size (p);
-      assert (new_size > 0 && new_size < HEAP_MAX_SIZE);
-      if (new_size + (HEAP_MAX_SIZE - prev_heap->size) < pad + MINSIZE + pagesz)
+      assert (new_size > 0 && new_size < max_size);
+      if (new_size + (max_size - prev_heap->size) < pad + MINSIZE
+						    + heap->pagesize)
         break;
       ar_ptr->system_mem -= heap->size;
       LIBC_PROBE (memory_heap_free, 2, heap, heap->size);
-      delete_heap (heap);
+      if ((char *) heap + max_size == aligned_heap_area)
+	aligned_heap_area = NULL;
+      __munmap (heap, max_size);
       heap = prev_heap;
       if (!prev_inuse (p)) /* consolidate backward */
         {
           p = prev_chunk (p);
           unlink_chunk (ar_ptr, p);
         }
-      assert (((unsigned long) ((char *) p + new_size) & (pagesz - 1)) == 0);
+      assert (((unsigned long) ((char *) p + new_size) & (heap->pagesize - 1))
+	      == 0);
       assert (((char *) p + new_size) == ((char *) heap + heap->size));
       top (ar_ptr) = top_chunk = p;
       set_head (top_chunk, new_size | PREV_INUSE);
@@ -644,7 +694,7 @@  heap_trim (heap_info *heap, size_t pad)
     return 0;
 
   /* Release in pagesize units and round down to the nearest page.  */
-  extra = ALIGN_DOWN(top_area - pad, pagesz);
+  extra = ALIGN_DOWN(top_area - pad, heap->pagesize);
   if (extra == 0)
     return 0;
 
diff --git a/malloc/malloc.c b/malloc/malloc.c
index 3e2f427d94..37fbd7e51d 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -5301,7 +5301,7 @@  static __always_inline int
 do_set_mmap_threshold (size_t value)
 {
   /* Forbid setting the threshold too high.  */
-  if (value <= HEAP_MAX_SIZE / 2)
+  if (value <= heap_max_size () / 2)
     {
       LIBC_PROBE (memory_mallopt_mmap_threshold, 3, value, mp_.mmap_threshold,
 		  mp_.no_dyn_threshold);