diff mbox series

[v5,2/7] malloc: Add THP/madvise support for sbrk

Message ID 20211214185806.4109231-3-adhemerval.zanella@linaro.org
State Committed
Commit 7478c9959ae409f7b3d63146943575d6ee745352
Headers
Series malloc: Improve Huge Page support |

Checks

Context Check Description
dj/TryBot-apply_patch success Patch applied to master at the time it was sent

Commit Message

Adhemerval Zanella Netto Dec. 14, 2021, 6:58 p.m. UTC 
  To increase effectiveness with Transparent Huge Page with madvise, the
large page size is use instead page size for sbrk increment for the
main arena.

Checked on x86_64-linux-gnu.
---
 include/libc-pointer-arith.h |  8 ++++++++
 malloc/malloc.c              | 34 +++++++++++++++++++++++++++++-----
 2 files changed, 37 insertions(+), 5 deletions(-)

Comments

DJ Delorie Dec. 15, 2021, 3:28 a.m. UTC | #1 
LGTM
Reviewed-by: DJ Delorie <dj@redhat.com>

Adhemerval Zanella via Libc-alpha <libc-alpha@sourceware.org> writes:
> diff --git a/include/libc-pointer-arith.h b/include/libc-pointer-arith.h
>  #define PTR_ALIGN_UP(base, size) \
>    ((__typeof__ (base)) ALIGN_UP ((uintptr_t) (base), (size)))
>  
> +/* Check if BASE is aligned on SIZE  */
> +#define PTR_IS_ALIGNED(base, size) \
> +  ((((uintptr_t) (base)) & (size - 1)) == 0)
> +
> +/* Returns the ptrdiff_t diference between P1 and P2.  */
> +#define PTR_DIFF(p1, p2) \
> +  ((ptrdiff_t)((uintptr_t)(p1) - (uintptr_t)(p2)))
> +

Ok.

> diff --git a/malloc/malloc.c b/malloc/malloc.c
> index b8103aaf10..8a66012503 100644
> --- a/malloc/malloc.c
> +++ b/malloc/malloc.c
> @@ -2023,6 +2023,16 @@ madvise_thp (void *p, INTERNAL_SIZE_T size)
>       not active.  */
>    if (mp_.thp_pagesize == 0 || size < mp_.thp_pagesize)
>      return;
> +
> +  /* Linux requires the input address to be page-aligned, and unaligned
> +     inputs happens only for initial data segment.  */
> +  if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
> +    {
> +      void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
> +      size += PTR_DIFF (p, q);
> +      p = q;
> +    }
> +
>    __madvise (p, size, MADV_HUGEPAGE);

Ok.

> @@ -2609,14 +2619,25 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av)
> -         Round to a multiple of page size.
> +         Round to a multiple of page size or huge page size.

Ok.

> -      size = ALIGN_UP (size, pagesize);
> +#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
> +      /* Defined in brk.c.  */
> +      extern void *__curbrk;
> +      if (__glibc_unlikely (mp_.thp_pagesize != 0))
> +	{
> +	  uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
> +				    mp_.thp_pagesize);
> +	  size = top - (uintptr_t) __curbrk;
> +	}
> +      else
> +#endif
> +	size = ALIGN_UP (size, GLRO(dl_pagesize));

Ok.

>        /*
>           Don't try to call MORECORE if argument is so big as to appear
> @@ -2899,10 +2920,8 @@ systrim (size_t pad, mstate av)
>    long released;         /* Amount actually released */
>    char *current_brk;     /* address returned by pre-check sbrk call */
>    char *new_brk;         /* address returned by post-check sbrk call */
> -  size_t pagesize;
>    long top_area;
>  
> -  pagesize = GLRO (dl_pagesize);
>    top_size = chunksize (av->top);
>  
>    top_area = top_size - MINSIZE - 1;
> @@ -2910,7 +2929,12 @@ systrim (size_t pad, mstate av)
>      return 0;
>  
>    /* Release in pagesize units and round down to the nearest page.  */
> -  extra = ALIGN_DOWN(top_area - pad, pagesize);
> +#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
> +  if (__glibc_unlikely (mp_.thp_pagesize != 0))
> +    extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
> +  else
> +#endif
> +    extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
>  
>    if (extra == 0)
>      return 0;

Ok.
diff mbox series

Patch

diff --git a/include/libc-pointer-arith.h b/include/libc-pointer-arith.h
index 04ba537617..55dccc10ac 100644
--- a/include/libc-pointer-arith.h
+++ b/include/libc-pointer-arith.h
@@ -60,4 +60,12 @@ 
 #define PTR_ALIGN_UP(base, size) \
   ((__typeof__ (base)) ALIGN_UP ((uintptr_t) (base), (size)))
 
+/* Check if BASE is aligned on SIZE  */
+#define PTR_IS_ALIGNED(base, size) \
+  ((((uintptr_t) (base)) & (size - 1)) == 0)
+
+/* Returns the ptrdiff_t diference between P1 and P2.  */
+#define PTR_DIFF(p1, p2) \
+  ((ptrdiff_t)((uintptr_t)(p1) - (uintptr_t)(p2)))
+
 #endif
diff --git a/malloc/malloc.c b/malloc/malloc.c
index b8103aaf10..8a66012503 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -2023,6 +2023,16 @@  madvise_thp (void *p, INTERNAL_SIZE_T size)
      not active.  */
   if (mp_.thp_pagesize == 0 || size < mp_.thp_pagesize)
     return;
+
+  /* Linux requires the input address to be page-aligned, and unaligned
+     inputs happens only for initial data segment.  */
+  if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
+    {
+      void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
+      size += PTR_DIFF (p, q);
+      p = q;
+    }
+
   __madvise (p, size, MADV_HUGEPAGE);
 #endif
 }
@@ -2609,14 +2619,25 @@  sysmalloc (INTERNAL_SIZE_T nb, mstate av)
         size -= old_size;
 
       /*
-         Round to a multiple of page size.
+         Round to a multiple of page size or huge page size.
          If MORECORE is not contiguous, this ensures that we only call it
          with whole-page arguments.  And if MORECORE is contiguous and
          this is not first time through, this preserves page-alignment of
          previous calls. Otherwise, we correct to page-align below.
        */
 
-      size = ALIGN_UP (size, pagesize);
+#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
+      /* Defined in brk.c.  */
+      extern void *__curbrk;
+      if (__glibc_unlikely (mp_.thp_pagesize != 0))
+	{
+	  uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
+				    mp_.thp_pagesize);
+	  size = top - (uintptr_t) __curbrk;
+	}
+      else
+#endif
+	size = ALIGN_UP (size, GLRO(dl_pagesize));
 
       /*
          Don't try to call MORECORE if argument is so big as to appear
@@ -2899,10 +2920,8 @@  systrim (size_t pad, mstate av)
   long released;         /* Amount actually released */
   char *current_brk;     /* address returned by pre-check sbrk call */
   char *new_brk;         /* address returned by post-check sbrk call */
-  size_t pagesize;
   long top_area;
 
-  pagesize = GLRO (dl_pagesize);
   top_size = chunksize (av->top);
 
   top_area = top_size - MINSIZE - 1;
@@ -2910,7 +2929,12 @@  systrim (size_t pad, mstate av)
     return 0;
 
   /* Release in pagesize units and round down to the nearest page.  */
-  extra = ALIGN_DOWN(top_area - pad, pagesize);
+#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
+  if (__glibc_unlikely (mp_.thp_pagesize != 0))
+    extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
+  else
+#endif
+    extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
 
   if (extra == 0)
     return 0;