diff mbox series

[v2,2/4] malloc: Add THP/madvise support for sbrk

Message ID 20210818142000.128752-3-adhemerval.zanella@linaro.org
State Superseded
Headers
Series malloc: Improve Huge Page support |

Checks

Context Check Description
dj/TryBot-apply_patch success Patch applied to master at the time it was sent

Commit Message

Adhemerval Zanella Aug. 18, 2021, 2:19 p.m. UTC 
  For the main arena, the sbrk() might the preferable syscall instead of
mmap().  And the granularity used when increasing the program segment
is the default page size.

To increase effectiveness with Transparent Huge Page with madvise, the
large page size is use instead.  This is enabled with the new tunable
'glibc.malloc.thp_pagesize'.

Checked on x86_64-linux-gnu.
---
 include/libc-pointer-arith.h | 10 ++++++++++
 malloc/malloc.c              | 35 ++++++++++++++++++++++++++++++-----
 2 files changed, 40 insertions(+), 5 deletions(-)
diff mbox series

Patch

diff --git a/include/libc-pointer-arith.h b/include/libc-pointer-arith.h
index 04ba537617..f592cbafec 100644
--- a/include/libc-pointer-arith.h
+++ b/include/libc-pointer-arith.h
@@ -37,6 +37,16 @@ 
 /* Cast an integer or a pointer VAL to integer with proper type.  */
 # define cast_to_integer(val) ((__integer_if_pointer_type (val)) (val))
 
+/* Check if SIZE is aligned on SIZE  */
+#define IS_ALIGNED(base, size) \
+  (((base) & (size - 1)) == 0)
+
+#define PTR_IS_ALIGNED(base, size) \
+  ((((uintptr_t) (base)) & (size - 1)) == 0)
+
+#define PTR_DIFF(p1, p2) \
+  ((ptrdiff_t)((uintptr_t)(p1) - (uintptr_t)(p2)))
+
 /* Cast an integer VAL to void * pointer.  */
 # define cast_to_pointer(val) ((void *) (uintptr_t) (val))
 
diff --git a/malloc/malloc.c b/malloc/malloc.c
index ad3eec41ac..1a2c798a35 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -2024,6 +2024,17 @@  sysmadvise_thp (void *p, INTERNAL_SIZE_T size)
      not active.  */
   if (mp_.thp_pagesize == 0 || size < mp_.thp_pagesize)
     return;
+
+  /* madvise() requires at least the input to be aligned to system page and
+     MADV_HUGEPAGE should handle unaligned address.  Also unaligned inputs
+     should happen only for the initial data segment.  */
+  if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
+    {
+      void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
+      size += PTR_DIFF (p, q);
+      p = q;
+    }
+
   __madvise (p, size, MADV_HUGEPAGE);
 #endif
 }
@@ -2610,14 +2621,25 @@  sysmalloc (INTERNAL_SIZE_T nb, mstate av)
         size -= old_size;
 
       /*
-         Round to a multiple of page size.
+         Round to a multiple of page size or huge page size.
          If MORECORE is not contiguous, this ensures that we only call it
          with whole-page arguments.  And if MORECORE is contiguous and
          this is not first time through, this preserves page-alignment of
          previous calls. Otherwise, we correct to page-align below.
        */
 
-      size = ALIGN_UP (size, pagesize);
+#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
+      /* Defined in brk.c.  */
+      extern void *__curbrk;
+      if (mp_.thp_pagesize != 0)
+	{
+	  uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
+				    mp_.thp_pagesize);
+	  size = top - (uintptr_t) __curbrk;
+	}
+      else
+#endif
+	size = ALIGN_UP (size, GLRO(dl_pagesize));
 
       /*
          Don't try to call MORECORE if argument is so big as to appear
@@ -2900,10 +2922,8 @@  systrim (size_t pad, mstate av)
   long released;         /* Amount actually released */
   char *current_brk;     /* address returned by pre-check sbrk call */
   char *new_brk;         /* address returned by post-check sbrk call */
-  size_t pagesize;
   long top_area;
 
-  pagesize = GLRO (dl_pagesize);
   top_size = chunksize (av->top);
 
   top_area = top_size - MINSIZE - 1;
@@ -2911,7 +2931,12 @@  systrim (size_t pad, mstate av)
     return 0;
 
   /* Release in pagesize units and round down to the nearest page.  */
-  extra = ALIGN_DOWN(top_area - pad, pagesize);
+#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
+  if (mp_.thp_pagesize != 0)
+    extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
+  else
+#endif
+    extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
 
   if (extra == 0)
     return 0;