[v5,2/7] malloc: Add THP/madvise support for sbrk
Checks
Context |
Check |
Description |
dj/TryBot-apply_patch |
success
|
Patch applied to master at the time it was sent
|
Commit Message
To increase effectiveness with Transparent Huge Page with madvise, the
large page size is use instead page size for sbrk increment for the
main arena.
Checked on x86_64-linux-gnu.
---
include/libc-pointer-arith.h | 8 ++++++++
malloc/malloc.c | 34 +++++++++++++++++++++++++++++-----
2 files changed, 37 insertions(+), 5 deletions(-)
Comments
LGTM
Reviewed-by: DJ Delorie <dj@redhat.com>
Adhemerval Zanella via Libc-alpha <libc-alpha@sourceware.org> writes:
> diff --git a/include/libc-pointer-arith.h b/include/libc-pointer-arith.h
> #define PTR_ALIGN_UP(base, size) \
> ((__typeof__ (base)) ALIGN_UP ((uintptr_t) (base), (size)))
>
> +/* Check if BASE is aligned on SIZE */
> +#define PTR_IS_ALIGNED(base, size) \
> + ((((uintptr_t) (base)) & (size - 1)) == 0)
> +
> +/* Returns the ptrdiff_t diference between P1 and P2. */
> +#define PTR_DIFF(p1, p2) \
> + ((ptrdiff_t)((uintptr_t)(p1) - (uintptr_t)(p2)))
> +
Ok.
> diff --git a/malloc/malloc.c b/malloc/malloc.c
> index b8103aaf10..8a66012503 100644
> --- a/malloc/malloc.c
> +++ b/malloc/malloc.c
> @@ -2023,6 +2023,16 @@ madvise_thp (void *p, INTERNAL_SIZE_T size)
> not active. */
> if (mp_.thp_pagesize == 0 || size < mp_.thp_pagesize)
> return;
> +
> + /* Linux requires the input address to be page-aligned, and unaligned
> + inputs happens only for initial data segment. */
> + if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
> + {
> + void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
> + size += PTR_DIFF (p, q);
> + p = q;
> + }
> +
> __madvise (p, size, MADV_HUGEPAGE);
Ok.
> @@ -2609,14 +2619,25 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av)
> - Round to a multiple of page size.
> + Round to a multiple of page size or huge page size.
Ok.
> - size = ALIGN_UP (size, pagesize);
> +#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
> + /* Defined in brk.c. */
> + extern void *__curbrk;
> + if (__glibc_unlikely (mp_.thp_pagesize != 0))
> + {
> + uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
> + mp_.thp_pagesize);
> + size = top - (uintptr_t) __curbrk;
> + }
> + else
> +#endif
> + size = ALIGN_UP (size, GLRO(dl_pagesize));
Ok.
> /*
> Don't try to call MORECORE if argument is so big as to appear
> @@ -2899,10 +2920,8 @@ systrim (size_t pad, mstate av)
> long released; /* Amount actually released */
> char *current_brk; /* address returned by pre-check sbrk call */
> char *new_brk; /* address returned by post-check sbrk call */
> - size_t pagesize;
> long top_area;
>
> - pagesize = GLRO (dl_pagesize);
> top_size = chunksize (av->top);
>
> top_area = top_size - MINSIZE - 1;
> @@ -2910,7 +2929,12 @@ systrim (size_t pad, mstate av)
> return 0;
>
> /* Release in pagesize units and round down to the nearest page. */
> - extra = ALIGN_DOWN(top_area - pad, pagesize);
> +#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
> + if (__glibc_unlikely (mp_.thp_pagesize != 0))
> + extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
> + else
> +#endif
> + extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
>
> if (extra == 0)
> return 0;
Ok.
@@ -60,4 +60,12 @@
#define PTR_ALIGN_UP(base, size) \
((__typeof__ (base)) ALIGN_UP ((uintptr_t) (base), (size)))
+/* Check if BASE is aligned on SIZE */
+#define PTR_IS_ALIGNED(base, size) \
+ ((((uintptr_t) (base)) & (size - 1)) == 0)
+
+/* Returns the ptrdiff_t diference between P1 and P2. */
+#define PTR_DIFF(p1, p2) \
+ ((ptrdiff_t)((uintptr_t)(p1) - (uintptr_t)(p2)))
+
#endif
@@ -2023,6 +2023,16 @@ madvise_thp (void *p, INTERNAL_SIZE_T size)
not active. */
if (mp_.thp_pagesize == 0 || size < mp_.thp_pagesize)
return;
+
+ /* Linux requires the input address to be page-aligned, and unaligned
+ inputs happens only for initial data segment. */
+ if (__glibc_unlikely (!PTR_IS_ALIGNED (p, GLRO (dl_pagesize))))
+ {
+ void *q = PTR_ALIGN_DOWN (p, GLRO (dl_pagesize));
+ size += PTR_DIFF (p, q);
+ p = q;
+ }
+
__madvise (p, size, MADV_HUGEPAGE);
#endif
}
@@ -2609,14 +2619,25 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av)
size -= old_size;
/*
- Round to a multiple of page size.
+ Round to a multiple of page size or huge page size.
If MORECORE is not contiguous, this ensures that we only call it
with whole-page arguments. And if MORECORE is contiguous and
this is not first time through, this preserves page-alignment of
previous calls. Otherwise, we correct to page-align below.
*/
- size = ALIGN_UP (size, pagesize);
+#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
+ /* Defined in brk.c. */
+ extern void *__curbrk;
+ if (__glibc_unlikely (mp_.thp_pagesize != 0))
+ {
+ uintptr_t top = ALIGN_UP ((uintptr_t) __curbrk + size,
+ mp_.thp_pagesize);
+ size = top - (uintptr_t) __curbrk;
+ }
+ else
+#endif
+ size = ALIGN_UP (size, GLRO(dl_pagesize));
/*
Don't try to call MORECORE if argument is so big as to appear
@@ -2899,10 +2920,8 @@ systrim (size_t pad, mstate av)
long released; /* Amount actually released */
char *current_brk; /* address returned by pre-check sbrk call */
char *new_brk; /* address returned by post-check sbrk call */
- size_t pagesize;
long top_area;
- pagesize = GLRO (dl_pagesize);
top_size = chunksize (av->top);
top_area = top_size - MINSIZE - 1;
@@ -2910,7 +2929,12 @@ systrim (size_t pad, mstate av)
return 0;
/* Release in pagesize units and round down to the nearest page. */
- extra = ALIGN_DOWN(top_area - pad, pagesize);
+#if HAVE_TUNABLES && defined (MADV_HUGEPAGE)
+ if (__glibc_unlikely (mp_.thp_pagesize != 0))
+ extra = ALIGN_DOWN (top_area - pad, mp_.thp_pagesize);
+ else
+#endif
+ extra = ALIGN_DOWN (top_area - pad, GLRO(dl_pagesize));
if (extra == 0)
return 0;