malloc: Reduce cache conflicts in mmap chunks
Checks
| Context |
Check |
Description |
| redhat-pt-bot/TryBot-apply_patch |
success
|
Patch applied to master at the time it was sent
|
| linaro-tcwg-bot/tcwg_glibc_build--master-aarch64 |
success
|
Build passed
|
| linaro-tcwg-bot/tcwg_glibc_check--master-aarch64 |
success
|
Test passed
|
| redhat-pt-bot/TryBot-32bit |
success
|
Build for i686
|
| linaro-tcwg-bot/tcwg_glibc_build--master-arm |
success
|
Build passed
|
| linaro-tcwg-bot/tcwg_glibc_check--master-arm |
success
|
Test passed
|
Commit Message
Currently mmap chunks always start at offset 0x010 within a page. Due to the
very low mmap threshold of 128KB there is a high chance of cache conflicts when
accessing multiple allocated arrays that all start at the same offset within a page.
Reduce this by using the unused tail padding to vary the starting offset in
a semi-random (but deterministic) way. Fix various checks that assume the
mmap base is fixed.
Performance of SPECFP2017 improves by 2.2% on Neoverse V2 using default malloc tuning.
---
Comments
On Thu, Oct 1, 2026 at 4:57 PM Wilco Dijkstra <Wilco.Dijkstra@arm.com> wrote:
Needs a logic change at the end.
> diff --git a/malloc/malloc-check.c b/malloc/malloc-check.c
> - unsigned long offset, page_mask = GLRO (dl_pagesize) - 1;
> -
> - /* mmap()ed chunks have MALLOC_ALIGNMENT or higher power-of-two
> - alignment relative to the beginning of a page. Check this
> - first. */
> - offset = (unsigned long) mem & page_mask;
> - if ((offset != MALLOC_ALIGNMENT && offset != 0 && offset != 0x10 &&
> - offset != 0x20 && offset != 0x40 && offset != 0x80 && offset != 0x100 &&
> - offset != 0x200 && offset != 0x400 && offset != 0x800 && offset != 0x1000 &&
> - offset < 0x2000) ||
> - !chunk_is_mmapped (p) || prev_inuse (p) ||
> + unsigned long page_mask = GLRO (dl_pagesize) - 1;
> +
> + if (!chunk_is_mmapped (p) || prev_inuse (p) ||
> (((mmap_base (p)) & page_mask) != 0) ||
> (mmap_size (p) & page_mask) != 0)
> return NULL;
Ok, I think. This is check code so "we can't check for that any more"
sounds reasonable. We already have a generic alignment check above.
> diff --git a/malloc/malloc.c b/malloc/malloc.c
> - mchunkptr p = mmap_set_chunk ((uintptr_t)mm, size, padding, extra_flags != 0);
> + /* Compute unused tail and use to create a semi-random start offset. */
> + size_t tail = (size - padding - nb - CHUNK_HDR_SZ) & -MALLOC_ALIGNMENT;
> + size_t offset = tail ? (MALLOC_ALIGNMENT * ((uintptr_t)mm >> 12)) % tail : 0;
> +
> + mchunkptr p = mmap_set_chunk ((uintptr_t)mm, size, padding + offset, extra_flags != 0);
We need space for a chunk header at the beginning, and one at the end, so...
PADDING HEADER [aligned mmap data] [extra] HEADER
Warning: this code assumes PADDING is at least SIZE_SZ bytes, as we
store the mmap offset in it.
size is all of that. nb is first header plus mmap data. So size
minus nb, padding, and the end header, is the extra. Ok.
mm>>12 is a random(ish) number. Multiply, it's like FFFF0. If
MALLOC_ALIGNMENT is bigger than CHUNK_HDR_SZ, though, the tail won't
be a multiple of MALLOC_ALIGNMENT, so using it as a modulus might
result in an offset that isn't malloc-aligned, but we trim it so that
it is.
Ok.
> @@ -2505,7 +2509,6 @@ munmap_chunk (mchunkptr p)
>
> assert (chunk_is_mmapped (p));
>
> - uintptr_t mem = (uintptr_t) chunk2mem (p);
Not used, ok.
> @@ -2513,8 +2516,7 @@ munmap_chunk (mchunkptr p)
> page size. But gcc does not recognize the optimization possibility
> (in the moment at least) so we combine the two values into one before
> the bit test. */
> - if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0
> - || __glibc_unlikely (!powerof2 (mem & (pagesize - 1))))
> + if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0)
> malloc_printerr ("munmap_chunk(): invalid pointer");
Ok.
> @@ -2540,10 +2542,8 @@ mremap_chunk (mchunkptr p, size_t new_size)
> uintptr_t block = mmap_base (p);
> - uintptr_t mem = (uintptr_t) chunk2mem(p);
> size_t total_size = mmap_size (p);
> - if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0
> - || __glibc_unlikely (!powerof2 (mem & (pagesize - 1))))
> + if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0)
> malloc_printerr("mremap_chunk(): invalid pointer");
Ok.
> @@ -4286,7 +4286,7 @@ _int_memalign (mstate av, size_t alignment, size_t bytes)
>
> if (chunk_is_mmapped (p))
> {
> - newp = mem2chunk (PTR_ALIGN_UP (m, alignment));
> + newp = mem2chunk (PTR_ALIGN_UP (mmap_base (p) + MINSIZE, alignment));
I think this code assumes that MINSIZE is at least CHUNK_HDR_SZ and
while that happens to be true, this logic doesn't assert that or imply
it. I think using CHUNK_HDR_SZ is the right thing to do since the
mmap'd page has to include a prev_size and cur_size before the data,
not a whole min-sized chunk.
> p = mmap_set_chunk (mmap_base (p), mmap_size (p),
> (uintptr_t)newp - mmap_base (p), mmap_is_hp (p));
> return chunk2mem (p);
@@ -137,17 +137,9 @@ mem2chunk_check (void *mem, unsigned char **magic_p)
}
else
{
- unsigned long offset, page_mask = GLRO (dl_pagesize) - 1;
-
- /* mmap()ed chunks have MALLOC_ALIGNMENT or higher power-of-two
- alignment relative to the beginning of a page. Check this
- first. */
- offset = (unsigned long) mem & page_mask;
- if ((offset != MALLOC_ALIGNMENT && offset != 0 && offset != 0x10 &&
- offset != 0x20 && offset != 0x40 && offset != 0x80 && offset != 0x100 &&
- offset != 0x200 && offset != 0x400 && offset != 0x800 && offset != 0x1000 &&
- offset < 0x2000) ||
- !chunk_is_mmapped (p) || prev_inuse (p) ||
+ unsigned long page_mask = GLRO (dl_pagesize) - 1;
+
+ if (!chunk_is_mmapped (p) || prev_inuse (p) ||
(((mmap_base (p)) & page_mask) != 0) ||
(mmap_size (p) & page_mask) != 0)
return NULL;
@@ -1959,7 +1959,11 @@ sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags)
__set_vma_name (mm, size, " glibc: malloc");
- mchunkptr p = mmap_set_chunk ((uintptr_t)mm, size, padding, extra_flags != 0);
+ /* Compute unused tail and use to create a semi-random start offset. */
+ size_t tail = (size - padding - nb - CHUNK_HDR_SZ) & -MALLOC_ALIGNMENT;
+ size_t offset = tail ? (MALLOC_ALIGNMENT * ((uintptr_t)mm >> 12)) % tail : 0;
+
+ mchunkptr p = mmap_set_chunk ((uintptr_t)mm, size, padding + offset, extra_flags != 0);
/* update statistics */
int new = atomic_fetch_add_relaxed (&mp_.n_mmaps, 1) + 1;
@@ -2505,7 +2509,6 @@ munmap_chunk (mchunkptr p)
assert (chunk_is_mmapped (p));
- uintptr_t mem = (uintptr_t) chunk2mem (p);
uintptr_t block = mmap_base (p);
size_t total_size = mmap_size (p);
/* Unfortunately we have to do the compilers job by hand here. Normally
@@ -2513,8 +2516,7 @@ munmap_chunk (mchunkptr p)
page size. But gcc does not recognize the optimization possibility
(in the moment at least) so we combine the two values into one before
the bit test. */
- if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0
- || __glibc_unlikely (!powerof2 (mem & (pagesize - 1))))
+ if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0)
malloc_printerr ("munmap_chunk(): invalid pointer");
atomic_fetch_add_relaxed (&mp_.n_mmaps, -1);
@@ -2540,10 +2542,8 @@ mremap_chunk (mchunkptr p, size_t new_size)
assert (chunk_is_mmapped (p));
uintptr_t block = mmap_base (p);
- uintptr_t mem = (uintptr_t) chunk2mem(p);
size_t total_size = mmap_size (p);
- if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0
- || __glibc_unlikely (!powerof2 (mem & (pagesize - 1))))
+ if (__glibc_unlikely ((block | total_size) & (pagesize - 1)) != 0)
malloc_printerr("mremap_chunk(): invalid pointer");
/* Note the extra CHUNK_HDR_SZ overhead as in mmap_chunk(). */
@@ -4286,7 +4286,7 @@ _int_memalign (mstate av, size_t alignment, size_t bytes)
if (chunk_is_mmapped (p))
{
- newp = mem2chunk (PTR_ALIGN_UP (m, alignment));
+ newp = mem2chunk (PTR_ALIGN_UP (mmap_base (p) + MINSIZE, alignment));
p = mmap_set_chunk (mmap_base (p), mmap_size (p),
(uintptr_t)newp - mmap_base (p), mmap_is_hp (p));
return chunk2mem (p);