Checks
| Context |
Check |
Description |
| redhat-pt-bot/TryBot-apply_patch |
success
|
Patch applied to master at the time it was sent
|
| linaro-tcwg-bot/tcwg_glibc_build--master-arm |
success
|
Build passed
|
| redhat-pt-bot/TryBot-32bit |
success
|
Build for i686
|
| linaro-tcwg-bot/tcwg_glibc_build--master-aarch64 |
success
|
Build passed
|
| linaro-tcwg-bot/tcwg_glibc_check--master-aarch64 |
success
|
Test passed
|
| linaro-tcwg-bot/tcwg_glibc_check--master-arm |
success
|
Test passed
|
| redhat-pt-bot/TryBot-still_applies |
warning
|
Patch no longer applies to master
|
Commit Message
Add an SVE2 strlen implementation and ifunc support. It processes
2 vectors per iteration which gives ~40% speedup on large sizes on
Neoverse V2. Latency is ~10% better in bench-strlen-random.
Passes regress, OK for commit?
---
Comments
On 01/10/26 16:36, Wilco Dijkstra wrote:
>
> Add an SVE2 strlen implementation and ifunc support. It processes
> 2 vectors per iteration which gives ~40% speedup on large sizes on
> Neoverse V2. Latency is ~10% better in bench-strlen-random.
>
> Passes regress, OK for commit?
LGTM with the change below.
Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
>
> ---
>
> diff --git a/SHARED-FILES b/SHARED-FILES
> index 63f003173265c126b85f67b8d180422b5ac53d10..67ccee54b4845e76d800e8277c75a4f88f010300 100644
> --- a/SHARED-FILES
> +++ b/SHARED-FILES
> @@ -829,6 +829,8 @@ arm-optimized-routines:
> # string/aarch64/memset-sve.S, revision 0f71101
> sysdeps/aarch64/multiarch/memset_sve_zva64.S
> # string/aarch64/memset.S, revision 098df8c
> + sysdeps/aarch64/multiarch/strlen_sve2.S
> + # string/aarch64/strlen-sve2.S, revision 375f32e
> sysdeps/aarch64/memset.S
> # string/aarch64/stpcpy.S, revision 189dfef
> sysdeps/aarch64/stpcpy.S
> diff --git a/sysdeps/aarch64/multiarch/Makefile b/sysdeps/aarch64/multiarch/Makefile
> index 1b70d5eecd7a8e073889ae61a918b211c9d6221d..054e59b64723f9a9ec142272c5d48930f8b26125 100644
> --- a/sysdeps/aarch64/multiarch/Makefile
> +++ b/sysdeps/aarch64/multiarch/Makefile
> @@ -20,6 +20,7 @@ sysdep_routines += \
> memset_zva64 \
> strlen_asimd \
> strlen_generic \
> + strlen_sve2 \
> # sysdep_routines
> endif # ifeq ($(subdir),string)
>
> diff --git a/sysdeps/aarch64/multiarch/ifunc-impl-list.c b/sysdeps/aarch64/multiarch/ifunc-impl-list.c
> index 486ee29436cc7e2803dd1eecc6d23bb953858d5e..9188208b5652fe04e3416f09b5b1fbe5b3a22fca 100644
> --- a/sysdeps/aarch64/multiarch/ifunc-impl-list.c
> +++ b/sysdeps/aarch64/multiarch/ifunc-impl-list.c
> @@ -61,8 +61,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
> IFUNC_IMPL_ADD (array, i, memset, mops, __memset_mops)
> IFUNC_IMPL_ADD (array, i, memset, 1, __memset_generic))
> IFUNC_IMPL (i, name, strlen,
> + IFUNC_IMPL_ADD (array, i, strlen, sve2 && !mte, __strlen_sve2)
> IFUNC_IMPL_ADD (array, i, strlen, !mte, __strlen_asimd)
> - IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_generic))
> -
> + IFUNC_IMPL_ADD (array, i, strlen, true, __strlen_generic))
I think this is not strickly required for this enablement, so maybe make
it a different patch.
> return 0;
> }
> diff --git a/sysdeps/aarch64/multiarch/strlen.c b/sysdeps/aarch64/multiarch/strlen.c
> index f7479758e48e000549699014b28c143b9f4a6aa0..47d87aa473af70528293e9673300d1f7d001816d 100644
> --- a/sysdeps/aarch64/multiarch/strlen.c
> +++ b/sysdeps/aarch64/multiarch/strlen.c
> @@ -30,8 +30,10 @@ extern __typeof (__redirect_strlen) __strlen;
>
> extern __typeof (__redirect_strlen) __strlen_generic attribute_hidden;
> extern __typeof (__redirect_strlen) __strlen_asimd attribute_hidden;
> +extern __typeof (__redirect_strlen) __strlen_sve2 attribute_hidden;
>
> -libc_ifunc (__strlen, (mte ? __strlen_generic : __strlen_asimd));
> +libc_ifunc (__strlen, (mte ? __strlen_generic : sve2 ? __strlen_sve2 :
> + __strlen_asimd));
>
> # undef strlen
> strong_alias (__strlen, strlen);
> diff --git a/sysdeps/aarch64/multiarch/strlen_sve2.S b/sysdeps/aarch64/multiarch/strlen_sve2.S
> new file mode 100644
> index 0000000000000000000000000000000000000000..069c7c38eafbb52750035d893d020eb08a708200
> --- /dev/null
> +++ b/sysdeps/aarch64/multiarch/strlen_sve2.S
> @@ -0,0 +1,122 @@
> +/* Optimized strlen implementation using SVE2.
> + Copyright (C) 2026 Free Software Foundation, Inc.
> +
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library. If not, see
> + <https://www.gnu.org/licenses/>. */
> +
> +#include <sysdep.h>
> +
> +.arch armv9-a+sve2
> +
> +/* Assumptions:
> + *
> + * AArch64
> + * SVE2 Available.
> + */
> +
> +#define srcin x0
> +#define result x0
> +
> +#define negalign x1
> +#define src x2
> +#define pgoffset x3 /* Offset in page. */
> +#define tmp x4
> +#define off1 x5
> +#define off2 x6
> +
> +#define zdata z0
> +#define zdata2 z1
> +
> +#define pall p0
> +#define pnul p1 /* Search vector with NUL. */
> +#define pnul2 p2
> +#define p1stchunk p3 /* Mask for first chunk. */
> +#define pindex p4 /* Mask for index of match. */
> +
> +ENTRY (__strlen_sve2)
> + ptrue pall.b /* Initialize all active. */
> + and pgoffset, srcin, 0xfff
> + incb pgoffset, all, mul #2
> + tbnz pgoffset, 12, L(pagecross)
> +
> + ldbsve zdata, [srcin] /* Read first chunk. */
> + ldbsve zdata2, [srcin, 1, mul vl] /* Read next chunk. */
> + cmpeq pnul2.b, pall/z, zdata2.b, 0 /* Find NUL. */
> + cset tmp, none
> + cmpeq pnul.b, pall/z, zdata.b, 0
> + csel tmp, tmp, xzr, none
> + cbnz tmp, L(preloop)
> + brkb pnul.b, pall/z, pnul.b
> + brkb pnul2.b, pall/z, pnul2.b
> + mov off2, 0
> + incb off2
> + cntp off1, pall, pnul.b
> + incp off2, pnul2.b
> + csel result, off1, off2, any
> + ret
> +
> + .p2align 4
> + nop
> +L(preloop):
> + mov negalign, 0
> + decb negalign
> + and src, srcin, negalign, lsl 1 /* Align src to 2xVL. */
> +
> + .p2align 4
> +L(loop):
> + ldbsve zdata, [src, 2, mul vl]
> + ldbsve zdata2, [src, 3, mul vl]
> + sub src, src, negalign, lsl 1 /* Increment src by 2xVL. */
> + umin zdata2.b, pall/m, zdata2.b, zdata.b
> + cmpeq pnul.b, pall/z, zdata2.b, 0 /* Find NUL. */
> + b.none L(loop)
> +
> + cmpeq pindex.b, pall/z, zdata.b, 0 /* Find NUL. */
> + brkb pnul.b, pall/z, pnul.b
> + brkb pindex.b, pall/z, pindex.b
> + sub off1, src, srcin
> + sub off2, off1, negalign
> + incp off1, pindex.b
> + incp off2, pnul.b
> + csel result, off1, off2, any
> + ret
> +
> + .p2align 4
> +L(pagecross):
> + mov negalign, 0
> + decb negalign
> + and src, srcin, negalign /* Align src to VL. */
> + sub tmp, src, negalign
> + whilehi p1stchunk.b, tmp, srcin /* Mask for first chunk. */
> + ldbsve zdata, [src] /* Read first chunk. */
> + cmpeq pnul.b, p1stchunk/z, zdata.b, 0 /* Find NUL. */
> + b.none L(2ndchunk) /* Next chunk if not found. */
> +
> + brkb pindex.b, p1stchunk/z, pnul.b /* Find first element. */
> + cntp result, pall, pindex.b
> + ret
> +
> +L(2ndchunk):
> + ldbsve zdata, [src, 1, mul vl] /* Read second chunk. */
> + and src, srcin, negalign, lsl 1 /* Align src to 2xVL. */
> + cmpeq pnul.b, pall/z, zdata.b, 0 /* Find NUL. */
> + b.none L(loop) /* Loop if not found. */
> +
> + sub result, tmp, srcin /* Length until current src. */
> + brkb pindex.b, pall/z, pnul.b /* Find first element. */
> + incp result, pindex.b /* Add index. */
> + ret
> +END (__strlen_sve2)
@@ -829,6 +829,8 @@ arm-optimized-routines:
# string/aarch64/memset-sve.S, revision 0f71101
sysdeps/aarch64/multiarch/memset_sve_zva64.S
# string/aarch64/memset.S, revision 098df8c
+ sysdeps/aarch64/multiarch/strlen_sve2.S
+ # string/aarch64/strlen-sve2.S, revision 375f32e
sysdeps/aarch64/memset.S
# string/aarch64/stpcpy.S, revision 189dfef
sysdeps/aarch64/stpcpy.S
@@ -20,6 +20,7 @@ sysdep_routines += \
memset_zva64 \
strlen_asimd \
strlen_generic \
+ strlen_sve2 \
# sysdep_routines
endif # ifeq ($(subdir),string)
@@ -61,8 +61,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, memset, mops, __memset_mops)
IFUNC_IMPL_ADD (array, i, memset, 1, __memset_generic))
IFUNC_IMPL (i, name, strlen,
+ IFUNC_IMPL_ADD (array, i, strlen, sve2 && !mte, __strlen_sve2)
IFUNC_IMPL_ADD (array, i, strlen, !mte, __strlen_asimd)
- IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_generic))
-
+ IFUNC_IMPL_ADD (array, i, strlen, true, __strlen_generic))
return 0;
}
@@ -30,8 +30,10 @@ extern __typeof (__redirect_strlen) __strlen;
extern __typeof (__redirect_strlen) __strlen_generic attribute_hidden;
extern __typeof (__redirect_strlen) __strlen_asimd attribute_hidden;
+extern __typeof (__redirect_strlen) __strlen_sve2 attribute_hidden;
-libc_ifunc (__strlen, (mte ? __strlen_generic : __strlen_asimd));
+libc_ifunc (__strlen, (mte ? __strlen_generic : sve2 ? __strlen_sve2 :
+ __strlen_asimd));
# undef strlen
strong_alias (__strlen, strlen);
new file mode 100644
@@ -0,0 +1,122 @@
+/* Optimized strlen implementation using SVE2.
+ Copyright (C) 2026 Free Software Foundation, Inc.
+
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library. If not, see
+ <https://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+.arch armv9-a+sve2
+
+/* Assumptions:
+ *
+ * AArch64
+ * SVE2 Available.
+ */
+
+#define srcin x0
+#define result x0
+
+#define negalign x1
+#define src x2
+#define pgoffset x3 /* Offset in page. */
+#define tmp x4
+#define off1 x5
+#define off2 x6
+
+#define zdata z0
+#define zdata2 z1
+
+#define pall p0
+#define pnul p1 /* Search vector with NUL. */
+#define pnul2 p2
+#define p1stchunk p3 /* Mask for first chunk. */
+#define pindex p4 /* Mask for index of match. */
+
+ENTRY (__strlen_sve2)
+ ptrue pall.b /* Initialize all active. */
+ and pgoffset, srcin, 0xfff
+ incb pgoffset, all, mul #2
+ tbnz pgoffset, 12, L(pagecross)
+
+ ldbsve zdata, [srcin] /* Read first chunk. */
+ ldbsve zdata2, [srcin, 1, mul vl] /* Read next chunk. */
+ cmpeq pnul2.b, pall/z, zdata2.b, 0 /* Find NUL. */
+ cset tmp, none
+ cmpeq pnul.b, pall/z, zdata.b, 0
+ csel tmp, tmp, xzr, none
+ cbnz tmp, L(preloop)
+ brkb pnul.b, pall/z, pnul.b
+ brkb pnul2.b, pall/z, pnul2.b
+ mov off2, 0
+ incb off2
+ cntp off1, pall, pnul.b
+ incp off2, pnul2.b
+ csel result, off1, off2, any
+ ret
+
+ .p2align 4
+ nop
+L(preloop):
+ mov negalign, 0
+ decb negalign
+ and src, srcin, negalign, lsl 1 /* Align src to 2xVL. */
+
+ .p2align 4
+L(loop):
+ ldbsve zdata, [src, 2, mul vl]
+ ldbsve zdata2, [src, 3, mul vl]
+ sub src, src, negalign, lsl 1 /* Increment src by 2xVL. */
+ umin zdata2.b, pall/m, zdata2.b, zdata.b
+ cmpeq pnul.b, pall/z, zdata2.b, 0 /* Find NUL. */
+ b.none L(loop)
+
+ cmpeq pindex.b, pall/z, zdata.b, 0 /* Find NUL. */
+ brkb pnul.b, pall/z, pnul.b
+ brkb pindex.b, pall/z, pindex.b
+ sub off1, src, srcin
+ sub off2, off1, negalign
+ incp off1, pindex.b
+ incp off2, pnul.b
+ csel result, off1, off2, any
+ ret
+
+ .p2align 4
+L(pagecross):
+ mov negalign, 0
+ decb negalign
+ and src, srcin, negalign /* Align src to VL. */
+ sub tmp, src, negalign
+ whilehi p1stchunk.b, tmp, srcin /* Mask for first chunk. */
+ ldbsve zdata, [src] /* Read first chunk. */
+ cmpeq pnul.b, p1stchunk/z, zdata.b, 0 /* Find NUL. */
+ b.none L(2ndchunk) /* Next chunk if not found. */
+
+ brkb pindex.b, p1stchunk/z, pnul.b /* Find first element. */
+ cntp result, pall, pindex.b
+ ret
+
+L(2ndchunk):
+ ldbsve zdata, [src, 1, mul vl] /* Read second chunk. */
+ and src, srcin, negalign, lsl 1 /* Align src to 2xVL. */
+ cmpeq pnul.b, pall/z, zdata.b, 0 /* Find NUL. */
+ b.none L(loop) /* Loop if not found. */
+
+ sub result, tmp, srcin /* Length until current src. */
+ brkb pindex.b, pall/z, pnul.b /* Find first element. */
+ incp result, pindex.b /* Add index. */
+ ret
+END (__strlen_sve2)