diff mbox series

riscv: Add RVV implementation of wcschr

Message ID 20260919151018.1543897-1-zhangjinhan@isrc.iscas.ac.cn (mailing list archive)
State New
Headers
Series riscv: Add RVV implementation of wcschr |

Checks

Context Check Description
redhat-pt-bot/TryBot-apply_patch success Patch applied to master at the time it was sent
redhat-pt-bot/TryBot-32bit success Build for i686
linaro-tcwg-bot/tcwg_glibc_build--master-aarch64 success Build passed
linaro-tcwg-bot/tcwg_glibc_build--master-arm success Build passed
linaro-tcwg-bot/tcwg_glibc_check--master-aarch64 success Test passed
linaro-tcwg-bot/tcwg_glibc_check--master-arm success Test passed

Commit Message

Zhang Jinhan Sept. 19, 2026, 3:10 p.m. UTC 
  Add an RVV implementation of wcschr and select it through the RISC-V
multiarch IFUNC resolver when the V extension is available.  Retain the
generic implementation as the fallback and register both implementations
in the IFUNC implementation list.

Check the first element before setting up the search loops.  Use
four-way-unrolled scalar loops for the first 32 elements when searching
for a nonzero character and the first 40 elements when searching for NUL.
Then probe with e32, LMUL=1 before switching to an LMUL=8 loop.  Use
ordinary vector loads within a 4 KiB boundary and fault-only-first loads
when crossing that boundary.  The NUL search uses a dedicated path that
only checks for zero elements.

On a SpacemiT K3/X100 with VLEN=256, bench-wcschr shows an overall
2.5036x speedup (+150.36%) over the generic implementation, equivalent
to a 60.06% reduction in execution time.  All 498 case medians improve;
the smallest individual speedup is 1.2161x (+21.61%).

The source baseline is glibc commit
afc3228df04b64d94519bd00b61907560208bbb1.
The measurements compare __wcschr_generic and __wcschr_vector in the
same patched build, paired in each run.  Execution was pinned to CPU 2,
with the userspace governor and an observed frequency of 2.2 GHz.
One warmup and six measured runs were performed, each covering 498
cases.  For each case, take the median of its six timings for each
implementation, then compute the geometric mean across cases.

The following table gives those geometric means.  Timing values use
the benchmark output scale; lower is better.  Speedup is the ratio
of generic to vector, and gain is (generic / vector - 1) * 100%.

Position     Cases      Generic       Vector   Speedup      Gain
All            498    97.362814    38.889854    2.5036x   +150.36%
0-7             42    12.029031     6.635501    1.8128x    +81.28%
8-15            64    23.475281    15.036105    1.5613x    +56.13%
16-31           84    35.789723    21.191702    1.6889x    +68.89%
>=32           308   228.633868    71.158150    3.2130x   +221.30%

The per-run geometric means across all 498 cases are:

Run                    Generic       Vector       Gain
1                    96.712354    38.672368    +150.08%
2                    97.209984    38.629858    +151.64%
3                    96.683859    38.656297    +150.11%
4                   106.445429    47.093983    +126.03%
5                   105.557872    46.824549    +125.43%
6                    96.895320    38.721093    +150.24%

The smallest individual gain is for length=16, pos=15, rand=1,
branch=1, perc-zero=1, alignment=0: the case median falls from
27.2750 to 22.4280 (+21.61%).  This is an individual case, not a
position-group geometric mean.  The largest individual gain is
+510.65%.  The two immediate-NUL cases improve by +58.82% and +49.95%,
and the two target-at-position-31 cases by +39.60% and +24.49%.

No case meets the repeatable-regression criterion: at least a 5%
slowdown in the case median and at least a 5% slowdown in four or
more of the six paired runs.

Validation on the test system:
- Patch application: PASS.
- Incremental RISC-V build: PASS.
- make test t=wcsmbs/test-wcschr: PASS, enumerating __wcschr_default,
  __wcschr_vector, and __wcschr_generic.
- benchtests/bench-wcschr: 498/498 cases covered.

Co-authored-by: Ning Tian <tianning24@iscas.ac.cn>
Co-authored-by: Yuansheng <yuansheng@isrc.iscas.ac.cn>
Signed-off-by: Ning Tian <tianning24@iscas.ac.cn>
Signed-off-by: Yuansheng <yuansheng@isrc.iscas.ac.cn>
Signed-off-by: Zhang Jinhan <zhangjinhan@isrc.iscas.ac.cn>
---
 sysdeps/riscv/multiarch/wcschr-generic.c      |  17 ++
 sysdeps/riscv/multiarch/wcschr-vector.S       |  13 ++
 sysdeps/riscv/rvv/wcschr.S                    | 200 ++++++++++++++++++
 .../unix/sysv/linux/riscv/multiarch/Makefile  |   7 +
 .../linux/riscv/multiarch/ifunc-impl-list.c   |   6 +
 .../unix/sysv/linux/riscv/multiarch/wcschr.c  |  44 ++++
 6 files changed, 287 insertions(+)
 create mode 100644 sysdeps/riscv/multiarch/wcschr-generic.c
 create mode 100644 sysdeps/riscv/multiarch/wcschr-vector.S
 create mode 100644 sysdeps/riscv/rvv/wcschr.S
 create mode 100644 sysdeps/unix/sysv/linux/riscv/multiarch/wcschr.c


base-commit: d5bd0f22305277fa0d431bb9d733aa9456706573
diff mbox series

Patch

diff --git a/sysdeps/riscv/multiarch/wcschr-generic.c b/sysdeps/riscv/multiarch/wcschr-generic.c
new file mode 100644
index 0000000000..ee5810baa3
--- /dev/null
+++ b/sysdeps/riscv/multiarch/wcschr-generic.c
@@ -0,0 +1,17 @@ 
+/* Re-include the default wcschr implementation.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Licensed under the GNU Lesser General Public License v2.1+; see COPYING.LESSER.  */
+
+#include <wchar.h>
+
+#if IS_IN(libc)
+# define WCSCHR __wcschr_generic
+# undef libc_hidden_def
+# define libc_hidden_def(name)
+# undef libc_hidden_weak
+# define libc_hidden_weak(name)
+# undef static_weak_alias
+# define static_weak_alias(name, alias)
+# include <wcsmbs/wcschr.c>
+#endif
diff --git a/sysdeps/riscv/multiarch/wcschr-vector.S b/sysdeps/riscv/multiarch/wcschr-vector.S
new file mode 100644
index 0000000000..a4b1cd215e
--- /dev/null
+++ b/sysdeps/riscv/multiarch/wcschr-vector.S
@@ -0,0 +1,13 @@ 
+/* RISC-V RVV based wcschr.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Licensed under the GNU Lesser General Public License v2.1+; see COPYING.LESSER.  */
+
+#if IS_IN(libc)
+# define WCSCHR __wcschr_vector
+# undef libc_hidden_builtin_def
+# define libc_hidden_builtin_def(name)
+# undef weak_alias
+# define weak_alias(name, alias)
+# include <sysdeps/riscv/rvv/wcschr.S>
+#endif
diff --git a/sysdeps/riscv/rvv/wcschr.S b/sysdeps/riscv/rvv/wcschr.S
new file mode 100644
index 0000000000..e3c1908714
--- /dev/null
+++ b/sysdeps/riscv/rvv/wcschr.S
@@ -0,0 +1,200 @@ 
+/* RISC-V RVV based wcschr.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Licensed under the GNU Lesser General Public License v2.1+; see COPYING.LESSER.  */
+
+#include <sysdep.h>
+#include <sys/asm.h>
+
+#ifndef WCSCHR
+# define WCSCHR __wcschr
+#endif
+
+#define wcs a0
+#define wc a1
+#define vl_reg a2
+#define step a3
+#define first a4
+#define post_vl a5
+#define page_size a6
+#define page_end a7
+#define scalar t0
+#define page_mask t1
+
+#define vchunk v16
+#define vhit v0
+#define vtmp v1
+
+ENTRY (WCSCHR)
+.option push
+.option arch, +v
+	/* Check the first element before selecting the target-specific path.  It
+	   keeps the most latency-sensitive match and NUL cases out of both loop
+	   setup and vector startup.  */
+	lw	scalar, 0(wcs)
+	beq	scalar, wc, L(ret)
+	beqz	scalar, L(not_found)
+	addi	wcs, wcs, 4
+	beqz	wc, L(zero_scalar)
+
+	/* Scan 32 elements in total before paying vector startup.  Process four
+	   elements per iteration to reduce pointer updates and code footprint,
+	   with only one loop backedge per group.  */
+	lw	scalar, 0(wcs)
+	beq	scalar, wc, L(ret)
+	beqz	scalar, L(not_found)
+	lw	scalar, 4(wcs)
+	beq	scalar, wc, L(ret4)
+	beqz	scalar, L(not_found)
+	lw	scalar, 8(wcs)
+	beq	scalar, wc, L(ret8)
+	beqz	scalar, L(not_found)
+	addi	wcs, wcs, 12
+	li	step, 7
+L(scalar_loop):
+	lw	scalar, 0(wcs)
+	beq	scalar, wc, L(ret)
+	beqz	scalar, L(not_found)
+	lw	scalar, 4(wcs)
+	beq	scalar, wc, L(ret4)
+	beqz	scalar, L(not_found)
+	lw	scalar, 8(wcs)
+	beq	scalar, wc, L(ret8)
+	beqz	scalar, L(not_found)
+	lw	scalar, 12(wcs)
+	beq	scalar, wc, L(ret12)
+	beqz	scalar, L(not_found)
+	addi	wcs, wcs, 16
+	addi	step, step, -1
+	bnez	step, L(scalar_loop)
+
+	vsetvli	vl_reg, zero, e32, m1, ta, ma
+	vle32ff.v	vchunk, (wcs)
+	vmseq.vx	vtmp, vchunk, wc
+	vmseq.vi	vhit, vchunk, 0
+	vmor.mm	vhit, vhit, vtmp
+	vfirst.m	first, vhit
+	bgez	first, L(found_nonzero)
+	/* A fault-only-first load may shorten vl.  Read it only on the miss
+	   path, where it is needed to advance the input pointer.  */
+	csrr	post_vl, vl
+	slli	post_vl, post_vl, 2
+	add	wcs, wcs, post_vl
+	vsetvli	vl_reg, zero, e32, m8, ta, ma
+	slli	step, vl_reg, 2
+	li	page_size, 4096
+	addi	page_mask, page_size, -1
+
+L(long_loop):
+	and	page_end, wcs, page_mask
+	add	page_end, page_end, step
+	bgtu	page_end, page_size, L(long_page_cross)
+	vle32.v	vchunk, (wcs)
+	vmseq.vx	vtmp, vchunk, wc
+	vmseq.vi	vhit, vchunk, 0
+	vmor.mm	vhit, vhit, vtmp
+	vfirst.m	first, vhit
+	bgez	first, L(found_nonzero)
+	add	wcs, wcs, step
+	j	L(long_loop)
+
+L(long_page_cross):
+	vle32ff.v	vchunk, (wcs)
+	vmseq.vx	vtmp, vchunk, wc
+	vmseq.vi	vhit, vchunk, 0
+	vmor.mm	vhit, vhit, vtmp
+	vfirst.m	first, vhit
+	bgez	first, L(found_nonzero)
+	csrr	post_vl, vl
+	slli	post_vl, post_vl, 2
+	add	wcs, wcs, post_vl
+	vsetvli	zero, zero, e32, m8, ta, ma
+	j	L(long_loop)
+
+L(found_nonzero):
+	slli	first, first, 2
+	add	wcs, wcs, first
+	lw	scalar, 0(wcs)
+	beq	scalar, wc, L(ret)
+L(not_found):
+	li	wcs, 0
+L(ret):
+	ret
+L(ret4):
+	addi	wcs, wcs, 4
+	ret
+L(ret8):
+	addi	wcs, wcs, 8
+	ret
+L(ret12):
+	addi	wcs, wcs, 12
+	ret
+
+	/* Searching for L'\0' needs only the NUL mask and does not require a
+	   scalar reload after vfirst.m.  */
+L(zero_scalar):
+	/* The first element was already checked.  Scan elements 1..39 with the
+	   same four-way unrolling before entering the NUL-only vector path.  */
+	lw	scalar, 0(wcs)
+	beqz	scalar, L(ret)
+	lw	scalar, 4(wcs)
+	beqz	scalar, L(ret4)
+	lw	scalar, 8(wcs)
+	beqz	scalar, L(ret8)
+	addi	wcs, wcs, 12
+	li	step, 9
+L(zero_scalar_loop):
+	lw	scalar, 0(wcs)
+	beqz	scalar, L(ret)
+	lw	scalar, 4(wcs)
+	beqz	scalar, L(ret4)
+	lw	scalar, 8(wcs)
+	beqz	scalar, L(ret8)
+	lw	scalar, 12(wcs)
+	beqz	scalar, L(ret12)
+	addi	wcs, wcs, 16
+	addi	step, step, -1
+	bnez	step, L(zero_scalar_loop)
+
+L(zero_first_vector):
+	vsetvli	vl_reg, zero, e32, m1, ta, ma
+	vle32ff.v	vchunk, (wcs)
+	vmseq.vi	vhit, vchunk, 0
+	vfirst.m	first, vhit
+	bgez	first, L(found_zero)
+	csrr	post_vl, vl
+	slli	post_vl, post_vl, 2
+	add	wcs, wcs, post_vl
+	vsetvli	vl_reg, zero, e32, m8, ta, ma
+	slli	step, vl_reg, 2
+	li	page_size, 4096
+	addi	page_mask, page_size, -1
+
+L(zero_long_loop):
+	and	page_end, wcs, page_mask
+	add	page_end, page_end, step
+	bgtu	page_end, page_size, L(zero_long_page_cross)
+	vle32.v	vchunk, (wcs)
+	vmseq.vi	vhit, vchunk, 0
+	vfirst.m	first, vhit
+	bgez	first, L(found_zero)
+	add	wcs, wcs, step
+	j	L(zero_long_loop)
+
+L(zero_long_page_cross):
+	vle32ff.v	vchunk, (wcs)
+	vmseq.vi	vhit, vchunk, 0
+	vfirst.m	first, vhit
+	bgez	first, L(found_zero)
+	csrr	post_vl, vl
+	slli	post_vl, post_vl, 2
+	add	wcs, wcs, post_vl
+	vsetvli	zero, zero, e32, m8, ta, ma
+	j	L(zero_long_loop)
+
+L(found_zero):
+	slli	first, first, 2
+	add	wcs, wcs, first
+	ret
+.option pop
+END (WCSCHR)
diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
index bdbd12a2b6..4039604951 100644
--- a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
+++ b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
@@ -59,3 +59,10 @@  CFLAGS-strlen-generic.op = $(no-stack-protector)
 
 CFLAGS-memcpy_noalignment.c += -mno-strict-align
 endif
+
+ifeq ($(subdir),wcsmbs)
+sysdep_routines += \
+  wcschr-generic \
+  wcschr-vector \
+  # sysdep_routines
+endif
diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
index ac2b25de74..ebcf24f677 100644
--- a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
+++ b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
@@ -18,6 +18,7 @@ 
 
 #include <ifunc-impl-list.h>
 #include <string.h>
+#include <wchar.h>
 #include <sys/hwprobe.h>
 
 size_t
@@ -120,5 +121,10 @@  __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 			      __strncpy_vector)
 	      IFUNC_IMPL_ADD (array, i, strncpy, 1, __strncpy_generic))
 
+  IFUNC_IMPL (i, name, wcschr,
+	      IFUNC_IMPL_ADD (array, i, wcschr, rvv_enabled,
+			      __wcschr_vector)
+	      IFUNC_IMPL_ADD (array, i, wcschr, 1, __wcschr_generic))
+
   return 0;
 }
diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/wcschr.c b/sysdeps/unix/sysv/linux/riscv/multiarch/wcschr.c
new file mode 100644
index 0000000000..a66474598d
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/riscv/multiarch/wcschr.c
@@ -0,0 +1,44 @@ 
+/* Multiple versions of wcschr.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Licensed under the GNU Lesser General Public License v2.1+; see COPYING.LESSER.  */
+
+#if IS_IN (libc)
+# undef wcschr
+# define wcschr __redirect_wcschr
+# include <stdint.h>
+# include <wchar.h>
+# include <ifunc-init.h>
+# include <riscv-ifunc.h>
+# include <sys/hwprobe.h>
+
+extern __typeof (__redirect_wcschr) __libc_wcschr;
+
+extern __typeof (__redirect_wcschr) __wcschr_generic attribute_hidden;
+extern __typeof (__redirect_wcschr) __wcschr_vector attribute_hidden;
+
+static inline __typeof (__redirect_wcschr) *
+select_wcschr_ifunc (uint64_t dl_hwcap, __riscv_hwprobe_t hwprobe_func)
+{
+  unsigned long long v;
+
+  if (__riscv_hwprobe_one (hwprobe_func, RISCV_HWPROBE_KEY_IMA_EXT_0, &v) == 0
+      && (v & RISCV_HWPROBE_IMA_V) == RISCV_HWPROBE_IMA_V)
+    return __wcschr_vector;
+
+  return __wcschr_generic;
+}
+
+riscv_libc_ifunc (__libc_wcschr, select_wcschr_ifunc);
+
+# undef wcschr
+strong_alias (__libc_wcschr, wcschr);
+strong_alias (wcschr, __wcschr)
+# ifdef SHARED
+__hidden_ver1 (wcschr, __GI_wcschr, __redirect_wcschr)
+  __attribute__ ((visibility ("hidden"))) __attribute_copy__ (wcschr);
+# endif
+#else
+# include <wcsmbs/wcschr.c>
+#endif