[v2,10/15] RISC-V: Add Zbb optimized memchr as ifunc

  When building with Zbb enabled, memchr benefits from using orc.b in
find_zero_all().  This patch changes the build system such, that a
non-Zbb version as well as a Zbb version of this routine is built.
Further, a ifunc resolver is provided that selects the right routine
based on the outcome of extension probing via hwprobe().

Signed-off-by: Christoph Müllner <christoph.muellner@vrull.eu>
---
 sysdeps/riscv/multiarch/memchr-generic.c      | 24 ++++++++
 sysdeps/riscv/multiarch/memchr-zbb.c          | 23 +++++++
 .../unix/sysv/linux/riscv/multiarch/Makefile  |  8 +++
 .../linux/riscv/multiarch/ifunc-impl-list.c   | 37 ++++++++++--
 .../unix/sysv/linux/riscv/multiarch/memchr.c  | 60 +++++++++++++++++++
 5 files changed, 147 insertions(+), 5 deletions(-)
 create mode 100644 sysdeps/riscv/multiarch/memchr-generic.c
 create mode 100644 sysdeps/riscv/multiarch/memchr-zbb.c
 create mode 100644 sysdeps/unix/sysv/linux/riscv/multiarch/memchr.c
  

On 27/05/24 08:18, Christoph Müllner wrote:
> When building with Zbb enabled, memchr benefits from using orc.b in
> find_zero_all().  This patch changes the build system such, that a
> non-Zbb version as well as a Zbb version of this routine is built.
> Further, a ifunc resolver is provided that selects the right routine
> based on the outcome of extension probing via hwprobe().
> 
> Signed-off-by: Christoph Müllner <christoph.muellner@vrull.eu>

I am not if this is worth, but you can remove the ifunc for the case
where compiler does not support the attribute target:

diff --git a/sysdeps/riscv/multiarch/memchr-generic.c b/sysdeps/riscv/multiarch/memchr-generic.c
index c44616c95c..9f95d5afa8 100644
--- a/sysdeps/riscv/multiarch/memchr-generic.c
+++ b/sysdeps/riscv/multiarch/memchr-generic.c
@@ -16,9 +16,11 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */

-#include <string.h>
+#if HAVE_RISCV_FATTRIBUTE_ZBB
+# include <string.h>

-#if IS_IN(libc)
-# define MEMCHR __memchr_generic
+# if IS_IN(libc)
+#  define MEMCHR __memchr_generic
+# endif
+# include <string/memchr.c>
 #endif
-#include <string/memchr.c>
diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
index a19d88fcf8..5b3ec422b8 100644
--- a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
+++ b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
@@ -55,11 +55,11 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
         fast_unaligned = true;
     }

-  IFUNC_IMPL (i, name, memchr,
 #if HAVE_RISCV_FATTRIBUTE_ZBB
+  IFUNC_IMPL (i, name, memchr,
              IFUNC_IMPL_ADD (array, i, memchr, has_zbb, __memchr_zbb)
-#endif
              IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_generic))
+#endif

   IFUNC_IMPL (i, name, memcpy,
              IFUNC_IMPL_ADD (array, i, memcpy, fast_unaligned,
diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/memchr.c b/sysdeps/unix/sysv/linux/riscv/multiarch/memchr.c
index 71a0c039bd..79bc409c4d 100644
--- a/sysdeps/unix/sysv/linux/riscv/multiarch/memchr.c
+++ b/sysdeps/unix/sysv/linux/riscv/multiarch/memchr.c
@@ -17,7 +17,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */

-#if IS_IN (libc)
+#if IS_IN (libc) && HAVE_RISCV_FATTRIBUTE_ZBB

 /* Redefine memchr so that the compiler won't complain about the type
    mismatch with the IFUNC selector in strong_alias, below.  */
@@ -37,12 +37,10 @@ extern __typeof (__redirect_memchr) __memchr_zbb attribute_hidden;
 static inline __typeof (__redirect_memchr) *
 select_memchr_ifunc (uint64_t dl_hwcap, __riscv_hwprobe_t hwprobe_func)
 {
-#if HAVE_RISCV_FATTRIBUTE_ZBB
   unsigned long long int v;
   if (__riscv_hwprobe_one (hwprobe_func, RISCV_HWPROBE_KEY_IMA_EXT_0, &v) == 0
       && (v & RISCV_HWPROBE_EXT_ZBB))
     return __memchr_zbb;
-#endif

   return __memchr_generic;
 }


The rest looks ok.

> ---
>  sysdeps/riscv/multiarch/memchr-generic.c      | 24 ++++++++
>  sysdeps/riscv/multiarch/memchr-zbb.c          | 23 +++++++
>  .../unix/sysv/linux/riscv/multiarch/Makefile  |  8 +++
>  .../linux/riscv/multiarch/ifunc-impl-list.c   | 37 ++++++++++--
>  .../unix/sysv/linux/riscv/multiarch/memchr.c  | 60 +++++++++++++++++++
>  5 files changed, 147 insertions(+), 5 deletions(-)
>  create mode 100644 sysdeps/riscv/multiarch/memchr-generic.c
>  create mode 100644 sysdeps/riscv/multiarch/memchr-zbb.c
>  create mode 100644 sysdeps/unix/sysv/linux/riscv/multiarch/memchr.c
> 
> diff --git a/sysdeps/riscv/multiarch/memchr-generic.c b/sysdeps/riscv/multiarch/memchr-generic.c
> new file mode 100644
> index 0000000000..c44616c95c
> --- /dev/null
> +++ b/sysdeps/riscv/multiarch/memchr-generic.c
> @@ -0,0 +1,24 @@
> +/* Re-include the default memchr implementation.
> +   Copyright (C) 2024 Free Software Foundation, Inc.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <https://www.gnu.org/licenses/>.  */
> +
> +#include <string.h>
> +
> +#if IS_IN(libc)
> +# define MEMCHR __memchr_generic
> +#endif
> +#include <string/memchr.c>
> diff --git a/sysdeps/riscv/multiarch/memchr-zbb.c b/sysdeps/riscv/multiarch/memchr-zbb.c
> new file mode 100644
> index 0000000000..e5de76ec18
> --- /dev/null
> +++ b/sysdeps/riscv/multiarch/memchr-zbb.c
> @@ -0,0 +1,23 @@
> +/* Re-include the default memchr implementation for Zbb.
> +   Copyright (C) 2024 Free Software Foundation, Inc.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <https://www.gnu.org/licenses/>.  */
> +
> +/* Build with Zbb enabled.  */
> +#define __CODEGEN_ATTRIBUTES __attribute__((target("arch=+zbb")))
> +#define USE_ZBB_ORCB 1
> +#define MEMCHR __memchr_zbb
> +#include <string/memchr.c>
> diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
> index fcef5659d4..12ebe0a960 100644
> --- a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
> +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
> @@ -1,9 +1,17 @@
>  ifeq ($(subdir),string)
>  sysdep_routines += \
> +  memchr \
> +  memchr-generic \
>    memcpy \
>    memcpy-generic \
>    memcpy_noalignment \
>    # sysdep_routines
>  
> +ifeq ($(have-fattribute-zbb-support),yes)
> +sysdep_routines += \
> +  memchr-zbb \
> +  # Zbb sysdep_routines
> +endif
> +
>  CFLAGS-memcpy_noalignment.c += -mno-strict-align
>  endif
> diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
> index 9f806d7a9e..e7bb604c5a 100644
> --- a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
> +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
> @@ -20,19 +20,46 @@
>  #include <string.h>
>  #include <sys/hwprobe.h>
>  
> +#define ARRAY_SIZE(A) (sizeof (A) / sizeof ((A)[0]))
> +
>  size_t
>  __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
>  			size_t max)
>  {
>    size_t i = max;
> +  struct riscv_hwprobe pairs[] = {
> +    { .key = RISCV_HWPROBE_KEY_IMA_EXT_0 },
> +    { .key = RISCV_HWPROBE_KEY_CPUPERF_0 },
> +  };
>  
>    bool fast_unaligned = false;
> +#if HAVE_RISCV_FATTRIBUTE_ZBB
> +  bool has_zbb = false;
> +#endif
> +
> +  if (__riscv_hwprobe (pairs, ARRAY_SIZE (pairs), 0, NULL, 0) == 0)
> +    {
> +      struct riscv_hwprobe *pair;
> +
> +      /* RISCV_HWPROBE_KEY_IMA_EXT_0  */
> +      pair = &pairs[0];
> +#if HAVE_RISCV_FATTRIBUTE_ZBB
> +      if (pair->value & RISCV_HWPROBE_EXT_ZBB)
> +        has_zbb = true;
> +#endif
> +
> +      /* RISCV_HWPROBE_KEY_CPUPERF_0  */
> +      pair = &pairs[1];
> +      if ((pair->value & RISCV_HWPROBE_MISALIGNED_MASK)
> +	   == RISCV_HWPROBE_MISALIGNED_FAST)
> +        fast_unaligned = true;
> +    }
>  
> -  struct riscv_hwprobe pair = { .key = RISCV_HWPROBE_KEY_CPUPERF_0 };
> -  if (__riscv_hwprobe (&pair, 1, 0, NULL, 0) == 0
> -      && (pair.value & RISCV_HWPROBE_MISALIGNED_MASK)
> -          == RISCV_HWPROBE_MISALIGNED_FAST)
> -    fast_unaligned = true;
> +  IFUNC_IMPL (i, name, memchr,
> +#if HAVE_RISCV_FATTRIBUTE_ZBB
> +	      IFUNC_IMPL_ADD (array, i, memchr, has_zbb, __memchr_zbb)
> +#endif
> +	      IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_generic))
>  
>    IFUNC_IMPL (i, name, memcpy,
>  	      IFUNC_IMPL_ADD (array, i, memcpy, fast_unaligned,
> diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/memchr.c b/sysdeps/unix/sysv/linux/riscv/multiarch/memchr.c
> new file mode 100644
> index 0000000000..71a0c039bd
> --- /dev/null
> +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/memchr.c
> @@ -0,0 +1,60 @@
> +/* Multiple versions of memchr.
> +   All versions must be listed in ifunc-impl-list.c.
> +   Copyright (C) 2017-2024 Free Software Foundation, Inc.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <https://www.gnu.org/licenses/>.  */
> +
> +#if IS_IN (libc)
> +
> +/* Redefine memchr so that the compiler won't complain about the type
> +   mismatch with the IFUNC selector in strong_alias, below.  */
> +# undef memchr
> +# define memchr __redirect_memchr
> +# include <stdint.h>
> +# include <string.h>
> +# include <ifunc-init.h>
> +# include <riscv-ifunc.h>
> +# include <sys/hwprobe.h>
> +
> +extern __typeof (__redirect_memchr) __libc_memchr;
> +
> +extern __typeof (__redirect_memchr) __memchr_generic attribute_hidden;
> +extern __typeof (__redirect_memchr) __memchr_zbb attribute_hidden;
> +
> +static inline __typeof (__redirect_memchr) *
> +select_memchr_ifunc (uint64_t dl_hwcap, __riscv_hwprobe_t hwprobe_func)
> +{
> +#if HAVE_RISCV_FATTRIBUTE_ZBB
> +  unsigned long long int v;
> +  if (__riscv_hwprobe_one (hwprobe_func, RISCV_HWPROBE_KEY_IMA_EXT_0, &v) == 0
> +      && (v & RISCV_HWPROBE_EXT_ZBB))
> +    return __memchr_zbb;
> +#endif
> +
> +  return __memchr_generic;
> +}
> +
> +riscv_libc_ifunc (__libc_memchr, select_memchr_ifunc);
> +
> +# undef memchr
> +strong_alias (__libc_memchr, memchr);
> +# ifdef SHARED
> +__hidden_ver1 (memchr, __GI_memchr, __redirect_memchr)
> +  __attribute__ ((visibility ("hidden"))) __attribute_copy__ (memchr);
> +# endif
> +#else
> +# include <string/memchr.c>
> +#endif