PowerPC: optimized strpbrk for POWER7
Commit Message
I have added a sanity check for early exit. Some comments also included.
On Sunday 09 March 2014 02:46 AM, Adhemerval Zanella wrote:
> I am resending this patch to keep track using the patchwork and with
> some corrections (string/strpbrk.c cleanup and the no need to align
> stack, since stack is already aligned to 16 bytes in PPC64 abi).
>
> This patch add an optimized strpbrk for POWER7 by using a different
> algorithm than default implementation: it constructs a table based on
> the 'accept' argument and use this table to check for any occurance on
> the input string. The idea is similar as x86_64 uses.
> For PowerPC some tunings were added, such as unroll loops and memory
> clear using VSX instructions.
>
> --
>
> 2014-03-08 Adhemerval Zanella <azanella@linux.vnet.ibm.com>
> Vidya Ranganathan <vidya@linux.vnet.ibm.com>
>
> * string/strpbrk.c (strpbrk): Using macro to redefine symbol name.
> * sysdeps/powerpc/powerpc64/multiarch/Makefile: Add strpbrk-power7
> and strpbrk-ppc64 objects.
> * sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
> (__libc_ifunc_impl_list): Add new strpbrk optimized symbols.
> * sysdeps/powerpc/powerpc64/multiarch/strpbrk-power7.S: New file:
> multiarch strpbrk for POWER7.
> * sysdeps/powerpc/powerpc64/multiarch/strpbrk-ppc64.c: New file:
> multiarch strpbrk for PPC64.
> * sysdeps/powerpc/powerpc64/multiarch/strpbrk.c: New file: strpbrk
> ifunc selector.
> * sysdeps/powerpc/powerpc64/power7/strpbrk.S: New file: optimited
> strpbrk for POWER7.
>
> ---
>
> diff --git a/string/strpbrk.c b/string/strpbrk.c
> index ce33b68..a694242 100644
> --- a/string/strpbrk.c
> +++ b/string/strpbrk.c
> @@ -15,21 +15,17 @@
> License along with the GNU C Library; if not, see
> <http://www.gnu.org/licenses/>. */
>
> -#ifdef HAVE_CONFIG_H
> -# include <config.h>
> -#endif
> -
> -#if defined _LIBC || defined HAVE_CONFIG_H
> -# include <string.h>
> -#endif
> +#include <string.h>
>
> #undef strpbrk
>
> +#ifndef STRPBRK
> +#define STRPBRK strpbrk
> +#endif
> +
> /* Find the first occurrence in S of any character in ACCEPT. */
> char *
> -strpbrk (s, accept)
> - const char *s;
> - const char *accept;
> +STRPBRK (const char *s, const char *accept)
> {
> while (*s != '\0')
> {
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile
> index d09f2e3..a93a737 100644
> --- a/sysdeps/powerpc/powerpc64/multiarch/Makefile
> +++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile
> @@ -14,7 +14,7 @@ sysdep_routines += memcpy-power7 memcpy-a2 memcpy-power6 memcpy-cell \
> wcsrchr-ppc64 wcscpy-power7 wcscpy-power6 wcscpy-ppc64 \
> wordcopy-power7 wordcopy-power6 wordcopy-ppc64 \
> strcpy-power7 strcpy-ppc64 stpcpy-power7 stpcpy-ppc64 \
> - strrchr-power7 strrchr-ppc64
> + strrchr-power7 strrchr-ppc64 strpbrk-power7 strpbrk-ppc64
>
> CFLAGS-strncase-power7.c += -mcpu=power7 -funroll-loops
> CFLAGS-strncase_l-power7.c += -mcpu=power7 -funroll-loops
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
> index 8789483..cfba0af 100644
> --- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
> +++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
> @@ -246,5 +246,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
> IFUNC_IMPL_ADD (array, i, strrchr, 1,
> __strrchr_ppc))
>
> + /* Support sysdeps/powerpc/powerpc64/multiarch/strpbrk.c. */
> + IFUNC_IMPL (i, name, strpbrk,
> + IFUNC_IMPL_ADD (array, i, strpbrk,
> + hwcap & PPC_FEATURE_HAS_VSX,
> + __strpbrk_power7)
> + IFUNC_IMPL_ADD (array, i, strpbrk, 1,
> + __strpbrk_ppc))
> +
> return i;
> }
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/strpbrk-power7.S b/sysdeps/powerpc/powerpc64/multiarch/strpbrk-power7.S
> new file mode 100644
> index 0000000..663ca36
> --- /dev/null
> +++ b/sysdeps/powerpc/powerpc64/multiarch/strpbrk-power7.S
> @@ -0,0 +1,40 @@
> +/* Optimized strpbrk implementation for POWER7.
> + Copyright (C) 2014 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +#include <sysdep.h>
> +
> +#undef EALIGN
> +#define EALIGN(name, alignt, words) \
> + .section ".text"; \
> + ENTRY_2(__strpbrk_power7) \
> + .align ALIGNARG(alignt); \
> + EALIGN_W_##words; \
> + BODY_LABEL(__strpbrk_power7): \
> + cfi_startproc; \
> + LOCALENTRY(__strpbrk_power7)
> +
> +#undef END
> +#define END(name) \
> + cfi_endproc; \
> + TRACEBACK(__strpbrk_power7) \
> + END_2(__strpbrk_power7)
> +
> +#undef libc_hidden_builtin_def
> +#define libc_hidden_builtin_def(name)
> +
> +#include <sysdeps/powerpc/powerpc64/power7/strpbrk.S>
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/strpbrk-ppc64.c b/sysdeps/powerpc/powerpc64/multiarch/strpbrk-ppc64.c
> new file mode 100644
> index 0000000..8dea70e
> --- /dev/null
> +++ b/sysdeps/powerpc/powerpc64/multiarch/strpbrk-ppc64.c
> @@ -0,0 +1,30 @@
> +/* Copyright (C) 2014 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +#include <string.h>
> +
> +#define STRPBRK __strpbrk_ppc
> +#ifdef SHARED
> +
> +# undef libc_hidden_builtin_def
> +# define libc_hidden_builtin_def(name) \
> + __hidden_ver1 (__strpbrk_ppc, __GI_strpbrk, __strpbrk_ppc);
> +#endif
> +
> +extern __typeof (strpbrk) __strpbrk_ppc attribute_hidden;
> +
> +#include <string/strpbrk.c>
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/strpbrk.c b/sysdeps/powerpc/powerpc64/multiarch/strpbrk.c
> new file mode 100644
> index 0000000..8b05536
> --- /dev/null
> +++ b/sysdeps/powerpc/powerpc64/multiarch/strpbrk.c
> @@ -0,0 +1,31 @@
> +/* Multiple versions of strpbrk. PowerPC64 version.
> + Copyright (C) 2014 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +#ifndef NOT_IN_libc
> +# include <string.h>
> +# include <shlib-compat.h>
> +# include "init-arch.h"
> +
> +extern __typeof (strpbrk) __strpbrk_ppc attribute_hidden;
> +extern __typeof (strpbrk) __strpbrk_power7 attribute_hidden;
> +
> +libc_ifunc (strpbrk,
> + (hwcap & PPC_FEATURE_HAS_VSX)
> + ? __strpbrk_power7
> + : __strpbrk_ppc);
> +#endif
> diff --git a/sysdeps/powerpc/powerpc64/power7/strpbrk.S b/sysdeps/powerpc/powerpc64/power7/strpbrk.S
> new file mode 100644
> index 0000000..4251eeb
> --- /dev/null
> +++ b/sysdeps/powerpc/powerpc64/power7/strpbrk.S
> @@ -0,0 +1,147 @@
> +/* Optimized strpbrk implementation for PowerPC64/POWER7.
> + Copyright (C) 2014 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +#include <sysdep.h>
> +
> +/* char [r3] *strpbrk(const char [r4] *s, const char [r5] *accept) */
> +
> + .machine power7
> +EALIGN (strpbrk, 4, 0)
> + CALL_MCOUNT 3
> +
> + /* The idea to speed up the algorithm is to create a lookup table
> + for fast check if input character should be considered. For ASCII
> + or ISO-8859-X character sets it has 256 positions. */
> + lbz r10,0(r4)
> +
> + /* First the table should be cleared and to avoid unaligned accesses
> + when using the VSX stores the table address is aligned to 16
> + bytes. */
> + xxlxor v0,v0,v0
> +
> + /* PPC64 ELF ABI stack is aligned to 16 bytes */
> + addi r9,r1,-256
> +
> + li r5,16
> + li r6,32
> + li r8,48
> + cmpdi cr7,r10,0 /* accept[0] == '\0' ? */
> + addi r12,r9,64
> + /* Clear the table with 0 values */
> + stxvw4x v0,r0,r9
> + addi r11,r9,128
> + addi r7,r9,192
> + stxvw4x v0,r9,r5
> + li r0,1
> + stxvw4x v0,r9,r6
> + stxvw4x v0,r9,r8
> + stxvw4x v0,r0,r12
> + stxvw4x v0,r12,r5
> + stxvw4x v0,r12,r6
> + stxvw4x v0,r12,r8
> + stxvw4x v0,r0,r11
> + stxvw4x v0,r11,r5
> + stxvw4x v0,r11,r6
> + stxvw4x v0,r11,r8
> + stxvw4x v0,r0,r7
> + stxvw4x v0,r7,r5
> + stxvw4x v0,r7,r6
> + stxvw4x v0,r7,r8
> + beq cr7,L(finish_table) /* If accept[0] == '\0' skip */
> +
> + /* Initialize the table as:
> + for (i=0; accept[i]; i++
> + table[accept[i]]] = 1 */
> + .p2align 4,,15
> +L(init_table):
> + stbx r0,r9,r10
> + lbzu r10,1(r4)
> + cmpdi r0,r10,0
> + bne cr0,L(init_table)
> +L(finish_table):
> + /* set table[0] = 1 */
> + li r4,1
> + stb r4,0(r9)
> + b L(mainloop)
> +
> + /* Unrool the loop 4 times and check using the table as:
> + i = 0;
> + while (1)
> + {
> + if (table[input[i++]] == 1)
> + return (s[i -1] ? s + i - 1: NULL);
> + if (table[input[i++]] == 1)
> + return (s[i -1] ? s + i - 1: NULL);
> + if (table[input[i++]] == 1)
> + return (s[i -1] ? s + i - 1: NULL);
> + if (table[input[i++]] == 1)
> + return (s[i -1] ? s + i - 1: NULL);
> + } */
> + .p2align 4,,15
> +L(unroll):
> + lbz r0,1(r3)
> + lbzx r8,r9,r0
> + cmpwi cr6,r8,1
> + beq cr6,L(checkend2)
> + lbz r10,2(r3)
> + lbzx r4,r9,r10
> + cmpwi cr7,r4,1
> + beq cr7,L(checkend3)
> + lbz r12,3(r3)
> + addi r3,r3,4
> + lbzx r11,r9,r12
> + cmpwi cr0,r11,1
> + beq cr0,L(checkend)
> +L(mainloop):
> + lbz r12,0(r3)
> + addi r11,r3,1
> + addi r5,r3,2
> + addi r7,r3,3
> + lbzx r6,r9,r12
> + cmpwi cr1,r6,1
> + bne cr1,L(unroll)
> + cmpdi cr0,r12,0
> + beq cr0,L(nullfound)
> +L(end):
> + blr
> +
> + .p2align 4,,15
> +L(checkend):
> + cmpdi cr1,r12,0
> + mr r3,r7
> + bne cr1,L(end)
> +L(nullfound):
> + /* return NULL */
> + li 3,0
> + blr
> +
> + .p2align 4,,15
> +L(checkend2):
> + cmpdi cr7,r0,0
> + mr r3,r11
> + beq cr7,L(nullfound)
> + blr
> +
> + .p2align 4,,15
> +L(checkend3):
> + cmpdi cr6,r10,0
> + mr r3,r5
> + beq cr6,L(nullfound)
> + blr
> +END (strpbrk)
> +libc_hidden_builtin_def (strpbrk)
>
>From 1b1c4f53b8225895ea0558d7f85d36a1ee554d32 Mon Sep 17 00:00:00 2001
From: Vidya Ranganathan <vidya@linux.vnet.ibm.com>
Date: Tue, 11 Mar 2014 06:22:21 -0400
Subject: [PATCH] Optimization for PowerPC on strpbrk. I have made a sanity
check for needle for early exit.
This patch add an optimized strpbrk for POWER7 by using a different
algorithm than default implementation: it constructs a table based on
the 'accept' argument and use this table to check for any occurance on
the input string. The idea is similar as x86_64 uses.
For PowerPC some tunings were added, such as unroll loops and memory
clear using VSX instructions.
Signed-off-by: Vidya Ranganathan <vidya@linux.vnet.ibm.com>
---
string/strpbrk.c | 6 +-
sysdeps/powerpc/powerpc64/multiarch/Makefile | 2 +-
.../powerpc/powerpc64/multiarch/ifunc-impl-list.c | 8 +
.../powerpc/powerpc64/multiarch/strpbrk-power7.S | 40 +++++
.../powerpc/powerpc64/multiarch/strpbrk-ppc64.c | 30 ++++
sysdeps/powerpc/powerpc64/multiarch/strpbrk.c | 31 ++++
sysdeps/powerpc/powerpc64/power7/strpbrk.S | 161 +++++++++++++++++++++
7 files changed, 276 insertions(+), 2 deletions(-)
create mode 100644 sysdeps/powerpc/powerpc64/multiarch/strpbrk-power7.S
create mode 100644 sysdeps/powerpc/powerpc64/multiarch/strpbrk-ppc64.c
create mode 100644 sysdeps/powerpc/powerpc64/multiarch/strpbrk.c
create mode 100644 sysdeps/powerpc/powerpc64/power7/strpbrk.S
@@ -25,9 +25,13 @@
#undef strpbrk
+#ifndef STRPBRK
+#define STRPBRK strpbrk
+#endif
+
/* Find the first occurrence in S of any character in ACCEPT. */
char *
-strpbrk (s, accept)
+STRPBRK (s, accept)
const char *s;
const char *accept;
{
@@ -15,7 +15,7 @@ sysdep_routines += memcpy-power7 memcpy-a2 memcpy-power6 memcpy-cell \
wordcopy-power7 wordcopy-power6 wordcopy-ppc64 \
strcpy-power7 strcpy-ppc64 stpcpy-power7 stpcpy-ppc64 \
strrchr-power7 strrchr-ppc64 strncat-power7 strncat-ppc64 \
- strspn-power7 strspn-ppc64
+ strspn-power7 strspn-ppc64 strpbrk-power7 strpbrk-ppc64
CFLAGS-strncase-power7.c += -mcpu=power7 -funroll-loops
CFLAGS-strncase_l-power7.c += -mcpu=power7 -funroll-loops
@@ -262,5 +262,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, strspn, 1,
__strspn_ppc))
+ /* Support sysdeps/powerpc/powerpc64/multiarch/strpbrk.c. */
+ IFUNC_IMPL (i, name, strpbrk,
+ IFUNC_IMPL_ADD (array, i, strpbrk,
+ hwcap & PPC_FEATURE_HAS_VSX,
+ __strpbrk_power7)
+ IFUNC_IMPL_ADD (array, i, strpbrk, 1,
+ __strpbrk_ppc))
+
return i;
}
new file mode 100644
@@ -0,0 +1,40 @@
+/* Optimized strpbrk implementation for POWER7.
+ Copyright (C) 2013-2014 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+#undef EALIGN
+#define EALIGN(name, alignt, words) \
+ .section ".text"; \
+ ENTRY_2(__strpbrk_power7) \
+ .align ALIGNARG(alignt); \
+ EALIGN_W_##words; \
+ BODY_LABEL(__strpbrk_power7): \
+ cfi_startproc; \
+ LOCALENTRY(__strpbrk_power7)
+
+#undef END
+#define END(name) \
+ cfi_endproc; \
+ TRACEBACK(__strpbrk_power7) \
+ END_2(__strpbrk_power7)
+
+#undef libc_hidden_builtin_def
+#define libc_hidden_builtin_def(name)
+
+#include <sysdeps/powerpc/powerpc64/power7/strpbrk.S>
new file mode 100644
@@ -0,0 +1,30 @@
+/* Copyright (C) 2014 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/ >. */
+
+#include <string.h>
+
+#define STRPBRK __strpbrk_ppc
+#ifdef SHARED
+
+# undef libc_hidden_builtin_def
+# define libc_hidden_builtin_def(name) \
+ __hidden_ver1 (__strpbrk_ppc, __GI_strpbrk, __strpbrk_ppc);
+#endif
+
+extern __typeof (strpbrk) __strpbrk_ppc attribute_hidden;
+
+#include <string/strpbrk.c>
new file mode 100644
@@ -0,0 +1,31 @@
+/* Multiple versions of strpbrk. PowerPC64 version.
+ Copyright (C) 2013-2014 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef NOT_IN_libc
+# include <string.h>
+# include <shlib-compat.h>
+# include "init-arch.h"
+
+extern __typeof (strpbrk) __strpbrk_ppc attribute_hidden;
+extern __typeof (strpbrk) __strpbrk_power7 attribute_hidden;
+
+libc_ifunc (strpbrk,
+ (hwcap & PPC_FEATURE_HAS_VSX)
+ ? __strpbrk_power7
+ : __strpbrk_ppc);
+#endif
new file mode 100644
@@ -0,0 +1,161 @@
+/* Optimized strpbrk implementation for PowerPC64/POWER7.
+ Copyright (C) 2014 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+/* char [r3] *strpbrk(const char [r4] *s, const char [r5] *accept) */
+
+
+/* Performance gains are grabbed through following techniques:
+ 1. A hash table/dictionary is created comprising
+ of 256 ascii character set.
+ 2. hash table to initialized to ONE when pivot for needle
+ is found at index for character in input string.
+ 3. if marker is found in input string on scanning
+ the string at that index is returned.
+ 4. if input string does not find a match in hash,
+ NULL is returned. */
+
+ .machine power7
+EALIGN (strpbrk, 4, 0)
+ CALL_MCOUNT 3
+
+ /* The idea to speed up the algorithm is to create a lookup table
+ for fast check if input character should be considered. For ASCII
+ or ISO-8859-X character sets it has 256 positions. */
+ lbz r10,0(r4)
+
+ cmpdi cr7,r10,0 /* accept[0] == '\0' ? */
+ beq cr7,L(nullfound)
+
+ /* First the table should be cleared and to avoid unaligned accesses
+ when using the VSX stores the table address is aligned to 16
+ bytes. */
+ xxlxor v0,v0,v0
+
+ /* PPC64 ELF ABI stack is aligned to 16 bytes */
+ addi r9,r1,-256
+
+ li r5,16 /* place holder for offset=16 */
+ li r6,32 /* place holder for offset=32 */
+ li r8,48 /* place holder for offset=48 */
+
+ addi r12,r9,64 /* index to 2nd quadword */
+ addi r11,r9,128 /* index to 3rd quadword */
+ addi r7,r9,192 /* index to 4th quadword */
+
+ /* Clear the table with 0 values */
+ stxvw4x v0,r0,r9
+ stxvw4x v0,r9,r5
+ li r0,1
+ stxvw4x v0,r9,r6
+ stxvw4x v0,r9,r8
+ stxvw4x v0,r0,r12
+ stxvw4x v0,r12,r5
+ stxvw4x v0,r12,r6
+ stxvw4x v0,r12,r8
+ stxvw4x v0,r0,r11
+ stxvw4x v0,r11,r5
+ stxvw4x v0,r11,r6
+ stxvw4x v0,r11,r8
+ stxvw4x v0,r0,r7
+ stxvw4x v0,r7,r5
+ stxvw4x v0,r7,r6
+ stxvw4x v0,r7,r8
+
+ /* Initialize the table as:
+ for (i=0; accept[i]; i++
+ table[accept[i]]] = 1 */
+ .p2align 4
+L(init_table):
+ stbx r0,r9,r10 /* update hash for needle pivot */
+ lbzu r10,1(r4) /* load needle into r10 and update */
+ cmpdi r0,r10,0 /* if needle is NULL, continue */
+ bne cr0,L(init_table) /* loop to hash the needle */
+L(finish_table):
+ /* set table[0] = 1 */
+ li r4,1
+ stb r4,0(r9)
+ b L(mainloop)
+
+ /* Unrool the loop 4 times and check using the table as:
+ i = 0;
+ while (1)
+ {
+ if (table[input[i++]] == 1)
+ return (s[i -1] ? s + i - 1: NULL);
+ if (table[input[i++]] == 1)
+ return (s[i -1] ? s + i - 1: NULL);
+ if (table[input[i++]] == 1)
+ return (s[i -1] ? s + i - 1: NULL);
+ if (table[input[i++]] == 1)
+ return (s[i -1] ? s + i - 1: NULL);
+ } */
+ .p2align 4
+L(unroll):
+ lbz r0,1(r3) /* load string[1] */
+ lbzx r8,r9,r0 /* load r8 with hash value at index */
+ cmpwi cr6,r8,1 /* we hit marker in hash, found needle */
+ beq cr6,L(checkend2)
+
+ lbz r10,2(r3) /* load string[2] */
+ lbzx r4,r9,r10 /* load r9 with hash value at index */
+ cmpwi cr7,r4,1 /* we hit marker in hash, found needle */
+ beq cr7,L(checkend3)
+
+ lbz r12,3(r3) /* load string[3] */
+ addi r3,r3,4 /* increment string by unroll factor */
+ lbzx r11,r9,r12 /* load r11 with hash value at index */
+ cmpwi cr0,r11,1 /* we hit marker in hash, found needle */
+ beq cr0,L(checkend)
+L(mainloop):
+ lbz r12,0(r3) /* r12 is holder for string[0] from r3 */
+ addi r11,r3,1 /* place holder ;increment counter by 1 */
+ addi r5,r3,2 /* place holder ;increment counter by 2 */
+ addi r7,r3,3 /* place holder ;increment counter by 3 */
+ lbzx r6,r9,r12 /* load r6 with hash value at index */
+ cmpwi cr1,r6,1 /* we hit marker in hash, found needle */
+ bne cr1,L(unroll) /* continue scanning */
+ cmpdi cr0,r12,0
+ beq cr0,L(nullfound)
+L(end):
+ blr
+
+L(checkend):
+ cmpdi cr1,r12,0
+ mr r3,r7
+ bne cr1,L(end)
+
+L(nullfound):
+ /* return NULL */
+ li 3,0
+ blr
+
+L(checkend2):
+ cmpdi cr7,r0,0
+ mr r3,r11
+ beq cr7,L(nullfound)
+ blr
+
+L(checkend3):
+ cmpdi cr6,r10,0
+ mr r3,r5
+ beq cr6,L(nullfound)
+ blr
+END (strpbrk)
+libc_hidden_builtin_def (strpbrk)
--
1.8.3.1