[v1] x86: Move strrchr SSE2 implementation to multiarch/strrchr-sse2.S
Checks
Context |
Check |
Description |
dj/TryBot-apply_patch |
success
|
Patch applied to master at the time it was sent
|
dj/TryBot-32bit |
success
|
Build for i686
|
Commit Message
This commit doesn't affect libc.so.6, its just housekeeping to prepare
for adding explicit ISA level support.
Tested build on x86_64 and x86_32 with/without multiarch.
---
sysdeps/x86_64/multiarch/strrchr-sse2.S | 358 ++++++++++++++++++++++-
sysdeps/x86_64/multiarch/wcsrchr-sse2.S | 10 +-
sysdeps/x86_64/strrchr.S | 364 +-----------------------
sysdeps/x86_64/wcsrchr.S | 11 +-
4 files changed, 366 insertions(+), 377 deletions(-)
Comments
On Tue, Jul 12, 2022 at 12:29 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> This commit doesn't affect libc.so.6, its just housekeeping to prepare
> for adding explicit ISA level support.
>
> Tested build on x86_64 and x86_32 with/without multiarch.
> ---
> sysdeps/x86_64/multiarch/strrchr-sse2.S | 358 ++++++++++++++++++++++-
> sysdeps/x86_64/multiarch/wcsrchr-sse2.S | 10 +-
> sysdeps/x86_64/strrchr.S | 364 +-----------------------
> sysdeps/x86_64/wcsrchr.S | 11 +-
> 4 files changed, 366 insertions(+), 377 deletions(-)
>
> diff --git a/sysdeps/x86_64/multiarch/strrchr-sse2.S b/sysdeps/x86_64/multiarch/strrchr-sse2.S
> index 866396e947..6ee7a5e33a 100644
> --- a/sysdeps/x86_64/multiarch/strrchr-sse2.S
> +++ b/sysdeps/x86_64/multiarch/strrchr-sse2.S
> @@ -17,12 +17,358 @@
> <https://www.gnu.org/licenses/>. */
>
> #if IS_IN (libc)
> -# define STRRCHR __strrchr_sse2
> +# ifndef STRRCHR
> +# define STRRCHR __strrchr_sse2
> +# endif
> +#endif
> +
> +#include <sysdep.h>
> +
> +#ifdef USE_AS_WCSRCHR
> +# define PCMPEQ pcmpeqd
> +# define CHAR_SIZE 4
> +# define PMINU pminud
> +#else
> +# define PCMPEQ pcmpeqb
> +# define CHAR_SIZE 1
> +# define PMINU pminub
> +#endif
> +
> +#define PAGE_SIZE 4096
> +#define VEC_SIZE 16
> +
> + .text
> +ENTRY(STRRCHR)
> + movd %esi, %xmm0
> + movq %rdi, %rax
> + andl $(PAGE_SIZE - 1), %eax
> +#ifndef USE_AS_WCSRCHR
> + punpcklbw %xmm0, %xmm0
> + punpcklwd %xmm0, %xmm0
> +#endif
> + pshufd $0, %xmm0, %xmm0
> + cmpl $(PAGE_SIZE - VEC_SIZE), %eax
> + ja L(cross_page)
> +
> +L(cross_page_continue):
> + movups (%rdi), %xmm1
> + pxor %xmm2, %xmm2
> + PCMPEQ %xmm1, %xmm2
> + pmovmskb %xmm2, %ecx
> + testl %ecx, %ecx
> + jz L(aligned_more)
> +
> + PCMPEQ %xmm0, %xmm1
> + pmovmskb %xmm1, %eax
> + leal -1(%rcx), %edx
> + xorl %edx, %ecx
> + andl %ecx, %eax
> + jz L(ret0)
> + bsrl %eax, %eax
> + addq %rdi, %rax
> + /* We are off by 3 for wcsrchr if search CHAR is non-zero. If
> + search CHAR is zero we are correct. Either way `andq
> + -CHAR_SIZE, %rax` gets the correct result. */
> +#ifdef USE_AS_WCSRCHR
> + andq $-CHAR_SIZE, %rax
> +#endif
> +L(ret0):
> + ret
> +
> + /* Returns for first vec x1/x2 have hard coded backward search
> + path for earlier matches. */
> + .p2align 4
> +L(first_vec_x0_test):
> + PCMPEQ %xmm0, %xmm1
> + pmovmskb %xmm1, %eax
> + testl %eax, %eax
> + jz L(ret0)
> + bsrl %eax, %eax
> + addq %r8, %rax
> +#ifdef USE_AS_WCSRCHR
> + andq $-CHAR_SIZE, %rax
> +#endif
> + ret
> +
> + .p2align 4
> +L(first_vec_x1):
> + PCMPEQ %xmm0, %xmm2
> + pmovmskb %xmm2, %eax
> + leal -1(%rcx), %edx
> + xorl %edx, %ecx
> + andl %ecx, %eax
> + jz L(first_vec_x0_test)
> + bsrl %eax, %eax
> + leaq (VEC_SIZE)(%rdi, %rax), %rax
> +#ifdef USE_AS_WCSRCHR
> + andq $-CHAR_SIZE, %rax
> +#endif
> + ret
> +
> + .p2align 4
> +L(first_vec_x1_test):
> + PCMPEQ %xmm0, %xmm2
> + pmovmskb %xmm2, %eax
> + testl %eax, %eax
> + jz L(first_vec_x0_test)
> + bsrl %eax, %eax
> + leaq (VEC_SIZE)(%rdi, %rax), %rax
> +#ifdef USE_AS_WCSRCHR
> + andq $-CHAR_SIZE, %rax
> +#endif
> + ret
> +
> + .p2align 4
> +L(first_vec_x2):
> + PCMPEQ %xmm0, %xmm3
> + pmovmskb %xmm3, %eax
> + leal -1(%rcx), %edx
> + xorl %edx, %ecx
> + andl %ecx, %eax
> + jz L(first_vec_x1_test)
> + bsrl %eax, %eax
> + leaq (VEC_SIZE * 2)(%rdi, %rax), %rax
> +#ifdef USE_AS_WCSRCHR
> + andq $-CHAR_SIZE, %rax
> +#endif
> + ret
> +
> + .p2align 4
> +L(aligned_more):
> + /* Save original pointer if match was in VEC 0. */
> + movq %rdi, %r8
> + andq $-VEC_SIZE, %rdi
> +
> + movaps VEC_SIZE(%rdi), %xmm2
> + pxor %xmm3, %xmm3
> + PCMPEQ %xmm2, %xmm3
> + pmovmskb %xmm3, %ecx
> + testl %ecx, %ecx
> + jnz L(first_vec_x1)
> +
> + movaps (VEC_SIZE * 2)(%rdi), %xmm3
> + pxor %xmm4, %xmm4
> + PCMPEQ %xmm3, %xmm4
> + pmovmskb %xmm4, %ecx
> + testl %ecx, %ecx
> + jnz L(first_vec_x2)
> +
> + addq $VEC_SIZE, %rdi
> + /* Save pointer again before realigning. */
> + movq %rdi, %rsi
> + andq $-(VEC_SIZE * 2), %rdi
> + .p2align 4
> +L(first_loop):
> + /* Do 2x VEC at a time. */
> + movaps (VEC_SIZE * 2)(%rdi), %xmm4
> + movaps (VEC_SIZE * 3)(%rdi), %xmm5
> + /* Since SSE2 no pminud so wcsrchr needs seperate logic for
> + detecting zero. Note if this is found to be a bottleneck it
> + may be worth adding an SSE4.1 wcsrchr implementation. */
> +#ifdef USE_AS_WCSRCHR
> + movaps %xmm5, %xmm6
> + pxor %xmm8, %xmm8
> +
> + PCMPEQ %xmm8, %xmm5
> + PCMPEQ %xmm4, %xmm8
> + por %xmm5, %xmm8
> +#else
> + movaps %xmm5, %xmm6
> + PMINU %xmm4, %xmm5
> +#endif
> +
> + movaps %xmm4, %xmm9
> + PCMPEQ %xmm0, %xmm4
> + PCMPEQ %xmm0, %xmm6
> + movaps %xmm6, %xmm7
> + por %xmm4, %xmm6
> +#ifndef USE_AS_WCSRCHR
> + pxor %xmm8, %xmm8
> + PCMPEQ %xmm5, %xmm8
> +#endif
> + pmovmskb %xmm8, %ecx
> + pmovmskb %xmm6, %eax
>
> -# undef weak_alias
> -# define weak_alias(strrchr, rindex)
> -# undef libc_hidden_builtin_def
> -# define libc_hidden_builtin_def(strrchr)
> + addq $(VEC_SIZE * 2), %rdi
> + /* Use `addl` 1) so we can undo it with `subl` and 2) it can
> + macro-fuse with `jz`. */
> + addl %ecx, %eax
> + jz L(first_loop)
> +
> + /* Check if there is zero match. */
> + testl %ecx, %ecx
> + jz L(second_loop_match)
> +
> + /* Check if there was a match in last iteration. */
> + subl %ecx, %eax
> + jnz L(new_match)
> +
> +L(first_loop_old_match):
> + PCMPEQ %xmm0, %xmm2
> + PCMPEQ %xmm0, %xmm3
> + pmovmskb %xmm2, %ecx
> + pmovmskb %xmm3, %eax
> + addl %eax, %ecx
> + jz L(first_vec_x0_test)
> + /* NB: We could move this shift to before the branch and save a
> + bit of code size / performance on the fall through. The
> + branch leads to the null case which generally seems hotter
> + than char in first 3x VEC. */
> + sall $16, %eax
> + orl %ecx, %eax
> +
> + bsrl %eax, %eax
> + addq %rsi, %rax
> +#ifdef USE_AS_WCSRCHR
> + andq $-CHAR_SIZE, %rax
> +#endif
> + ret
> +
> + .p2align 4
> +L(new_match):
> + pxor %xmm6, %xmm6
> + PCMPEQ %xmm9, %xmm6
> + pmovmskb %xmm6, %eax
> + sall $16, %ecx
> + orl %eax, %ecx
> +
> + /* We can't reuse either of the old comparisons as since we mask
> + of zeros after first zero (instead of using the full
> + comparison) we can't gurantee no interference between match
> + after end of string and valid match. */
> + pmovmskb %xmm4, %eax
> + pmovmskb %xmm7, %edx
> + sall $16, %edx
> + orl %edx, %eax
> +
> + leal -1(%ecx), %edx
> + xorl %edx, %ecx
> + andl %ecx, %eax
> + jz L(first_loop_old_match)
> + bsrl %eax, %eax
> + addq %rdi, %rax
> +#ifdef USE_AS_WCSRCHR
> + andq $-CHAR_SIZE, %rax
> +#endif
> + ret
> +
> + /* Save minimum state for getting most recent match. We can
> + throw out all previous work. */
> + .p2align 4
> +L(second_loop_match):
> + movq %rdi, %rsi
> + movaps %xmm4, %xmm2
> + movaps %xmm7, %xmm3
> +
> + .p2align 4
> +L(second_loop):
> + movaps (VEC_SIZE * 2)(%rdi), %xmm4
> + movaps (VEC_SIZE * 3)(%rdi), %xmm5
> + /* Since SSE2 no pminud so wcsrchr needs seperate logic for
> + detecting zero. Note if this is found to be a bottleneck it
> + may be worth adding an SSE4.1 wcsrchr implementation. */
> +#ifdef USE_AS_WCSRCHR
> + movaps %xmm5, %xmm6
> + pxor %xmm8, %xmm8
> +
> + PCMPEQ %xmm8, %xmm5
> + PCMPEQ %xmm4, %xmm8
> + por %xmm5, %xmm8
> +#else
> + movaps %xmm5, %xmm6
> + PMINU %xmm4, %xmm5
> +#endif
> +
> + movaps %xmm4, %xmm9
> + PCMPEQ %xmm0, %xmm4
> + PCMPEQ %xmm0, %xmm6
> + movaps %xmm6, %xmm7
> + por %xmm4, %xmm6
> +#ifndef USE_AS_WCSRCHR
> + pxor %xmm8, %xmm8
> + PCMPEQ %xmm5, %xmm8
> #endif
>
> -#include "../strrchr.S"
> + pmovmskb %xmm8, %ecx
> + pmovmskb %xmm6, %eax
> +
> + addq $(VEC_SIZE * 2), %rdi
> + /* Either null term or new occurence of CHAR. */
> + addl %ecx, %eax
> + jz L(second_loop)
> +
> + /* No null term so much be new occurence of CHAR. */
> + testl %ecx, %ecx
> + jz L(second_loop_match)
> +
> +
> + subl %ecx, %eax
> + jnz L(second_loop_new_match)
> +
> +L(second_loop_old_match):
> + pmovmskb %xmm2, %ecx
> + pmovmskb %xmm3, %eax
> + sall $16, %eax
> + orl %ecx, %eax
> + bsrl %eax, %eax
> + addq %rsi, %rax
> +#ifdef USE_AS_WCSRCHR
> + andq $-CHAR_SIZE, %rax
> +#endif
> + ret
> +
> + .p2align 4
> +L(second_loop_new_match):
> + pxor %xmm6, %xmm6
> + PCMPEQ %xmm9, %xmm6
> + pmovmskb %xmm6, %eax
> + sall $16, %ecx
> + orl %eax, %ecx
> +
> + /* We can't reuse either of the old comparisons as since we mask
> + of zeros after first zero (instead of using the full
> + comparison) we can't gurantee no interference between match
> + after end of string and valid match. */
> + pmovmskb %xmm4, %eax
> + pmovmskb %xmm7, %edx
> + sall $16, %edx
> + orl %edx, %eax
> +
> + leal -1(%ecx), %edx
> + xorl %edx, %ecx
> + andl %ecx, %eax
> + jz L(second_loop_old_match)
> + bsrl %eax, %eax
> + addq %rdi, %rax
> +#ifdef USE_AS_WCSRCHR
> + andq $-CHAR_SIZE, %rax
> +#endif
> + ret
> +
> + .p2align 4,, 4
> +L(cross_page):
> + movq %rdi, %rsi
> + andq $-VEC_SIZE, %rsi
> + movaps (%rsi), %xmm1
> + pxor %xmm2, %xmm2
> + PCMPEQ %xmm1, %xmm2
> + pmovmskb %xmm2, %edx
> + movl %edi, %ecx
> + andl $(VEC_SIZE - 1), %ecx
> + sarl %cl, %edx
> + jz L(cross_page_continue)
> + PCMPEQ %xmm0, %xmm1
> + pmovmskb %xmm1, %eax
> + sarl %cl, %eax
> + leal -1(%rdx), %ecx
> + xorl %edx, %ecx
> + andl %ecx, %eax
> + jz L(ret1)
> + bsrl %eax, %eax
> + addq %rdi, %rax
> +#ifdef USE_AS_WCSRCHR
> + andq $-CHAR_SIZE, %rax
> +#endif
> +L(ret1):
> + ret
> +END(STRRCHR)
> diff --git a/sysdeps/x86_64/multiarch/wcsrchr-sse2.S b/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
> index 69d2f3cdb1..d9259720f8 100644
> --- a/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
> +++ b/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
> @@ -17,6 +17,12 @@
> <https://www.gnu.org/licenses/>. */
>
> #if IS_IN (libc)
> -# define STRRCHR __wcsrchr_sse2
> +# ifndef STRRCHR
> +# define STRRCHR __wcsrchr_sse2
> +# endif
> #endif
> -#include "../wcsrchr.S"
> +
> +#define USE_AS_WCSRCHR 1
> +#define NO_PMINU 1
> +
> +#include "strrchr-sse2.S"
> diff --git a/sysdeps/x86_64/strrchr.S b/sysdeps/x86_64/strrchr.S
> index 4d7ba4ceb2..f39da60454 100644
> --- a/sysdeps/x86_64/strrchr.S
> +++ b/sysdeps/x86_64/strrchr.S
> @@ -16,363 +16,7 @@
> License along with the GNU C Library; if not, see
> <https://www.gnu.org/licenses/>. */
>
> -
> -#include <sysdep.h>
> -
> -#ifndef STRRCHR
> -# define STRRCHR strrchr
> -#endif
> -
> -#ifdef USE_AS_WCSRCHR
> -# define PCMPEQ pcmpeqd
> -# define CHAR_SIZE 4
> -# define PMINU pminud
> -#else
> -# define PCMPEQ pcmpeqb
> -# define CHAR_SIZE 1
> -# define PMINU pminub
> -#endif
> -
> -#define PAGE_SIZE 4096
> -#define VEC_SIZE 16
> -
> - .text
> -ENTRY(STRRCHR)
> - movd %esi, %xmm0
> - movq %rdi, %rax
> - andl $(PAGE_SIZE - 1), %eax
> -#ifndef USE_AS_WCSRCHR
> - punpcklbw %xmm0, %xmm0
> - punpcklwd %xmm0, %xmm0
> -#endif
> - pshufd $0, %xmm0, %xmm0
> - cmpl $(PAGE_SIZE - VEC_SIZE), %eax
> - ja L(cross_page)
> -
> -L(cross_page_continue):
> - movups (%rdi), %xmm1
> - pxor %xmm2, %xmm2
> - PCMPEQ %xmm1, %xmm2
> - pmovmskb %xmm2, %ecx
> - testl %ecx, %ecx
> - jz L(aligned_more)
> -
> - PCMPEQ %xmm0, %xmm1
> - pmovmskb %xmm1, %eax
> - leal -1(%rcx), %edx
> - xorl %edx, %ecx
> - andl %ecx, %eax
> - jz L(ret0)
> - bsrl %eax, %eax
> - addq %rdi, %rax
> - /* We are off by 3 for wcsrchr if search CHAR is non-zero. If
> - search CHAR is zero we are correct. Either way `andq
> - -CHAR_SIZE, %rax` gets the correct result. */
> -#ifdef USE_AS_WCSRCHR
> - andq $-CHAR_SIZE, %rax
> -#endif
> -L(ret0):
> - ret
> -
> - /* Returns for first vec x1/x2 have hard coded backward search
> - path for earlier matches. */
> - .p2align 4
> -L(first_vec_x0_test):
> - PCMPEQ %xmm0, %xmm1
> - pmovmskb %xmm1, %eax
> - testl %eax, %eax
> - jz L(ret0)
> - bsrl %eax, %eax
> - addq %r8, %rax
> -#ifdef USE_AS_WCSRCHR
> - andq $-CHAR_SIZE, %rax
> -#endif
> - ret
> -
> - .p2align 4
> -L(first_vec_x1):
> - PCMPEQ %xmm0, %xmm2
> - pmovmskb %xmm2, %eax
> - leal -1(%rcx), %edx
> - xorl %edx, %ecx
> - andl %ecx, %eax
> - jz L(first_vec_x0_test)
> - bsrl %eax, %eax
> - leaq (VEC_SIZE)(%rdi, %rax), %rax
> -#ifdef USE_AS_WCSRCHR
> - andq $-CHAR_SIZE, %rax
> -#endif
> - ret
> -
> - .p2align 4
> -L(first_vec_x1_test):
> - PCMPEQ %xmm0, %xmm2
> - pmovmskb %xmm2, %eax
> - testl %eax, %eax
> - jz L(first_vec_x0_test)
> - bsrl %eax, %eax
> - leaq (VEC_SIZE)(%rdi, %rax), %rax
> -#ifdef USE_AS_WCSRCHR
> - andq $-CHAR_SIZE, %rax
> -#endif
> - ret
> -
> - .p2align 4
> -L(first_vec_x2):
> - PCMPEQ %xmm0, %xmm3
> - pmovmskb %xmm3, %eax
> - leal -1(%rcx), %edx
> - xorl %edx, %ecx
> - andl %ecx, %eax
> - jz L(first_vec_x1_test)
> - bsrl %eax, %eax
> - leaq (VEC_SIZE * 2)(%rdi, %rax), %rax
> -#ifdef USE_AS_WCSRCHR
> - andq $-CHAR_SIZE, %rax
> -#endif
> - ret
> -
> - .p2align 4
> -L(aligned_more):
> - /* Save original pointer if match was in VEC 0. */
> - movq %rdi, %r8
> - andq $-VEC_SIZE, %rdi
> -
> - movaps VEC_SIZE(%rdi), %xmm2
> - pxor %xmm3, %xmm3
> - PCMPEQ %xmm2, %xmm3
> - pmovmskb %xmm3, %ecx
> - testl %ecx, %ecx
> - jnz L(first_vec_x1)
> -
> - movaps (VEC_SIZE * 2)(%rdi), %xmm3
> - pxor %xmm4, %xmm4
> - PCMPEQ %xmm3, %xmm4
> - pmovmskb %xmm4, %ecx
> - testl %ecx, %ecx
> - jnz L(first_vec_x2)
> -
> - addq $VEC_SIZE, %rdi
> - /* Save pointer again before realigning. */
> - movq %rdi, %rsi
> - andq $-(VEC_SIZE * 2), %rdi
> - .p2align 4
> -L(first_loop):
> - /* Do 2x VEC at a time. */
> - movaps (VEC_SIZE * 2)(%rdi), %xmm4
> - movaps (VEC_SIZE * 3)(%rdi), %xmm5
> - /* Since SSE2 no pminud so wcsrchr needs seperate logic for
> - detecting zero. Note if this is found to be a bottleneck it
> - may be worth adding an SSE4.1 wcsrchr implementation. */
> -#ifdef USE_AS_WCSRCHR
> - movaps %xmm5, %xmm6
> - pxor %xmm8, %xmm8
> -
> - PCMPEQ %xmm8, %xmm5
> - PCMPEQ %xmm4, %xmm8
> - por %xmm5, %xmm8
> -#else
> - movaps %xmm5, %xmm6
> - PMINU %xmm4, %xmm5
> -#endif
> -
> - movaps %xmm4, %xmm9
> - PCMPEQ %xmm0, %xmm4
> - PCMPEQ %xmm0, %xmm6
> - movaps %xmm6, %xmm7
> - por %xmm4, %xmm6
> -#ifndef USE_AS_WCSRCHR
> - pxor %xmm8, %xmm8
> - PCMPEQ %xmm5, %xmm8
> -#endif
> - pmovmskb %xmm8, %ecx
> - pmovmskb %xmm6, %eax
> -
> - addq $(VEC_SIZE * 2), %rdi
> - /* Use `addl` 1) so we can undo it with `subl` and 2) it can
> - macro-fuse with `jz`. */
> - addl %ecx, %eax
> - jz L(first_loop)
> -
> - /* Check if there is zero match. */
> - testl %ecx, %ecx
> - jz L(second_loop_match)
> -
> - /* Check if there was a match in last iteration. */
> - subl %ecx, %eax
> - jnz L(new_match)
> -
> -L(first_loop_old_match):
> - PCMPEQ %xmm0, %xmm2
> - PCMPEQ %xmm0, %xmm3
> - pmovmskb %xmm2, %ecx
> - pmovmskb %xmm3, %eax
> - addl %eax, %ecx
> - jz L(first_vec_x0_test)
> - /* NB: We could move this shift to before the branch and save a
> - bit of code size / performance on the fall through. The
> - branch leads to the null case which generally seems hotter
> - than char in first 3x VEC. */
> - sall $16, %eax
> - orl %ecx, %eax
> -
> - bsrl %eax, %eax
> - addq %rsi, %rax
> -#ifdef USE_AS_WCSRCHR
> - andq $-CHAR_SIZE, %rax
> -#endif
> - ret
> -
> - .p2align 4
> -L(new_match):
> - pxor %xmm6, %xmm6
> - PCMPEQ %xmm9, %xmm6
> - pmovmskb %xmm6, %eax
> - sall $16, %ecx
> - orl %eax, %ecx
> -
> - /* We can't reuse either of the old comparisons as since we mask
> - of zeros after first zero (instead of using the full
> - comparison) we can't gurantee no interference between match
> - after end of string and valid match. */
> - pmovmskb %xmm4, %eax
> - pmovmskb %xmm7, %edx
> - sall $16, %edx
> - orl %edx, %eax
> -
> - leal -1(%ecx), %edx
> - xorl %edx, %ecx
> - andl %ecx, %eax
> - jz L(first_loop_old_match)
> - bsrl %eax, %eax
> - addq %rdi, %rax
> -#ifdef USE_AS_WCSRCHR
> - andq $-CHAR_SIZE, %rax
> -#endif
> - ret
> -
> - /* Save minimum state for getting most recent match. We can
> - throw out all previous work. */
> - .p2align 4
> -L(second_loop_match):
> - movq %rdi, %rsi
> - movaps %xmm4, %xmm2
> - movaps %xmm7, %xmm3
> -
> - .p2align 4
> -L(second_loop):
> - movaps (VEC_SIZE * 2)(%rdi), %xmm4
> - movaps (VEC_SIZE * 3)(%rdi), %xmm5
> - /* Since SSE2 no pminud so wcsrchr needs seperate logic for
> - detecting zero. Note if this is found to be a bottleneck it
> - may be worth adding an SSE4.1 wcsrchr implementation. */
> -#ifdef USE_AS_WCSRCHR
> - movaps %xmm5, %xmm6
> - pxor %xmm8, %xmm8
> -
> - PCMPEQ %xmm8, %xmm5
> - PCMPEQ %xmm4, %xmm8
> - por %xmm5, %xmm8
> -#else
> - movaps %xmm5, %xmm6
> - PMINU %xmm4, %xmm5
> -#endif
> -
> - movaps %xmm4, %xmm9
> - PCMPEQ %xmm0, %xmm4
> - PCMPEQ %xmm0, %xmm6
> - movaps %xmm6, %xmm7
> - por %xmm4, %xmm6
> -#ifndef USE_AS_WCSRCHR
> - pxor %xmm8, %xmm8
> - PCMPEQ %xmm5, %xmm8
> -#endif
> -
> - pmovmskb %xmm8, %ecx
> - pmovmskb %xmm6, %eax
> -
> - addq $(VEC_SIZE * 2), %rdi
> - /* Either null term or new occurence of CHAR. */
> - addl %ecx, %eax
> - jz L(second_loop)
> -
> - /* No null term so much be new occurence of CHAR. */
> - testl %ecx, %ecx
> - jz L(second_loop_match)
> -
> -
> - subl %ecx, %eax
> - jnz L(second_loop_new_match)
> -
> -L(second_loop_old_match):
> - pmovmskb %xmm2, %ecx
> - pmovmskb %xmm3, %eax
> - sall $16, %eax
> - orl %ecx, %eax
> - bsrl %eax, %eax
> - addq %rsi, %rax
> -#ifdef USE_AS_WCSRCHR
> - andq $-CHAR_SIZE, %rax
> -#endif
> - ret
> -
> - .p2align 4
> -L(second_loop_new_match):
> - pxor %xmm6, %xmm6
> - PCMPEQ %xmm9, %xmm6
> - pmovmskb %xmm6, %eax
> - sall $16, %ecx
> - orl %eax, %ecx
> -
> - /* We can't reuse either of the old comparisons as since we mask
> - of zeros after first zero (instead of using the full
> - comparison) we can't gurantee no interference between match
> - after end of string and valid match. */
> - pmovmskb %xmm4, %eax
> - pmovmskb %xmm7, %edx
> - sall $16, %edx
> - orl %edx, %eax
> -
> - leal -1(%ecx), %edx
> - xorl %edx, %ecx
> - andl %ecx, %eax
> - jz L(second_loop_old_match)
> - bsrl %eax, %eax
> - addq %rdi, %rax
> -#ifdef USE_AS_WCSRCHR
> - andq $-CHAR_SIZE, %rax
> -#endif
> - ret
> -
> - .p2align 4,, 4
> -L(cross_page):
> - movq %rdi, %rsi
> - andq $-VEC_SIZE, %rsi
> - movaps (%rsi), %xmm1
> - pxor %xmm2, %xmm2
> - PCMPEQ %xmm1, %xmm2
> - pmovmskb %xmm2, %edx
> - movl %edi, %ecx
> - andl $(VEC_SIZE - 1), %ecx
> - sarl %cl, %edx
> - jz L(cross_page_continue)
> - PCMPEQ %xmm0, %xmm1
> - pmovmskb %xmm1, %eax
> - sarl %cl, %eax
> - leal -1(%rdx), %ecx
> - xorl %edx, %ecx
> - andl %ecx, %eax
> - jz L(ret1)
> - bsrl %eax, %eax
> - addq %rdi, %rax
> -#ifdef USE_AS_WCSRCHR
> - andq $-CHAR_SIZE, %rax
> -#endif
> -L(ret1):
> - ret
> -END(STRRCHR)
> -
> -#ifndef USE_AS_WCSRCHR
> - weak_alias (STRRCHR, rindex)
> - libc_hidden_builtin_def (STRRCHR)
> -#endif
> +#define STRRCHR strrchr
> +#include "multiarch/strrchr-sse2.S"
> +weak_alias (strrchr, rindex)
> +libc_hidden_builtin_def (strrchr)
> diff --git a/sysdeps/x86_64/wcsrchr.S b/sysdeps/x86_64/wcsrchr.S
> index 2b80efc5ef..1d4b1eb21c 100644
> --- a/sysdeps/x86_64/wcsrchr.S
> +++ b/sysdeps/x86_64/wcsrchr.S
> @@ -16,12 +16,5 @@
> License along with the GNU C Library; if not, see
> <https://www.gnu.org/licenses/>. */
>
> -
> -#define USE_AS_WCSRCHR 1
> -#define NO_PMINU 1
> -
> -#ifndef STRRCHR
> -# define STRRCHR wcsrchr
> -#endif
> -
> -#include "../strrchr.S"
> +#define STRRCHR wcsrchr
> +#include "multiarch/wcsrchr-sse2.S"
> --
> 2.34.1
>
LGTM.
Thanks.
@@ -17,12 +17,358 @@
<https://www.gnu.org/licenses/>. */
#if IS_IN (libc)
-# define STRRCHR __strrchr_sse2
+# ifndef STRRCHR
+# define STRRCHR __strrchr_sse2
+# endif
+#endif
+
+#include <sysdep.h>
+
+#ifdef USE_AS_WCSRCHR
+# define PCMPEQ pcmpeqd
+# define CHAR_SIZE 4
+# define PMINU pminud
+#else
+# define PCMPEQ pcmpeqb
+# define CHAR_SIZE 1
+# define PMINU pminub
+#endif
+
+#define PAGE_SIZE 4096
+#define VEC_SIZE 16
+
+ .text
+ENTRY(STRRCHR)
+ movd %esi, %xmm0
+ movq %rdi, %rax
+ andl $(PAGE_SIZE - 1), %eax
+#ifndef USE_AS_WCSRCHR
+ punpcklbw %xmm0, %xmm0
+ punpcklwd %xmm0, %xmm0
+#endif
+ pshufd $0, %xmm0, %xmm0
+ cmpl $(PAGE_SIZE - VEC_SIZE), %eax
+ ja L(cross_page)
+
+L(cross_page_continue):
+ movups (%rdi), %xmm1
+ pxor %xmm2, %xmm2
+ PCMPEQ %xmm1, %xmm2
+ pmovmskb %xmm2, %ecx
+ testl %ecx, %ecx
+ jz L(aligned_more)
+
+ PCMPEQ %xmm0, %xmm1
+ pmovmskb %xmm1, %eax
+ leal -1(%rcx), %edx
+ xorl %edx, %ecx
+ andl %ecx, %eax
+ jz L(ret0)
+ bsrl %eax, %eax
+ addq %rdi, %rax
+ /* We are off by 3 for wcsrchr if search CHAR is non-zero. If
+ search CHAR is zero we are correct. Either way `andq
+ -CHAR_SIZE, %rax` gets the correct result. */
+#ifdef USE_AS_WCSRCHR
+ andq $-CHAR_SIZE, %rax
+#endif
+L(ret0):
+ ret
+
+ /* Returns for first vec x1/x2 have hard coded backward search
+ path for earlier matches. */
+ .p2align 4
+L(first_vec_x0_test):
+ PCMPEQ %xmm0, %xmm1
+ pmovmskb %xmm1, %eax
+ testl %eax, %eax
+ jz L(ret0)
+ bsrl %eax, %eax
+ addq %r8, %rax
+#ifdef USE_AS_WCSRCHR
+ andq $-CHAR_SIZE, %rax
+#endif
+ ret
+
+ .p2align 4
+L(first_vec_x1):
+ PCMPEQ %xmm0, %xmm2
+ pmovmskb %xmm2, %eax
+ leal -1(%rcx), %edx
+ xorl %edx, %ecx
+ andl %ecx, %eax
+ jz L(first_vec_x0_test)
+ bsrl %eax, %eax
+ leaq (VEC_SIZE)(%rdi, %rax), %rax
+#ifdef USE_AS_WCSRCHR
+ andq $-CHAR_SIZE, %rax
+#endif
+ ret
+
+ .p2align 4
+L(first_vec_x1_test):
+ PCMPEQ %xmm0, %xmm2
+ pmovmskb %xmm2, %eax
+ testl %eax, %eax
+ jz L(first_vec_x0_test)
+ bsrl %eax, %eax
+ leaq (VEC_SIZE)(%rdi, %rax), %rax
+#ifdef USE_AS_WCSRCHR
+ andq $-CHAR_SIZE, %rax
+#endif
+ ret
+
+ .p2align 4
+L(first_vec_x2):
+ PCMPEQ %xmm0, %xmm3
+ pmovmskb %xmm3, %eax
+ leal -1(%rcx), %edx
+ xorl %edx, %ecx
+ andl %ecx, %eax
+ jz L(first_vec_x1_test)
+ bsrl %eax, %eax
+ leaq (VEC_SIZE * 2)(%rdi, %rax), %rax
+#ifdef USE_AS_WCSRCHR
+ andq $-CHAR_SIZE, %rax
+#endif
+ ret
+
+ .p2align 4
+L(aligned_more):
+ /* Save original pointer if match was in VEC 0. */
+ movq %rdi, %r8
+ andq $-VEC_SIZE, %rdi
+
+ movaps VEC_SIZE(%rdi), %xmm2
+ pxor %xmm3, %xmm3
+ PCMPEQ %xmm2, %xmm3
+ pmovmskb %xmm3, %ecx
+ testl %ecx, %ecx
+ jnz L(first_vec_x1)
+
+ movaps (VEC_SIZE * 2)(%rdi), %xmm3
+ pxor %xmm4, %xmm4
+ PCMPEQ %xmm3, %xmm4
+ pmovmskb %xmm4, %ecx
+ testl %ecx, %ecx
+ jnz L(first_vec_x2)
+
+ addq $VEC_SIZE, %rdi
+ /* Save pointer again before realigning. */
+ movq %rdi, %rsi
+ andq $-(VEC_SIZE * 2), %rdi
+ .p2align 4
+L(first_loop):
+ /* Do 2x VEC at a time. */
+ movaps (VEC_SIZE * 2)(%rdi), %xmm4
+ movaps (VEC_SIZE * 3)(%rdi), %xmm5
+ /* Since SSE2 no pminud so wcsrchr needs seperate logic for
+ detecting zero. Note if this is found to be a bottleneck it
+ may be worth adding an SSE4.1 wcsrchr implementation. */
+#ifdef USE_AS_WCSRCHR
+ movaps %xmm5, %xmm6
+ pxor %xmm8, %xmm8
+
+ PCMPEQ %xmm8, %xmm5
+ PCMPEQ %xmm4, %xmm8
+ por %xmm5, %xmm8
+#else
+ movaps %xmm5, %xmm6
+ PMINU %xmm4, %xmm5
+#endif
+
+ movaps %xmm4, %xmm9
+ PCMPEQ %xmm0, %xmm4
+ PCMPEQ %xmm0, %xmm6
+ movaps %xmm6, %xmm7
+ por %xmm4, %xmm6
+#ifndef USE_AS_WCSRCHR
+ pxor %xmm8, %xmm8
+ PCMPEQ %xmm5, %xmm8
+#endif
+ pmovmskb %xmm8, %ecx
+ pmovmskb %xmm6, %eax
-# undef weak_alias
-# define weak_alias(strrchr, rindex)
-# undef libc_hidden_builtin_def
-# define libc_hidden_builtin_def(strrchr)
+ addq $(VEC_SIZE * 2), %rdi
+ /* Use `addl` 1) so we can undo it with `subl` and 2) it can
+ macro-fuse with `jz`. */
+ addl %ecx, %eax
+ jz L(first_loop)
+
+ /* Check if there is zero match. */
+ testl %ecx, %ecx
+ jz L(second_loop_match)
+
+ /* Check if there was a match in last iteration. */
+ subl %ecx, %eax
+ jnz L(new_match)
+
+L(first_loop_old_match):
+ PCMPEQ %xmm0, %xmm2
+ PCMPEQ %xmm0, %xmm3
+ pmovmskb %xmm2, %ecx
+ pmovmskb %xmm3, %eax
+ addl %eax, %ecx
+ jz L(first_vec_x0_test)
+ /* NB: We could move this shift to before the branch and save a
+ bit of code size / performance on the fall through. The
+ branch leads to the null case which generally seems hotter
+ than char in first 3x VEC. */
+ sall $16, %eax
+ orl %ecx, %eax
+
+ bsrl %eax, %eax
+ addq %rsi, %rax
+#ifdef USE_AS_WCSRCHR
+ andq $-CHAR_SIZE, %rax
+#endif
+ ret
+
+ .p2align 4
+L(new_match):
+ pxor %xmm6, %xmm6
+ PCMPEQ %xmm9, %xmm6
+ pmovmskb %xmm6, %eax
+ sall $16, %ecx
+ orl %eax, %ecx
+
+ /* We can't reuse either of the old comparisons as since we mask
+ of zeros after first zero (instead of using the full
+ comparison) we can't gurantee no interference between match
+ after end of string and valid match. */
+ pmovmskb %xmm4, %eax
+ pmovmskb %xmm7, %edx
+ sall $16, %edx
+ orl %edx, %eax
+
+ leal -1(%ecx), %edx
+ xorl %edx, %ecx
+ andl %ecx, %eax
+ jz L(first_loop_old_match)
+ bsrl %eax, %eax
+ addq %rdi, %rax
+#ifdef USE_AS_WCSRCHR
+ andq $-CHAR_SIZE, %rax
+#endif
+ ret
+
+ /* Save minimum state for getting most recent match. We can
+ throw out all previous work. */
+ .p2align 4
+L(second_loop_match):
+ movq %rdi, %rsi
+ movaps %xmm4, %xmm2
+ movaps %xmm7, %xmm3
+
+ .p2align 4
+L(second_loop):
+ movaps (VEC_SIZE * 2)(%rdi), %xmm4
+ movaps (VEC_SIZE * 3)(%rdi), %xmm5
+ /* Since SSE2 no pminud so wcsrchr needs seperate logic for
+ detecting zero. Note if this is found to be a bottleneck it
+ may be worth adding an SSE4.1 wcsrchr implementation. */
+#ifdef USE_AS_WCSRCHR
+ movaps %xmm5, %xmm6
+ pxor %xmm8, %xmm8
+
+ PCMPEQ %xmm8, %xmm5
+ PCMPEQ %xmm4, %xmm8
+ por %xmm5, %xmm8
+#else
+ movaps %xmm5, %xmm6
+ PMINU %xmm4, %xmm5
+#endif
+
+ movaps %xmm4, %xmm9
+ PCMPEQ %xmm0, %xmm4
+ PCMPEQ %xmm0, %xmm6
+ movaps %xmm6, %xmm7
+ por %xmm4, %xmm6
+#ifndef USE_AS_WCSRCHR
+ pxor %xmm8, %xmm8
+ PCMPEQ %xmm5, %xmm8
#endif
-#include "../strrchr.S"
+ pmovmskb %xmm8, %ecx
+ pmovmskb %xmm6, %eax
+
+ addq $(VEC_SIZE * 2), %rdi
+ /* Either null term or new occurence of CHAR. */
+ addl %ecx, %eax
+ jz L(second_loop)
+
+ /* No null term so much be new occurence of CHAR. */
+ testl %ecx, %ecx
+ jz L(second_loop_match)
+
+
+ subl %ecx, %eax
+ jnz L(second_loop_new_match)
+
+L(second_loop_old_match):
+ pmovmskb %xmm2, %ecx
+ pmovmskb %xmm3, %eax
+ sall $16, %eax
+ orl %ecx, %eax
+ bsrl %eax, %eax
+ addq %rsi, %rax
+#ifdef USE_AS_WCSRCHR
+ andq $-CHAR_SIZE, %rax
+#endif
+ ret
+
+ .p2align 4
+L(second_loop_new_match):
+ pxor %xmm6, %xmm6
+ PCMPEQ %xmm9, %xmm6
+ pmovmskb %xmm6, %eax
+ sall $16, %ecx
+ orl %eax, %ecx
+
+ /* We can't reuse either of the old comparisons as since we mask
+ of zeros after first zero (instead of using the full
+ comparison) we can't gurantee no interference between match
+ after end of string and valid match. */
+ pmovmskb %xmm4, %eax
+ pmovmskb %xmm7, %edx
+ sall $16, %edx
+ orl %edx, %eax
+
+ leal -1(%ecx), %edx
+ xorl %edx, %ecx
+ andl %ecx, %eax
+ jz L(second_loop_old_match)
+ bsrl %eax, %eax
+ addq %rdi, %rax
+#ifdef USE_AS_WCSRCHR
+ andq $-CHAR_SIZE, %rax
+#endif
+ ret
+
+ .p2align 4,, 4
+L(cross_page):
+ movq %rdi, %rsi
+ andq $-VEC_SIZE, %rsi
+ movaps (%rsi), %xmm1
+ pxor %xmm2, %xmm2
+ PCMPEQ %xmm1, %xmm2
+ pmovmskb %xmm2, %edx
+ movl %edi, %ecx
+ andl $(VEC_SIZE - 1), %ecx
+ sarl %cl, %edx
+ jz L(cross_page_continue)
+ PCMPEQ %xmm0, %xmm1
+ pmovmskb %xmm1, %eax
+ sarl %cl, %eax
+ leal -1(%rdx), %ecx
+ xorl %edx, %ecx
+ andl %ecx, %eax
+ jz L(ret1)
+ bsrl %eax, %eax
+ addq %rdi, %rax
+#ifdef USE_AS_WCSRCHR
+ andq $-CHAR_SIZE, %rax
+#endif
+L(ret1):
+ ret
+END(STRRCHR)
@@ -17,6 +17,12 @@
<https://www.gnu.org/licenses/>. */
#if IS_IN (libc)
-# define STRRCHR __wcsrchr_sse2
+# ifndef STRRCHR
+# define STRRCHR __wcsrchr_sse2
+# endif
#endif
-#include "../wcsrchr.S"
+
+#define USE_AS_WCSRCHR 1
+#define NO_PMINU 1
+
+#include "strrchr-sse2.S"
@@ -16,363 +16,7 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-
-#include <sysdep.h>
-
-#ifndef STRRCHR
-# define STRRCHR strrchr
-#endif
-
-#ifdef USE_AS_WCSRCHR
-# define PCMPEQ pcmpeqd
-# define CHAR_SIZE 4
-# define PMINU pminud
-#else
-# define PCMPEQ pcmpeqb
-# define CHAR_SIZE 1
-# define PMINU pminub
-#endif
-
-#define PAGE_SIZE 4096
-#define VEC_SIZE 16
-
- .text
-ENTRY(STRRCHR)
- movd %esi, %xmm0
- movq %rdi, %rax
- andl $(PAGE_SIZE - 1), %eax
-#ifndef USE_AS_WCSRCHR
- punpcklbw %xmm0, %xmm0
- punpcklwd %xmm0, %xmm0
-#endif
- pshufd $0, %xmm0, %xmm0
- cmpl $(PAGE_SIZE - VEC_SIZE), %eax
- ja L(cross_page)
-
-L(cross_page_continue):
- movups (%rdi), %xmm1
- pxor %xmm2, %xmm2
- PCMPEQ %xmm1, %xmm2
- pmovmskb %xmm2, %ecx
- testl %ecx, %ecx
- jz L(aligned_more)
-
- PCMPEQ %xmm0, %xmm1
- pmovmskb %xmm1, %eax
- leal -1(%rcx), %edx
- xorl %edx, %ecx
- andl %ecx, %eax
- jz L(ret0)
- bsrl %eax, %eax
- addq %rdi, %rax
- /* We are off by 3 for wcsrchr if search CHAR is non-zero. If
- search CHAR is zero we are correct. Either way `andq
- -CHAR_SIZE, %rax` gets the correct result. */
-#ifdef USE_AS_WCSRCHR
- andq $-CHAR_SIZE, %rax
-#endif
-L(ret0):
- ret
-
- /* Returns for first vec x1/x2 have hard coded backward search
- path for earlier matches. */
- .p2align 4
-L(first_vec_x0_test):
- PCMPEQ %xmm0, %xmm1
- pmovmskb %xmm1, %eax
- testl %eax, %eax
- jz L(ret0)
- bsrl %eax, %eax
- addq %r8, %rax
-#ifdef USE_AS_WCSRCHR
- andq $-CHAR_SIZE, %rax
-#endif
- ret
-
- .p2align 4
-L(first_vec_x1):
- PCMPEQ %xmm0, %xmm2
- pmovmskb %xmm2, %eax
- leal -1(%rcx), %edx
- xorl %edx, %ecx
- andl %ecx, %eax
- jz L(first_vec_x0_test)
- bsrl %eax, %eax
- leaq (VEC_SIZE)(%rdi, %rax), %rax
-#ifdef USE_AS_WCSRCHR
- andq $-CHAR_SIZE, %rax
-#endif
- ret
-
- .p2align 4
-L(first_vec_x1_test):
- PCMPEQ %xmm0, %xmm2
- pmovmskb %xmm2, %eax
- testl %eax, %eax
- jz L(first_vec_x0_test)
- bsrl %eax, %eax
- leaq (VEC_SIZE)(%rdi, %rax), %rax
-#ifdef USE_AS_WCSRCHR
- andq $-CHAR_SIZE, %rax
-#endif
- ret
-
- .p2align 4
-L(first_vec_x2):
- PCMPEQ %xmm0, %xmm3
- pmovmskb %xmm3, %eax
- leal -1(%rcx), %edx
- xorl %edx, %ecx
- andl %ecx, %eax
- jz L(first_vec_x1_test)
- bsrl %eax, %eax
- leaq (VEC_SIZE * 2)(%rdi, %rax), %rax
-#ifdef USE_AS_WCSRCHR
- andq $-CHAR_SIZE, %rax
-#endif
- ret
-
- .p2align 4
-L(aligned_more):
- /* Save original pointer if match was in VEC 0. */
- movq %rdi, %r8
- andq $-VEC_SIZE, %rdi
-
- movaps VEC_SIZE(%rdi), %xmm2
- pxor %xmm3, %xmm3
- PCMPEQ %xmm2, %xmm3
- pmovmskb %xmm3, %ecx
- testl %ecx, %ecx
- jnz L(first_vec_x1)
-
- movaps (VEC_SIZE * 2)(%rdi), %xmm3
- pxor %xmm4, %xmm4
- PCMPEQ %xmm3, %xmm4
- pmovmskb %xmm4, %ecx
- testl %ecx, %ecx
- jnz L(first_vec_x2)
-
- addq $VEC_SIZE, %rdi
- /* Save pointer again before realigning. */
- movq %rdi, %rsi
- andq $-(VEC_SIZE * 2), %rdi
- .p2align 4
-L(first_loop):
- /* Do 2x VEC at a time. */
- movaps (VEC_SIZE * 2)(%rdi), %xmm4
- movaps (VEC_SIZE * 3)(%rdi), %xmm5
- /* Since SSE2 no pminud so wcsrchr needs seperate logic for
- detecting zero. Note if this is found to be a bottleneck it
- may be worth adding an SSE4.1 wcsrchr implementation. */
-#ifdef USE_AS_WCSRCHR
- movaps %xmm5, %xmm6
- pxor %xmm8, %xmm8
-
- PCMPEQ %xmm8, %xmm5
- PCMPEQ %xmm4, %xmm8
- por %xmm5, %xmm8
-#else
- movaps %xmm5, %xmm6
- PMINU %xmm4, %xmm5
-#endif
-
- movaps %xmm4, %xmm9
- PCMPEQ %xmm0, %xmm4
- PCMPEQ %xmm0, %xmm6
- movaps %xmm6, %xmm7
- por %xmm4, %xmm6
-#ifndef USE_AS_WCSRCHR
- pxor %xmm8, %xmm8
- PCMPEQ %xmm5, %xmm8
-#endif
- pmovmskb %xmm8, %ecx
- pmovmskb %xmm6, %eax
-
- addq $(VEC_SIZE * 2), %rdi
- /* Use `addl` 1) so we can undo it with `subl` and 2) it can
- macro-fuse with `jz`. */
- addl %ecx, %eax
- jz L(first_loop)
-
- /* Check if there is zero match. */
- testl %ecx, %ecx
- jz L(second_loop_match)
-
- /* Check if there was a match in last iteration. */
- subl %ecx, %eax
- jnz L(new_match)
-
-L(first_loop_old_match):
- PCMPEQ %xmm0, %xmm2
- PCMPEQ %xmm0, %xmm3
- pmovmskb %xmm2, %ecx
- pmovmskb %xmm3, %eax
- addl %eax, %ecx
- jz L(first_vec_x0_test)
- /* NB: We could move this shift to before the branch and save a
- bit of code size / performance on the fall through. The
- branch leads to the null case which generally seems hotter
- than char in first 3x VEC. */
- sall $16, %eax
- orl %ecx, %eax
-
- bsrl %eax, %eax
- addq %rsi, %rax
-#ifdef USE_AS_WCSRCHR
- andq $-CHAR_SIZE, %rax
-#endif
- ret
-
- .p2align 4
-L(new_match):
- pxor %xmm6, %xmm6
- PCMPEQ %xmm9, %xmm6
- pmovmskb %xmm6, %eax
- sall $16, %ecx
- orl %eax, %ecx
-
- /* We can't reuse either of the old comparisons as since we mask
- of zeros after first zero (instead of using the full
- comparison) we can't gurantee no interference between match
- after end of string and valid match. */
- pmovmskb %xmm4, %eax
- pmovmskb %xmm7, %edx
- sall $16, %edx
- orl %edx, %eax
-
- leal -1(%ecx), %edx
- xorl %edx, %ecx
- andl %ecx, %eax
- jz L(first_loop_old_match)
- bsrl %eax, %eax
- addq %rdi, %rax
-#ifdef USE_AS_WCSRCHR
- andq $-CHAR_SIZE, %rax
-#endif
- ret
-
- /* Save minimum state for getting most recent match. We can
- throw out all previous work. */
- .p2align 4
-L(second_loop_match):
- movq %rdi, %rsi
- movaps %xmm4, %xmm2
- movaps %xmm7, %xmm3
-
- .p2align 4
-L(second_loop):
- movaps (VEC_SIZE * 2)(%rdi), %xmm4
- movaps (VEC_SIZE * 3)(%rdi), %xmm5
- /* Since SSE2 no pminud so wcsrchr needs seperate logic for
- detecting zero. Note if this is found to be a bottleneck it
- may be worth adding an SSE4.1 wcsrchr implementation. */
-#ifdef USE_AS_WCSRCHR
- movaps %xmm5, %xmm6
- pxor %xmm8, %xmm8
-
- PCMPEQ %xmm8, %xmm5
- PCMPEQ %xmm4, %xmm8
- por %xmm5, %xmm8
-#else
- movaps %xmm5, %xmm6
- PMINU %xmm4, %xmm5
-#endif
-
- movaps %xmm4, %xmm9
- PCMPEQ %xmm0, %xmm4
- PCMPEQ %xmm0, %xmm6
- movaps %xmm6, %xmm7
- por %xmm4, %xmm6
-#ifndef USE_AS_WCSRCHR
- pxor %xmm8, %xmm8
- PCMPEQ %xmm5, %xmm8
-#endif
-
- pmovmskb %xmm8, %ecx
- pmovmskb %xmm6, %eax
-
- addq $(VEC_SIZE * 2), %rdi
- /* Either null term or new occurence of CHAR. */
- addl %ecx, %eax
- jz L(second_loop)
-
- /* No null term so much be new occurence of CHAR. */
- testl %ecx, %ecx
- jz L(second_loop_match)
-
-
- subl %ecx, %eax
- jnz L(second_loop_new_match)
-
-L(second_loop_old_match):
- pmovmskb %xmm2, %ecx
- pmovmskb %xmm3, %eax
- sall $16, %eax
- orl %ecx, %eax
- bsrl %eax, %eax
- addq %rsi, %rax
-#ifdef USE_AS_WCSRCHR
- andq $-CHAR_SIZE, %rax
-#endif
- ret
-
- .p2align 4
-L(second_loop_new_match):
- pxor %xmm6, %xmm6
- PCMPEQ %xmm9, %xmm6
- pmovmskb %xmm6, %eax
- sall $16, %ecx
- orl %eax, %ecx
-
- /* We can't reuse either of the old comparisons as since we mask
- of zeros after first zero (instead of using the full
- comparison) we can't gurantee no interference between match
- after end of string and valid match. */
- pmovmskb %xmm4, %eax
- pmovmskb %xmm7, %edx
- sall $16, %edx
- orl %edx, %eax
-
- leal -1(%ecx), %edx
- xorl %edx, %ecx
- andl %ecx, %eax
- jz L(second_loop_old_match)
- bsrl %eax, %eax
- addq %rdi, %rax
-#ifdef USE_AS_WCSRCHR
- andq $-CHAR_SIZE, %rax
-#endif
- ret
-
- .p2align 4,, 4
-L(cross_page):
- movq %rdi, %rsi
- andq $-VEC_SIZE, %rsi
- movaps (%rsi), %xmm1
- pxor %xmm2, %xmm2
- PCMPEQ %xmm1, %xmm2
- pmovmskb %xmm2, %edx
- movl %edi, %ecx
- andl $(VEC_SIZE - 1), %ecx
- sarl %cl, %edx
- jz L(cross_page_continue)
- PCMPEQ %xmm0, %xmm1
- pmovmskb %xmm1, %eax
- sarl %cl, %eax
- leal -1(%rdx), %ecx
- xorl %edx, %ecx
- andl %ecx, %eax
- jz L(ret1)
- bsrl %eax, %eax
- addq %rdi, %rax
-#ifdef USE_AS_WCSRCHR
- andq $-CHAR_SIZE, %rax
-#endif
-L(ret1):
- ret
-END(STRRCHR)
-
-#ifndef USE_AS_WCSRCHR
- weak_alias (STRRCHR, rindex)
- libc_hidden_builtin_def (STRRCHR)
-#endif
+#define STRRCHR strrchr
+#include "multiarch/strrchr-sse2.S"
+weak_alias (strrchr, rindex)
+libc_hidden_builtin_def (strrchr)
@@ -16,12 +16,5 @@
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
-
-#define USE_AS_WCSRCHR 1
-#define NO_PMINU 1
-
-#ifndef STRRCHR
-# define STRRCHR wcsrchr
-#endif
-
-#include "../strrchr.S"
+#define STRRCHR wcsrchr
+#include "multiarch/wcsrchr-sse2.S"