powerpc: P9 vector load instruction change in memcpy and memmove
Commit Message
From: Rajalakshmi Srinivasaraghavan <raji@linux.vnet.ibm.com>
POWER9 DD2.1 and earlier has an issue where some cache inhibited
vector load traps to the kernel. To handle this in memcpy
and memmove, lvx/stvx is used for aligned addresses instead
of lxvd2x/stxvd2x. The remaining part of the optimization remains
same as existing POWER7 code.
Reference: https://patchwork.ozlabs.org/patch/814059/
Tested on powerpc64le.
2017-10-19 Rajalakshmi Srinivasaraghavan <raji@linux.vnet.ibm.com>
* sysdeps/powerpc/powerpc64/multiarch/Makefile
(sysdep_routines): Add memcpy_power9 and memmove_power9.
* sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
(memcpy): Add __memcpy_power9 to list of memcpy functions.
(memmove): Add __memmove_power9 to list of memmove functions.
(bcopy): Add __bcopy_power9 to list of bcopy functions.
* sysdeps/powerpc/powerpc64/multiarch/memcpy.c
(memcpy): Add __memcpy_power9 to ifunc list.
* sysdeps/powerpc/powerpc64/power9/memcpy.S: New File.
* sysdeps/powerpc/powerpc64/multiarch/memcpy-power9.S: Likewise.
* sysdeps/powerpc/powerpc64/multiarch/bcopy.c
(bcopy): Add __bcopy_power9 to ifunc list.
* sysdeps/powerpc/powerpc64/multiarch/memmove-power7.S
Change bcopy as __bcopy.
* sysdeps/powerpc/powerpc64/multiarch/memmove.c
(memmove): Add __memmove_power9 to ifunc list.
* sysdeps/powerpc/powerpc64/power7/memmove.S:
Alias bcopy only if not defined before.
* sysdeps/powerpc/powerpc64/multiarch/memmove-power9.S:
New file.
* sysdeps/powerpc/powerpc64/power9/memmove.S: Likewise.
---
sysdeps/powerpc/powerpc64/multiarch/Makefile | 7 +-
sysdeps/powerpc/powerpc64/multiarch/bcopy.c | 6 +-
.../powerpc/powerpc64/multiarch/ifunc-impl-list.c | 6 +
.../powerpc/powerpc64/multiarch/memcpy-power9.S | 26 +
sysdeps/powerpc/powerpc64/multiarch/memcpy.c | 3 +
.../powerpc/powerpc64/multiarch/memmove-power7.S | 4 +-
.../powerpc/powerpc64/multiarch/memmove-power9.S | 29 +
sysdeps/powerpc/powerpc64/multiarch/memmove.c | 5 +-
sysdeps/powerpc/powerpc64/power7/memmove.S | 2 +
sysdeps/powerpc/powerpc64/power9/memcpy.S | 429 +++++++++++
sysdeps/powerpc/powerpc64/power9/memmove.S | 837 +++++++++++++++++++++
11 files changed, 1347 insertions(+), 7 deletions(-)
create mode 100644 sysdeps/powerpc/powerpc64/multiarch/memcpy-power9.S
create mode 100644 sysdeps/powerpc/powerpc64/multiarch/memmove-power9.S
create mode 100644 sysdeps/powerpc/powerpc64/power9/memcpy.S
create mode 100644 sysdeps/powerpc/powerpc64/power9/memmove.S
Comments
On 10/19/2017 05:25 PM, Tulio Magno Quites Machado Filho wrote:
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/bcopy.c b/sysdeps/powerpc/powerpc64/multiarch/bcopy.c
> index 05d46e2..4a4ee6e 100644
> --- a/sysdeps/powerpc/powerpc64/multiarch/bcopy.c
> +++ b/sysdeps/powerpc/powerpc64/multiarch/bcopy.c
> @@ -22,8 +22,12 @@
> extern __typeof (bcopy) __bcopy_ppc attribute_hidden;
> /* __bcopy_power7 symbol is implemented at memmove-power7.S */
> extern __typeof (bcopy) __bcopy_power7 attribute_hidden;
> +/* __bcopy_power9 symbol is implemented at memmove-power9.S. */
> +extern __typeof (bcopy) __bcopy_power9 attribute_hidden;
>
> libc_ifunc (bcopy,
> - (hwcap & PPC_FEATURE_HAS_VSX)
> + (hwcap2 & PPC_FEATURE2_ARCH_3_00)
> + ? __bcopy_power9
> + : (hwcap & PPC_FEATURE_HAS_VSX)
> ? __bcopy_power7
> : __bcopy_ppc);
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
> index 6a88536..9040bbc 100644
> --- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
> +++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
> @@ -51,6 +51,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
> #ifdef SHARED
> /* Support sysdeps/powerpc/powerpc64/multiarch/memcpy.c. */
> IFUNC_IMPL (i, name, memcpy,
> + IFUNC_IMPL_ADD (array, i, memcpy, hwcap2 & PPC_FEATURE2_ARCH_3_00,
> + __memcpy_power9)
> IFUNC_IMPL_ADD (array, i, memcpy, hwcap & PPC_FEATURE_HAS_VSX,
> __memcpy_power7)
> IFUNC_IMPL_ADD (array, i, memcpy, hwcap & PPC_FEATURE_ARCH_2_06,
> @@ -65,6 +67,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
>
> /* Support sysdeps/powerpc/powerpc64/multiarch/memmove.c. */
> IFUNC_IMPL (i, name, memmove,
> + IFUNC_IMPL_ADD (array, i, memmove, hwcap2 & PPC_FEATURE2_ARCH_3_00,
> + __memmove_power9)
> IFUNC_IMPL_ADD (array, i, memmove, hwcap & PPC_FEATURE_HAS_VSX,
> __memmove_power7)
> IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_ppc))
> @@ -168,6 +172,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
>
> /* Support sysdeps/powerpc/powerpc64/multiarch/bcopy.c. */
> IFUNC_IMPL (i, name, bcopy,
> + IFUNC_IMPL_ADD (array, i, bcopy, hwcap2 & PPC_FEATURE2_ARCH_3_00,
> + __bcopy_power9)
> IFUNC_IMPL_ADD (array, i, bcopy, hwcap & PPC_FEATURE_HAS_VSX,
> __bcopy_power7)
> IFUNC_IMPL_ADD (array, i, bcopy, 1, __bcopy_ppc))
I'm concerned that this needs *another* change to recognize post-DD2.1
POWER 9 hardware which has the fix (or perhaps OpenPOWER implementations
which never had the bug).
Is there any other way you could select the workaround that is more
specific? Or is the performance hit from avoiding the affected vector
instructions not so severe that it would be an immediate concern for
future silicon?
Thanks,
Florian
On 19/10/2017 13:25, Tulio Magno Quites Machado Filho wrote:
> From: Rajalakshmi Srinivasaraghavan <raji@linux.vnet.ibm.com>
>
> POWER9 DD2.1 and earlier has an issue where some cache inhibited
> vector load traps to the kernel. To handle this in memcpy
> and memmove, lvx/stvx is used for aligned addresses instead
> of lxvd2x/stxvd2x. The remaining part of the optimization remains
> same as existing POWER7 code.
>
> Reference: https://patchwork.ozlabs.org/patch/814059/
> Tested on powerpc64le.
According to "POWER8 Processor User’s Manual for the Single-Chip Module"
(it is buried on a sign wall at [1]), both lxv2dx/lvx and stxvd2x/stvx
uses the same pipeline, have the same latency and same throughput. The
only difference is lxv2dx/stxv2x have microcode handling for unaligned
case and for 4k crossing or 32-byte cross L1 miss (which should not
occur in the with aligned address).
Why not change POWER7 implementation instead of dropping another one
which is exactly the same for POWER9?
[1] https://www-355.ibm.com/systems/power/openpower/tgcmDocumentRepository.xhtml?aliasId=POWER8
>
> 2017-10-19 Rajalakshmi Srinivasaraghavan <raji@linux.vnet.ibm.com>
>
> * sysdeps/powerpc/powerpc64/multiarch/Makefile
> (sysdep_routines): Add memcpy_power9 and memmove_power9.
> * sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
> (memcpy): Add __memcpy_power9 to list of memcpy functions.
> (memmove): Add __memmove_power9 to list of memmove functions.
> (bcopy): Add __bcopy_power9 to list of bcopy functions.
> * sysdeps/powerpc/powerpc64/multiarch/memcpy.c
> (memcpy): Add __memcpy_power9 to ifunc list.
> * sysdeps/powerpc/powerpc64/power9/memcpy.S: New File.
> * sysdeps/powerpc/powerpc64/multiarch/memcpy-power9.S: Likewise.
> * sysdeps/powerpc/powerpc64/multiarch/bcopy.c
> (bcopy): Add __bcopy_power9 to ifunc list.
> * sysdeps/powerpc/powerpc64/multiarch/memmove-power7.S
> Change bcopy as __bcopy.
> * sysdeps/powerpc/powerpc64/multiarch/memmove.c
> (memmove): Add __memmove_power9 to ifunc list.
> * sysdeps/powerpc/powerpc64/power7/memmove.S:
> Alias bcopy only if not defined before.
> * sysdeps/powerpc/powerpc64/multiarch/memmove-power9.S:
> New file.
> * sysdeps/powerpc/powerpc64/power9/memmove.S: Likewise.
> ---
> sysdeps/powerpc/powerpc64/multiarch/Makefile | 7 +-
> sysdeps/powerpc/powerpc64/multiarch/bcopy.c | 6 +-
> .../powerpc/powerpc64/multiarch/ifunc-impl-list.c | 6 +
> .../powerpc/powerpc64/multiarch/memcpy-power9.S | 26 +
> sysdeps/powerpc/powerpc64/multiarch/memcpy.c | 3 +
> .../powerpc/powerpc64/multiarch/memmove-power7.S | 4 +-
> .../powerpc/powerpc64/multiarch/memmove-power9.S | 29 +
> sysdeps/powerpc/powerpc64/multiarch/memmove.c | 5 +-
> sysdeps/powerpc/powerpc64/power7/memmove.S | 2 +
> sysdeps/powerpc/powerpc64/power9/memcpy.S | 429 +++++++++++
> sysdeps/powerpc/powerpc64/power9/memmove.S | 837 +++++++++++++++++++++
> 11 files changed, 1347 insertions(+), 7 deletions(-)
> create mode 100644 sysdeps/powerpc/powerpc64/multiarch/memcpy-power9.S
> create mode 100644 sysdeps/powerpc/powerpc64/multiarch/memmove-power9.S
> create mode 100644 sysdeps/powerpc/powerpc64/power9/memcpy.S
> create mode 100644 sysdeps/powerpc/powerpc64/power9/memmove.S
>
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile
> index dea49ac..82728fa 100644
> --- a/sysdeps/powerpc/powerpc64/multiarch/Makefile
> +++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile
> @@ -1,6 +1,6 @@
> ifeq ($(subdir),string)
> -sysdep_routines += memcpy-power7 memcpy-a2 memcpy-power6 memcpy-cell \
> - memcpy-power4 memcpy-ppc64 \
> +sysdep_routines += memcpy-power9 memcpy-power7 memcpy-a2 memcpy-power6 \
> + memcpy-cell memcpy-power4 memcpy-ppc64 \
> memcmp-power8 memcmp-power7 memcmp-power4 memcmp-ppc64 \
> memset-power7 memset-power6 memset-power4 \
> memset-ppc64 memset-power8 \
> @@ -24,7 +24,8 @@ sysdep_routines += memcpy-power7 memcpy-a2 memcpy-power6 memcpy-cell \
> stpncpy-power8 stpncpy-power7 stpncpy-ppc64 \
> strcmp-power9 strcmp-power8 strcmp-power7 strcmp-ppc64 \
> strcat-power8 strcat-power7 strcat-ppc64 \
> - memmove-power7 memmove-ppc64 wordcopy-ppc64 bcopy-ppc64 \
> + memmove-power9 memmove-power7 memmove-ppc64 \
> + wordcopy-ppc64 bcopy-ppc64 \
> strncpy-power8 strstr-power7 strstr-ppc64 \
> strspn-power8 strspn-ppc64 strcspn-power8 strcspn-ppc64 \
> strlen-power8 strcasestr-power8 strcasestr-ppc64 \
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/bcopy.c b/sysdeps/powerpc/powerpc64/multiarch/bcopy.c
> index 05d46e2..4a4ee6e 100644
> --- a/sysdeps/powerpc/powerpc64/multiarch/bcopy.c
> +++ b/sysdeps/powerpc/powerpc64/multiarch/bcopy.c
> @@ -22,8 +22,12 @@
> extern __typeof (bcopy) __bcopy_ppc attribute_hidden;
> /* __bcopy_power7 symbol is implemented at memmove-power7.S */
> extern __typeof (bcopy) __bcopy_power7 attribute_hidden;
> +/* __bcopy_power9 symbol is implemented at memmove-power9.S. */
> +extern __typeof (bcopy) __bcopy_power9 attribute_hidden;
>
> libc_ifunc (bcopy,
> - (hwcap & PPC_FEATURE_HAS_VSX)
> + (hwcap2 & PPC_FEATURE2_ARCH_3_00)
> + ? __bcopy_power9
> + : (hwcap & PPC_FEATURE_HAS_VSX)
> ? __bcopy_power7
> : __bcopy_ppc);
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
> index 6a88536..9040bbc 100644
> --- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
> +++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
> @@ -51,6 +51,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
> #ifdef SHARED
> /* Support sysdeps/powerpc/powerpc64/multiarch/memcpy.c. */
> IFUNC_IMPL (i, name, memcpy,
> + IFUNC_IMPL_ADD (array, i, memcpy, hwcap2 & PPC_FEATURE2_ARCH_3_00,
> + __memcpy_power9)
> IFUNC_IMPL_ADD (array, i, memcpy, hwcap & PPC_FEATURE_HAS_VSX,
> __memcpy_power7)
> IFUNC_IMPL_ADD (array, i, memcpy, hwcap & PPC_FEATURE_ARCH_2_06,
> @@ -65,6 +67,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
>
> /* Support sysdeps/powerpc/powerpc64/multiarch/memmove.c. */
> IFUNC_IMPL (i, name, memmove,
> + IFUNC_IMPL_ADD (array, i, memmove, hwcap2 & PPC_FEATURE2_ARCH_3_00,
> + __memmove_power9)
> IFUNC_IMPL_ADD (array, i, memmove, hwcap & PPC_FEATURE_HAS_VSX,
> __memmove_power7)
> IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_ppc))
> @@ -168,6 +172,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
>
> /* Support sysdeps/powerpc/powerpc64/multiarch/bcopy.c. */
> IFUNC_IMPL (i, name, bcopy,
> + IFUNC_IMPL_ADD (array, i, bcopy, hwcap2 & PPC_FEATURE2_ARCH_3_00,
> + __bcopy_power9)
> IFUNC_IMPL_ADD (array, i, bcopy, hwcap & PPC_FEATURE_HAS_VSX,
> __bcopy_power7)
> IFUNC_IMPL_ADD (array, i, bcopy, 1, __bcopy_ppc))
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memcpy-power9.S b/sysdeps/powerpc/powerpc64/multiarch/memcpy-power9.S
> new file mode 100644
> index 0000000..fbd0788
> --- /dev/null
> +++ b/sysdeps/powerpc/powerpc64/multiarch/memcpy-power9.S
> @@ -0,0 +1,26 @@
> +/* Optimized memcpy implementation for PowerPC/POWER9.
> + Copyright (C) 2017 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +#include <sysdep.h>
> +
> +#define MEMCPY __memcpy_power9
> +
> +#undef libc_hidden_builtin_def
> +#define libc_hidden_builtin_def(name)
> +
> +#include <sysdeps/powerpc/powerpc64/power9/memcpy.S>
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memcpy.c b/sysdeps/powerpc/powerpc64/multiarch/memcpy.c
> index 9f4286c..4c16fa0 100644
> --- a/sysdeps/powerpc/powerpc64/multiarch/memcpy.c
> +++ b/sysdeps/powerpc/powerpc64/multiarch/memcpy.c
> @@ -35,8 +35,11 @@ extern __typeof (__redirect_memcpy) __memcpy_cell attribute_hidden;
> extern __typeof (__redirect_memcpy) __memcpy_power6 attribute_hidden;
> extern __typeof (__redirect_memcpy) __memcpy_a2 attribute_hidden;
> extern __typeof (__redirect_memcpy) __memcpy_power7 attribute_hidden;
> +extern __typeof (__redirect_memcpy) __memcpy_power9 attribute_hidden;
>
> libc_ifunc (__libc_memcpy,
> + (hwcap2 & PPC_FEATURE2_ARCH_3_00)
> + ? __memcpy_power9 :
> (hwcap & PPC_FEATURE_HAS_VSX)
> ? __memcpy_power7 :
> (hwcap & PPC_FEATURE_ARCH_2_06)
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memmove-power7.S b/sysdeps/powerpc/powerpc64/multiarch/memmove-power7.S
> index a9435fa..0599a39 100644
> --- a/sysdeps/powerpc/powerpc64/multiarch/memmove-power7.S
> +++ b/sysdeps/powerpc/powerpc64/multiarch/memmove-power7.S
> @@ -23,7 +23,7 @@
> #undef libc_hidden_builtin_def
> #define libc_hidden_builtin_def(name)
>
> -#undef bcopy
> -#define bcopy __bcopy_power7
> +#undef __bcopy
> +#define __bcopy __bcopy_power7
>
> #include <sysdeps/powerpc/powerpc64/power7/memmove.S>
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memmove-power9.S b/sysdeps/powerpc/powerpc64/multiarch/memmove-power9.S
> new file mode 100644
> index 0000000..16a2267
> --- /dev/null
> +++ b/sysdeps/powerpc/powerpc64/multiarch/memmove-power9.S
> @@ -0,0 +1,29 @@
> +/* Optimized memmove implementation for PowerPC64/POWER7.
> + Copyright (C) 2017 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +#include <sysdep.h>
> +
> +#define MEMMOVE __memmove_power9
> +
> +#undef libc_hidden_builtin_def
> +#define libc_hidden_builtin_def(name)
> +
> +#undef __bcopy
> +#define __bcopy __bcopy_power9
> +
> +#include <sysdeps/powerpc/powerpc64/power9/memmove.S>
> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memmove.c b/sysdeps/powerpc/powerpc64/multiarch/memmove.c
> index db2bbc7..f02498e 100644
> --- a/sysdeps/powerpc/powerpc64/multiarch/memmove.c
> +++ b/sysdeps/powerpc/powerpc64/multiarch/memmove.c
> @@ -31,9 +31,12 @@ extern __typeof (__redirect_memmove) __libc_memmove;
>
> extern __typeof (__redirect_memmove) __memmove_ppc attribute_hidden;
> extern __typeof (__redirect_memmove) __memmove_power7 attribute_hidden;
> +extern __typeof (__redirect_memmove) __memmove_power9 attribute_hidden;
>
> libc_ifunc (__libc_memmove,
> - (hwcap & PPC_FEATURE_HAS_VSX)
> + (hwcap2 & PPC_FEATURE2_ARCH_3_00)
> + ? __memmove_power9
> + : (hwcap & PPC_FEATURE_HAS_VSX)
> ? __memmove_power7
> : __memmove_ppc);
>
> diff --git a/sysdeps/powerpc/powerpc64/power7/memmove.S b/sysdeps/powerpc/powerpc64/power7/memmove.S
> index 93baa69..0bb8ddc 100644
> --- a/sysdeps/powerpc/powerpc64/power7/memmove.S
> +++ b/sysdeps/powerpc/powerpc64/power7/memmove.S
> @@ -832,4 +832,6 @@ ENTRY_TOCLESS (__bcopy)
> mr r4,r6
> b L(_memmove)
> END (__bcopy)
> +#ifndef __bcopy
> weak_alias (__bcopy, bcopy)
> +#endif
> diff --git a/sysdeps/powerpc/powerpc64/power9/memcpy.S b/sysdeps/powerpc/powerpc64/power9/memcpy.S
> new file mode 100644
> index 0000000..0731bac
> --- /dev/null
> +++ b/sysdeps/powerpc/powerpc64/power9/memcpy.S
> @@ -0,0 +1,429 @@
> +/* Optimized memcpy implementation for PowerPC64/POWER9.
> + Copyright (C) 2017 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +#include <sysdep.h>
> +
> +
> +/* void * [r3] memcpy (void *dst [r3], void *src [r4], size_t len [r5]);
> + Returns 'dst'. */
> +
> +#ifndef MEMCPY
> +# define MEMCPY memcpy
> +#endif
> +
> +#define dst 11 /* Use r11 so r3 kept unchanged. */
> +#define src 4
> +#define cnt 5
> +
> + .machine power7
> +ENTRY_TOCLESS (MEMCPY, 5)
> + CALL_MCOUNT 3
> +
> + cmpldi cr1,cnt,31
> + neg 0,3
> + ble cr1, L(copy_LT_32) /* If move < 32 bytes use short move
> + code. */
> +
> +/* Align copies using VSX instructions to quadword. It is to avoid alignment
> + traps when memcpy is used on non-cacheable memory (for instance, memory
> + mapped I/O). */
> + andi. 10,3,15
> + clrldi 11,4,60
> + cmpld cr6,10,11 /* SRC and DST alignments match? */
> +
> + mr dst,3
> + bne cr6,L(copy_GE_32_unaligned)
> + beq L(aligned_copy)
> +
> + mtocrf 0x01,0
> + clrldi 0,0,60
> +
> +/* Get the DST and SRC aligned to 16 bytes. */
> +1:
> + bf 31,2f
> + lbz 6,0(src)
> + addi src,src,1
> + stb 6,0(dst)
> + addi dst,dst,1
> +2:
> + bf 30,4f
> + lhz 6,0(src)
> + addi src,src,2
> + sth 6,0(dst)
> + addi dst,dst,2
> +4:
> + bf 29,8f
> + lwz 6,0(src)
> + addi src,src,4
> + stw 6,0(dst)
> + addi dst,dst,4
> +8:
> + bf 28,16f
> + ld 6,0(src)
> + addi src,src,8
> + std 6,0(dst)
> + addi dst,dst,8
> +16:
> + subf cnt,0,cnt
> +
> +/* Main aligned copy loop. Copies 128 bytes at a time. */
> +L(aligned_copy):
> + li 6,16
> + li 7,32
> + li 8,48
> + mtocrf 0x02,cnt
> + srdi 12,cnt,7
> + cmpdi 12,0
> + beq L(aligned_tail)
> + lvx 6,0,src
> + lvx 7,src,6
> + mtctr 12
> + b L(aligned_128loop)
> +
> + .align 4
> +L(aligned_128head):
> + /* for the 2nd + iteration of this loop. */
> + lvx 6,0,src
> + lvx 7,src,6
> +L(aligned_128loop):
> + lvx 8,src,7
> + lvx 9,src,8
> + stvx 6,0,dst
> + addi src,src,64
> + stvx 7,dst,6
> + stvx 8,dst,7
> + stvx 9,dst,8
> + lvx 6,0,src
> + lvx 7,src,6
> + addi dst,dst,64
> + lvx 8,src,7
> + lvx 9,src,8
> + addi src,src,64
> + stvx 6,0,dst
> + stvx 7,dst,6
> + stvx 8,dst,7
> + stvx 9,dst,8
> + addi dst,dst,64
> + bdnz L(aligned_128head)
> +
> +L(aligned_tail):
> + mtocrf 0x01,cnt
> + bf 25,32f
> + lvx 6,0,src
> + lvx 7,src,6
> + lvx 8,src,7
> + lvx 9,src,8
> + addi src,src,64
> + stvx 6,0,dst
> + stvx 7,dst,6
> + stvx 8,dst,7
> + stvx 9,dst,8
> + addi dst,dst,64
> +32:
> + bf 26,16f
> + lvx 6,0,src
> + lvx 7,src,6
> + addi src,src,32
> + stvx 6,0,dst
> + stvx 7,dst,6
> + addi dst,dst,32
> +16:
> + bf 27,8f
> + lvx 6,0,src
> + addi src,src,16
> + stvx 6,0,dst
> + addi dst,dst,16
> +8:
> + bf 28,4f
> + ld 6,0(src)
> + addi src,src,8
> + std 6,0(dst)
> + addi dst,dst,8
> +4: /* Copies 4~7 bytes. */
> + bf 29,L(tail2)
> + lwz 6,0(src)
> + stw 6,0(dst)
> + bf 30,L(tail5)
> + lhz 7,4(src)
> + sth 7,4(dst)
> + bflr 31
> + lbz 8,6(src)
> + stb 8,6(dst)
> + /* Return original DST pointer. */
> + blr
> +
> +
> +/* Handle copies of 0~31 bytes. */
> + .align 4
> +L(copy_LT_32):
> + mr dst,3
> + cmpldi cr6,cnt,8
> + mtocrf 0x01,cnt
> + ble cr6,L(copy_LE_8)
> +
> + /* At least 9 bytes to go. */
> + neg 8,4
> + andi. 0,8,3
> + cmpldi cr1,cnt,16
> + beq L(copy_LT_32_aligned)
> +
> + /* Force 4-byte alignment for SRC. */
> + mtocrf 0x01,0
> + subf cnt,0,cnt
> +2:
> + bf 30,1f
> + lhz 6,0(src)
> + addi src,src,2
> + sth 6,0(dst)
> + addi dst,dst,2
> +1:
> + bf 31,L(end_4bytes_alignment)
> + lbz 6,0(src)
> + addi src,src,1
> + stb 6,0(dst)
> + addi dst,dst,1
> +
> + .align 4
> +L(end_4bytes_alignment):
> + cmpldi cr1,cnt,16
> + mtocrf 0x01,cnt
> +
> +L(copy_LT_32_aligned):
> + /* At least 6 bytes to go, and SRC is word-aligned. */
> + blt cr1,8f
> +
> + /* Copy 16 bytes. */
> + lwz 6,0(src)
> + lwz 7,4(src)
> + stw 6,0(dst)
> + lwz 8,8(src)
> + stw 7,4(dst)
> + lwz 6,12(src)
> + addi src,src,16
> + stw 8,8(dst)
> + stw 6,12(dst)
> + addi dst,dst,16
> +8: /* Copy 8 bytes. */
> + bf 28,L(tail4)
> + lwz 6,0(src)
> + lwz 7,4(src)
> + addi src,src,8
> + stw 6,0(dst)
> + stw 7,4(dst)
> + addi dst,dst,8
> +
> + .align 4
> +/* Copies 4~7 bytes. */
> +L(tail4):
> + bf 29,L(tail2)
> + lwz 6,0(src)
> + stw 6,0(dst)
> + bf 30,L(tail5)
> + lhz 7,4(src)
> + sth 7,4(dst)
> + bflr 31
> + lbz 8,6(src)
> + stb 8,6(dst)
> + /* Return original DST pointer. */
> + blr
> +
> + .align 4
> +/* Copies 2~3 bytes. */
> +L(tail2):
> + bf 30,1f
> + lhz 6,0(src)
> + sth 6,0(dst)
> + bflr 31
> + lbz 7,2(src)
> + stb 7,2(dst)
> + blr
> +
> + .align 4
> +L(tail5):
> + bflr 31
> + lbz 6,4(src)
> + stb 6,4(dst)
> + blr
> +
> + .align 4
> +1:
> + bflr 31
> + lbz 6,0(src)
> + stb 6,0(dst)
> + /* Return original DST pointer. */
> + blr
> +
> +
> +/* Handles copies of 0~8 bytes. */
> + .align 4
> +L(copy_LE_8):
> + bne cr6,L(tail4)
> +
> + /* Though we could've used ld/std here, they are still
> + slow for unaligned cases. */
> +
> + lwz 6,0(src)
> + lwz 7,4(src)
> + stw 6,0(dst)
> + stw 7,4(dst)
> + blr
> +
> +
> +/* Handle copies of 32+ bytes where DST is aligned (to quadword) but
> + SRC is not. Use aligned quadword loads from SRC, shifted to realign
> + the data, allowing for aligned DST stores. */
> + .align 4
> +L(copy_GE_32_unaligned):
> + clrldi 0,0,60 /* Number of bytes until the 1st dst quadword. */
> + srdi 9,cnt,4 /* Number of full quadwords remaining. */
> +
> + beq L(copy_GE_32_unaligned_cont)
> +
> + /* DST is not quadword aligned, get it aligned. */
> +
> + mtocrf 0x01,0
> + subf cnt,0,cnt
> +
> + /* Vector instructions work best when proper alignment (16-bytes)
> + is present. Move 0~15 bytes as needed to get DST quadword-aligned. */
> +1:
> + bf 31,2f
> + lbz 6,0(src)
> + addi src,src,1
> + stb 6,0(dst)
> + addi dst,dst,1
> +2:
> + bf 30,4f
> + lhz 6,0(src)
> + addi src,src,2
> + sth 6,0(dst)
> + addi dst,dst,2
> +4:
> + bf 29,8f
> + lwz 6,0(src)
> + addi src,src,4
> + stw 6,0(dst)
> + addi dst,dst,4
> +8:
> + bf 28,0f
> + ld 6,0(src)
> + addi src,src,8
> + std 6,0(dst)
> + addi dst,dst,8
> +0:
> + srdi 9,cnt,4 /* Number of full quadwords remaining. */
> +
> + /* The proper alignment is present, it is OK to copy the bytes now. */
> +L(copy_GE_32_unaligned_cont):
> +
> + /* Setup two indexes to speed up the indexed vector operations. */
> + clrldi 10,cnt,60
> + li 6,16 /* Index for 16-bytes offsets. */
> + li 7,32 /* Index for 32-bytes offsets. */
> + cmpldi cr1,10,0
> + srdi 8,cnt,5 /* Setup the loop counter. */
> + mtocrf 0x01,9
> + cmpldi cr6,9,1
> +#ifdef __LITTLE_ENDIAN__
> + lvsr 5,0,src
> +#else
> + lvsl 5,0,src
> +#endif
> + lvx 3,0,src
> + li 0,0
> + bf 31,L(setup_unaligned_loop)
> +
> + /* Copy another 16 bytes to align to 32-bytes due to the loop. */
> + lvx 4,src,6
> +#ifdef __LITTLE_ENDIAN__
> + vperm 6,4,3,5
> +#else
> + vperm 6,3,4,5
> +#endif
> + addi src,src,16
> + stvx 6,0,dst
> + addi dst,dst,16
> + vor 3,4,4
> + clrrdi 0,src,60
> +
> +L(setup_unaligned_loop):
> + mtctr 8
> + ble cr6,L(end_unaligned_loop)
> +
> + /* Copy 32 bytes at a time using vector instructions. */
> + .align 4
> +L(unaligned_loop):
> +
> + /* Note: vr6/vr10 may contain data that was already copied,
> + but in order to get proper alignment, we may have to copy
> + some portions again. This is faster than having unaligned
> + vector instructions though. */
> +
> + lvx 4,src,6
> +#ifdef __LITTLE_ENDIAN__
> + vperm 6,4,3,5
> +#else
> + vperm 6,3,4,5
> +#endif
> + lvx 3,src,7
> +#ifdef __LITTLE_ENDIAN__
> + vperm 10,3,4,5
> +#else
> + vperm 10,4,3,5
> +#endif
> + addi src,src,32
> + stvx 6,0,dst
> + stvx 10,dst,6
> + addi dst,dst,32
> + bdnz L(unaligned_loop)
> +
> + clrrdi 0,src,60
> +
> + .align 4
> +L(end_unaligned_loop):
> +
> + /* Check for tail bytes. */
> + mtocrf 0x01,cnt
> + beqlr cr1
> +
> + add src,src,0
> +
> + /* We have 1~15 tail bytes to copy, and DST is quadword aligned. */
> + /* Copy 8 bytes. */
> + bf 28,4f
> + lwz 6,0(src)
> + lwz 7,4(src)
> + addi src,src,8
> + stw 6,0(dst)
> + stw 7,4(dst)
> + addi dst,dst,8
> +4: /* Copy 4~7 bytes. */
> + bf 29,L(tail2)
> + lwz 6,0(src)
> + stw 6,0(dst)
> + bf 30,L(tail5)
> + lhz 7,4(src)
> + sth 7,4(dst)
> + bflr 31
> + lbz 8,6(src)
> + stb 8,6(dst)
> + /* Return original DST pointer. */
> + blr
> +
> +END_GEN_TB (MEMCPY,TB_TOCLESS)
> +libc_hidden_builtin_def (memcpy)
> diff --git a/sysdeps/powerpc/powerpc64/power9/memmove.S b/sysdeps/powerpc/powerpc64/power9/memmove.S
> new file mode 100644
> index 0000000..9ed8f77
> --- /dev/null
> +++ b/sysdeps/powerpc/powerpc64/power9/memmove.S
> @@ -0,0 +1,837 @@
> +/* Optimized memmove implementation for PowerPC64/POWER7.
> + Copyright (C) 2017 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <http://www.gnu.org/licenses/>. */
> +
> +#include <sysdep.h>
> +
> +
> +/* void* [r3] memmove (void *dest [r3], const void *src [r4], size_t len [r5])
> +
> + This optimization check if memory 'dest' overlaps with 'src'. If it does
> + not then it calls an optimized memcpy call (similar to memcpy for POWER7,
> + embedded here to gain some cycles).
> + If source and destiny overlaps, a optimized backwards memcpy is used
> + instead. */
> +
> +#ifndef MEMMOVE
> +# define MEMMOVE memmove
> +#endif
> + .machine power7
> +ENTRY_TOCLESS (MEMMOVE, 5)
> + CALL_MCOUNT 3
> +
> +L(_memmove):
> + subf r9,r4,r3
> + cmpld cr7,r9,r5
> + blt cr7,L(memmove_bwd)
> +
> + cmpldi cr1,r5,31
> + neg 0,3
> + ble cr1, L(copy_LT_32) /* If move < 32 bytes use short move
> + code. */
> +
> + andi. 10,3,15
> + clrldi 11,4,60
> + cmpld cr6,10,11 /* SRC and DST alignments match? */
> +
> + mr r11,3
> + bne cr6,L(copy_GE_32_unaligned)
> + beq L(aligned_copy)
> +
> + mtocrf 0x01,0
> + clrldi 0,0,60
> +
> +/* Get the DST and SRC aligned to 8 bytes (16 for little-endian). */
> +1:
> + bf 31,2f
> + lbz 6,0(r4)
> + addi r4,r4,1
> + stb 6,0(r11)
> + addi r11,r11,1
> +2:
> + bf 30,4f
> + lhz 6,0(r4)
> + addi r4,r4,2
> + sth 6,0(r11)
> + addi r11,r11,2
> +4:
> + bf 29,8f
> + lwz 6,0(r4)
> + addi r4,r4,4
> + stw 6,0(r11)
> + addi r11,r11,4
> +8:
> + bf 28,16f
> + ld 6,0(r4)
> + addi r4,r4,8
> + std 6,0(r11)
> + addi r11,r11,8
> +16:
> + subf r5,0,r5
> +
> +/* Main aligned copy loop. Copies 128 bytes at a time. */
> +L(aligned_copy):
> + li 6,16
> + li 7,32
> + li 8,48
> + mtocrf 0x02,r5
> + srdi 12,r5,7
> + cmpdi 12,0
> + beq L(aligned_tail)
> + lvx 6,0,r4
> + lvx 7,r4,6
> + mtctr 12
> + b L(aligned_128loop)
> +
> + .align 4
> +L(aligned_128head):
> + /* for the 2nd + iteration of this loop. */
> + lvx 6,0,r4
> + lvx 7,r4,6
> +L(aligned_128loop):
> + lvx 8,r4,7
> + lvx 9,r4,8
> + stvx 6,0,r11
> + addi r4,r4,64
> + stvx 7,r11,6
> + stvx 8,r11,7
> + stvx 9,r11,8
> + lvx 6,0,r4
> + lvx 7,r4,6
> + addi r11,r11,64
> + lvx 8,r4,7
> + lvx 9,r4,8
> + addi r4,r4,64
> + stvx 6,0,r11
> + stvx 7,r11,6
> + stvx 8,r11,7
> + stvx 9,r11,8
> + addi r11,r11,64
> + bdnz L(aligned_128head)
> +
> +L(aligned_tail):
> + mtocrf 0x01,r5
> + bf 25,32f
> + lvx 6,0,r4
> + lvx 7,r4,6
> + lvx 8,r4,7
> + lvx 9,r4,8
> + addi r4,r4,64
> + stvx 6,0,r11
> + stvx 7,r11,6
> + stvx 8,r11,7
> + stvx 9,r11,8
> + addi r11,r11,64
> +32:
> + bf 26,16f
> + lvx 6,0,r4
> + lvx 7,r4,6
> + addi r4,r4,32
> + stvx 6,0,r11
> + stvx 7,r11,6
> + addi r11,r11,32
> +16:
> + bf 27,8f
> + lvx 6,0,r4
> + addi r4,r4,16
> + stvx 6,0,r11
> + addi r11,r11,16
> +8:
> + bf 28,4f
> + ld 6,0(r4)
> + addi r4,r4,8
> + std 6,0(r11)
> + addi r11,r11,8
> +4: /* Copies 4~7 bytes. */
> + bf 29,L(tail2)
> + lwz 6,0(r4)
> + stw 6,0(r11)
> + bf 30,L(tail5)
> + lhz 7,4(r4)
> + sth 7,4(r11)
> + bflr 31
> + lbz 8,6(r4)
> + stb 8,6(r11)
> + /* Return original DST pointer. */
> + blr
> +
> +/* Handle copies of 0~31 bytes. */
> + .align 4
> +L(copy_LT_32):
> + mr r11,3
> + cmpldi cr6,r5,8
> + mtocrf 0x01,r5
> + ble cr6,L(copy_LE_8)
> +
> + /* At least 9 bytes to go. */
> + neg 8,4
> + andi. 0,8,3
> + cmpldi cr1,r5,16
> + beq L(copy_LT_32_aligned)
> +
> + /* Force 4-byte alignment for SRC. */
> + mtocrf 0x01,0
> + subf r5,0,r5
> +2:
> + bf 30,1f
> + lhz 6,0(r4)
> + addi r4,r4,2
> + sth 6,0(r11)
> + addi r11,r11,2
> +1:
> + bf 31,L(end_4bytes_alignment)
> + lbz 6,0(r4)
> + addi r4,r4,1
> + stb 6,0(r11)
> + addi r11,r11,1
> +
> + .align 4
> +L(end_4bytes_alignment):
> + cmpldi cr1,r5,16
> + mtocrf 0x01,r5
> +
> +L(copy_LT_32_aligned):
> + /* At least 6 bytes to go, and SRC is word-aligned. */
> + blt cr1,8f
> +
> + /* Copy 16 bytes. */
> + lwz 6,0(r4)
> + lwz 7,4(r4)
> + stw 6,0(r11)
> + lwz 8,8(r4)
> + stw 7,4(r11)
> + lwz 6,12(r4)
> + addi r4,r4,16
> + stw 8,8(r11)
> + stw 6,12(r11)
> + addi r11,r11,16
> +8: /* Copy 8 bytes. */
> + bf 28,L(tail4)
> + lwz 6,0(r4)
> + lwz 7,4(r4)
> + addi r4,r4,8
> + stw 6,0(r11)
> + stw 7,4(r11)
> + addi r11,r11,8
> +
> + .align 4
> +/* Copies 4~7 bytes. */
> +L(tail4):
> + bf 29,L(tail2)
> + lwz 6,0(r4)
> + stw 6,0(r11)
> + bf 30,L(tail5)
> + lhz 7,4(r4)
> + sth 7,4(r11)
> + bflr 31
> + lbz 8,6(r4)
> + stb 8,6(r11)
> + /* Return original DST pointer. */
> + blr
> +
> + .align 4
> +/* Copies 2~3 bytes. */
> +L(tail2):
> + bf 30,1f
> + lhz 6,0(r4)
> + sth 6,0(r11)
> + bflr 31
> + lbz 7,2(r4)
> + stb 7,2(r11)
> + blr
> +
> + .align 4
> +L(tail5):
> + bflr 31
> + lbz 6,4(r4)
> + stb 6,4(r11)
> + blr
> +
> + .align 4
> +1:
> + bflr 31
> + lbz 6,0(r4)
> + stb 6,0(r11)
> + /* Return original DST pointer. */
> + blr
> +
> +/* Handles copies of 0~8 bytes. */
> + .align 4
> +L(copy_LE_8):
> + bne cr6,L(tail4)
> +
> + /* Though we could've used ld/std here, they are still
> + slow for unaligned cases. */
> +
> + lwz 6,0(r4)
> + lwz 7,4(r4)
> + stw 6,0(r11)
> + stw 7,4(r11)
> + blr
> +
> +
> +/* Handle copies of 32+ bytes where DST is aligned (to quadword) but
> + SRC is not. Use aligned quadword loads from SRC, shifted to realign
> + the data, allowing for aligned DST stores. */
> + .align 4
> +L(copy_GE_32_unaligned):
> + clrldi 0,0,60 /* Number of bytes until the 1st r11 quadword. */
> + srdi 9,r5,4 /* Number of full quadwords remaining. */
> +
> + beq L(copy_GE_32_unaligned_cont)
> +
> + /* DST is not quadword aligned, get it aligned. */
> +
> + mtocrf 0x01,0
> + subf r5,0,r5
> +
> + /* Vector instructions work best when proper alignment (16-bytes)
> + is present. Move 0~15 bytes as needed to get DST quadword-aligned. */
> +1:
> + bf 31,2f
> + lbz 6,0(r4)
> + addi r4,r4,1
> + stb 6,0(r11)
> + addi r11,r11,1
> +2:
> + bf 30,4f
> + lhz 6,0(r4)
> + addi r4,r4,2
> + sth 6,0(r11)
> + addi r11,r11,2
> +4:
> + bf 29,8f
> + lwz 6,0(r4)
> + addi r4,r4,4
> + stw 6,0(r11)
> + addi r11,r11,4
> +8:
> + bf 28,0f
> + ld 6,0(r4)
> + addi r4,r4,8
> + std 6,0(r11)
> + addi r11,r11,8
> +0:
> + srdi 9,r5,4 /* Number of full quadwords remaining. */
> +
> + /* The proper alignment is present, it is OK to copy the bytes now. */
> +L(copy_GE_32_unaligned_cont):
> +
> + /* Setup two indexes to speed up the indexed vector operations. */
> + clrldi 10,r5,60
> + li 6,16 /* Index for 16-bytes offsets. */
> + li 7,32 /* Index for 32-bytes offsets. */
> + cmpldi cr1,10,0
> + srdi 8,r5,5 /* Setup the loop counter. */
> + mtocrf 0x01,9
> + cmpldi cr6,9,1
> +#ifdef __LITTLE_ENDIAN__
> + lvsr 5,0,r4
> +#else
> + lvsl 5,0,r4
> +#endif
> + lvx 3,0,r4
> + li 0,0
> + bf 31,L(setup_unaligned_loop)
> +
> + /* Copy another 16 bytes to align to 32-bytes due to the loop. */
> + lvx 4,r4,6
> +#ifdef __LITTLE_ENDIAN__
> + vperm 6,4,3,5
> +#else
> + vperm 6,3,4,5
> +#endif
> + addi r4,r4,16
> + stvx 6,0,r11
> + addi r11,r11,16
> + vor 3,4,4
> + clrrdi 0,r4,60
> +
> +L(setup_unaligned_loop):
> + mtctr 8
> + ble cr6,L(end_unaligned_loop)
> +
> + /* Copy 32 bytes at a time using vector instructions. */
> + .align 4
> +L(unaligned_loop):
> +
> + /* Note: vr6/vr10 may contain data that was already copied,
> + but in order to get proper alignment, we may have to copy
> + some portions again. This is faster than having unaligned
> + vector instructions though. */
> +
> + lvx 4,r4,6
> +#ifdef __LITTLE_ENDIAN__
> + vperm 6,4,3,5
> +#else
> + vperm 6,3,4,5
> +#endif
> + lvx 3,r4,7
> +#ifdef __LITTLE_ENDIAN__
> + vperm 10,3,4,5
> +#else
> + vperm 10,4,3,5
> +#endif
> + addi r4,r4,32
> + stvx 6,0,r11
> + stvx 10,r11,6
> + addi r11,r11,32
> + bdnz L(unaligned_loop)
> +
> + clrrdi 0,r4,60
> +
> + .align 4
> +L(end_unaligned_loop):
> +
> + /* Check for tail bytes. */
> + mtocrf 0x01,r5
> + beqlr cr1
> +
> + add r4,r4,0
> +
> + /* We have 1~15 tail bytes to copy, and DST is quadword aligned. */
> + /* Copy 8 bytes. */
> + bf 28,4f
> + lwz 6,0(r4)
> + lwz 7,4(r4)
> + addi r4,r4,8
> + stw 6,0(r11)
> + stw 7,4(r11)
> + addi r11,r11,8
> +4: /* Copy 4~7 bytes. */
> + bf 29,L(tail2)
> + lwz 6,0(r4)
> + stw 6,0(r11)
> + bf 30,L(tail5)
> + lhz 7,4(r4)
> + sth 7,4(r11)
> + bflr 31
> + lbz 8,6(r4)
> + stb 8,6(r11)
> + /* Return original DST pointer. */
> + blr
> +
> + /* Start to memcpy backward implementation: the algorith first check if
> + src and dest have the same alignment and if it does align both to 16
> + bytes and copy using VSX instructions.
> + If does not, align dest to 16 bytes and use VMX (altivec) instruction
> + to read two 16 bytes at time, shift/permute the bytes read and write
> + aligned to dest. */
> +L(memmove_bwd):
> + cmpldi cr1,r5,31
> + /* Copy is done backwards: update the pointers and check alignment. */
> + add r11,r3,r5
> + add r4,r4,r5
> + mr r0,r11
> + ble cr1, L(copy_LT_32_bwd) /* If move < 32 bytes use short move
> + code. */
> +
> + andi. r10,r11,15 /* Check if r11 is aligned to 16 bytes */
> + clrldi r9,r4,60 /* Check if r4 is aligned to 16 bytes */
> + cmpld cr6,r10,r9 /* SRC and DST alignments match? */
> +
> + bne cr6,L(copy_GE_32_unaligned_bwd)
> + beq L(aligned_copy_bwd)
> +
> + mtocrf 0x01,r0
> + clrldi r0,r0,60
> +
> +/* Get the DST and SRC aligned to 16 bytes. */
> +1:
> + bf 31,2f
> + lbz r6,-1(r4)
> + subi r4,r4,1
> + stb r6,-1(r11)
> + subi r11,r11,1
> +2:
> + bf 30,4f
> + lhz r6,-2(r4)
> + subi r4,r4,2
> + sth r6,-2(r11)
> + subi r11,r11,2
> +4:
> + bf 29,8f
> + lwz r6,-4(r4)
> + subi r4,r4,4
> + stw r6,-4(r11)
> + subi r11,r11,4
> +8:
> + bf 28,16f
> + ld r6,-8(r4)
> + subi r4,r4,8
> + std r6,-8(r11)
> + subi r11,r11,8
> +16:
> + subf r5,0,r5
> +
> +/* Main aligned copy loop. Copies 128 bytes at a time. */
> +L(aligned_copy_bwd):
> + li r6,-16
> + li r7,-32
> + li r8,-48
> + li r9,-64
> + mtocrf 0x02,r5
> + srdi r12,r5,7
> + cmpdi r12,0
> + beq L(aligned_tail_bwd)
> + lvx v6,r4,r6
> + lvx v7,r4,r7
> + mtctr 12
> + b L(aligned_128loop_bwd)
> +
> + .align 4
> +L(aligned_128head_bwd):
> + /* for the 2nd + iteration of this loop. */
> + lvx v6,r4,r6
> + lvx v7,r4,r7
> +L(aligned_128loop_bwd):
> + lvx v8,r4,r8
> + lvx v9,r4,r9
> + stvx v6,r11,r6
> + subi r4,r4,64
> + stvx v7,r11,r7
> + stvx v8,r11,r8
> + stvx v9,r11,r9
> + lvx v6,r4,r6
> + lvx v7,r4,7
> + subi r11,r11,64
> + lvx v8,r4,r8
> + lvx v9,r4,r9
> + subi r4,r4,64
> + stvx v6,r11,r6
> + stvx v7,r11,r7
> + stvx v8,r11,r8
> + stvx v9,r11,r9
> + subi r11,r11,64
> + bdnz L(aligned_128head_bwd)
> +
> +L(aligned_tail_bwd):
> + mtocrf 0x01,r5
> + bf 25,32f
> + lvx v6,r4,r6
> + lvx v7,r4,r7
> + lvx v8,r4,r8
> + lvx v9,r4,r9
> + subi r4,r4,64
> + stvx v6,r11,r6
> + stvx v7,r11,r7
> + stvx v8,r11,r8
> + stvx v9,r11,r9
> + subi r11,r11,64
> +32:
> + bf 26,16f
> + lvx v6,r4,r6
> + lvx v7,r4,r7
> + subi r4,r4,32
> + stvx v6,r11,r6
> + stvx v7,r11,r7
> + subi r11,r11,32
> +16:
> + bf 27,8f
> + lvx v6,r4,r6
> + subi r4,r4,16
> + stvx v6,r11,r6
> + subi r11,r11,16
> +8:
> + bf 28,4f
> + ld r6,-8(r4)
> + subi r4,r4,8
> + std r6,-8(r11)
> + subi r11,r11,8
> +4: /* Copies 4~7 bytes. */
> + bf 29,L(tail2_bwd)
> + lwz r6,-4(r4)
> + stw r6,-4(r11)
> + bf 30,L(tail5_bwd)
> + lhz r7,-6(r4)
> + sth r7,-6(r11)
> + bflr 31
> + lbz r8,-7(r4)
> + stb r8,-7(r11)
> + /* Return original DST pointer. */
> + blr
> +
> +/* Handle copies of 0~31 bytes. */
> + .align 4
> +L(copy_LT_32_bwd):
> + cmpldi cr6,r5,8
> + mtocrf 0x01,r5
> + ble cr6,L(copy_LE_8_bwd)
> +
> + /* At least 9 bytes to go. */
> + neg r8,r4
> + andi. r0,r8,3
> + cmpldi cr1,r5,16
> + beq L(copy_LT_32_aligned_bwd)
> +
> + /* Force 4-byte alignment for SRC. */
> + mtocrf 0x01,0
> + subf r5,0,r5
> +2:
> + bf 30,1f
> + lhz r6,-2(r4)
> + subi r4,r4,2
> + sth r6,-2(r11)
> + subi r11,r11,2
> +1:
> + bf 31,L(end_4bytes_alignment_bwd)
> + lbz 6,-1(r4)
> + subi r4,r4,1
> + stb 6,-1(r11)
> + subi r11,r11,1
> +
> + .align 4
> +L(end_4bytes_alignment_bwd):
> + cmpldi cr1,r5,16
> + mtocrf 0x01,r5
> +
> +L(copy_LT_32_aligned_bwd):
> + /* At least 6 bytes to go, and SRC is word-aligned. */
> + blt cr1,8f
> +
> + /* Copy 16 bytes. */
> + lwz r6,-4(r4)
> + lwz r7,-8(r4)
> + stw r6,-4(r11)
> + lwz r8,-12(r4)
> + stw r7,-8(r11)
> + lwz r6,-16(r4)
> + subi r4,r4,16
> + stw r8,-12(r11)
> + stw r6,-16(r11)
> + subi r11,r11,16
> +8: /* Copy 8 bytes. */
> + bf 28,L(tail4_bwd)
> + lwz r6,-4(r4)
> + lwz r7,-8(r4)
> + subi r4,r4,8
> + stw r6,-4(r11)
> + stw r7,-8(r11)
> + subi r11,r11,8
> +
> + .align 4
> +/* Copies 4~7 bytes. */
> +L(tail4_bwd):
> + bf 29,L(tail2_bwd)
> + lwz 6,-4(r4)
> + stw 6,-4(r11)
> + bf 30,L(tail5_bwd)
> + lhz 7,-6(r4)
> + sth 7,-6(r11)
> + bflr 31
> + lbz 8,-7(r4)
> + stb 8,-7(r11)
> + /* Return original DST pointer. */
> + blr
> +
> + .align 4
> +/* Copies 2~3 bytes. */
> +L(tail2_bwd):
> + bf 30,1f
> + lhz 6,-2(r4)
> + sth 6,-2(r11)
> + bflr 31
> + lbz 7,-3(r4)
> + stb 7,-3(r11)
> + blr
> +
> + .align 4
> +L(tail5_bwd):
> + bflr 31
> + lbz 6,-5(r4)
> + stb 6,-5(r11)
> + blr
> +
> + .align 4
> +1:
> + bflr 31
> + lbz 6,-1(r4)
> + stb 6,-1(r11)
> + /* Return original DST pointer. */
> + blr
> +
> +
> +/* Handles copies of 0~8 bytes. */
> + .align 4
> +L(copy_LE_8_bwd):
> + bne cr6,L(tail4_bwd)
> +
> + /* Though we could've used ld/std here, they are still
> + slow for unaligned cases. */
> + lwz 6,-8(r4)
> + lwz 7,-4(r4)
> + stw 6,-8(r11)
> + stw 7,-4(r11)
> + blr
> +
> +
> +/* Handle copies of 32+ bytes where DST is aligned (to quadword) but
> + SRC is not. Use aligned quadword loads from SRC, shifted to realign
> + the data, allowing for aligned DST stores. */
> + .align 4
> +L(copy_GE_32_unaligned_bwd):
> + andi. r10,r11,15 /* Check alignment of DST against 16 bytes.. */
> + srdi r9,r5,4 /* Number of full quadwords remaining. */
> +
> + beq L(copy_GE_32_unaligned_cont_bwd)
> +
> + /* DST is not quadword aligned and r10 holds the address masked to
> + compare alignments. */
> + mtocrf 0x01,r10
> + subf r5,r10,r5
> +
> + /* Vector instructions work best when proper alignment (16-bytes)
> + is present. Move 0~15 bytes as needed to get DST quadword-aligned. */
> +1:
> + bf 31,2f
> + lbz r6,-1(r4)
> + subi r4,r4,1
> + stb r6,-1(r11)
> + subi r11,r11,1
> +2:
> + bf 30,4f
> + lhz r6,-2(r4)
> + subi r4,r4,2
> + sth r6,-2(r11)
> + subi r11,r11,2
> +4:
> + bf 29,8f
> + lwz r6,-4(r4)
> + subi r4,r4,4
> + stw r6,-4(r11)
> + subi r11,r11,4
> +8:
> + bf 28,0f
> + ld r6,-8(r4)
> + subi r4,r4,8
> + std r6,-8(r11)
> + subi r11,r11,8
> +0:
> + srdi r9,r5,4 /* Number of full quadwords remaining. */
> +
> + /* The proper alignment is present, it is OK to copy the bytes now. */
> +L(copy_GE_32_unaligned_cont_bwd):
> +
> + /* Setup two indexes to speed up the indexed vector operations. */
> + clrldi r10,r5,60
> + li r6,-16 /* Index for 16-bytes offsets. */
> + li r7,-32 /* Index for 32-bytes offsets. */
> + cmpldi cr1,10,0
> + srdi r8,r5,5 /* Setup the loop counter. */
> + mtocrf 0x01,9
> + cmpldi cr6,r9,1
> +#ifdef __LITTLE_ENDIAN__
> + lvsr v5,r0,r4
> +#else
> + lvsl v5,r0,r4
> +#endif
> + lvx v3,0,r4
> + li r0,0
> + bf 31,L(setup_unaligned_loop_bwd)
> +
> + /* Copy another 16 bytes to align to 32-bytes due to the loop. */
> + lvx v4,r4,r6
> +#ifdef __LITTLE_ENDIAN__
> + vperm v6,v3,v4,v5
> +#else
> + vperm v6,v4,v3,v5
> +#endif
> + subi r4,r4,16
> + stvx v6,r11,r6
> + subi r11,r11,16
> + vor v3,v4,v4
> + clrrdi r0,r4,60
> +
> +L(setup_unaligned_loop_bwd):
> + mtctr r8
> + ble cr6,L(end_unaligned_loop_bwd)
> +
> + /* Copy 32 bytes at a time using vector instructions. */
> + .align 4
> +L(unaligned_loop_bwd):
> +
> + /* Note: vr6/vr10 may contain data that was already copied,
> + but in order to get proper alignment, we may have to copy
> + some portions again. This is faster than having unaligned
> + vector instructions though. */
> +
> + lvx v4,r4,r6
> +#ifdef __LITTLE_ENDIAN__
> + vperm v6,v3,v4,v5
> +#else
> + vperm v6,v4,v3,v5
> +#endif
> + lvx v3,r4,r7
> +#ifdef __LITTLE_ENDIAN__
> + vperm v10,v4,v3,v5
> +#else
> + vperm v10,v3,v4,v5
> +#endif
> + subi r4,r4,32
> + stvx v6,r11,r6
> + stvx v10,r11,r7
> + subi r11,r11,32
> + bdnz L(unaligned_loop_bwd)
> +
> + clrrdi r0,r4,60
> +
> + .align 4
> +L(end_unaligned_loop_bwd):
> +
> + /* Check for tail bytes. */
> + mtocrf 0x01,r5
> + beqlr cr1
> +
> + add r4,r4,0
> +
> + /* We have 1~15 tail bytes to copy, and DST is quadword aligned. */
> + /* Copy 8 bytes. */
> + bf 28,4f
> + lwz r6,-4(r4)
> + lwz r7,-8(r4)
> + subi r4,r4,8
> + stw r6,-4(r11)
> + stw r7,-8(r11)
> + subi r11,r11,8
> +4: /* Copy 4~7 bytes. */
> + bf 29,L(tail2_bwd)
> + lwz r6,-4(r4)
> + stw r6,-4(r11)
> + bf 30,L(tail5_bwd)
> + lhz r7,-6(r4)
> + sth r7,-6(r11)
> + bflr 31
> + lbz r8,-7(r4)
> + stb r8,-7(r11)
> + /* Return original DST pointer. */
> + blr
> +END_GEN_TB (MEMMOVE, TB_TOCLESS)
> +libc_hidden_builtin_def (memmove)
> +
> +
> +/* void bcopy(const void *src [r3], void *dest [r4], size_t n [r5])
> + Implemented in this file to avoid linker create a stub function call
> + in the branch to '_memmove'. */
> +ENTRY_TOCLESS (__bcopy)
> + mr r6,r3
> + mr r3,r4
> + mr r4,r6
> + b L(_memmove)
> +END (__bcopy)
> +#ifndef __bcopy
> +weak_alias (__bcopy, bcopy)
> +#endif
>
Florian Weimer <fweimer@redhat.com> writes:
> On 10/19/2017 05:25 PM, Tulio Magno Quites Machado Filho wrote:
>> diff --git a/sysdeps/powerpc/powerpc64/multiarch/bcopy.c b/sysdeps/powerpc/powerpc64/multiarch/bcopy.c
>> index 05d46e2..4a4ee6e 100644
>> --- a/sysdeps/powerpc/powerpc64/multiarch/bcopy.c
>> +++ b/sysdeps/powerpc/powerpc64/multiarch/bcopy.c
>> @@ -22,8 +22,12 @@
>> extern __typeof (bcopy) __bcopy_ppc attribute_hidden;
>> /* __bcopy_power7 symbol is implemented at memmove-power7.S */
>> extern __typeof (bcopy) __bcopy_power7 attribute_hidden;
>> +/* __bcopy_power9 symbol is implemented at memmove-power9.S. */
>> +extern __typeof (bcopy) __bcopy_power9 attribute_hidden;
>>
>> libc_ifunc (bcopy,
>> - (hwcap & PPC_FEATURE_HAS_VSX)
>> + (hwcap2 & PPC_FEATURE2_ARCH_3_00)
>> + ? __bcopy_power9
>> + : (hwcap & PPC_FEATURE_HAS_VSX)
>> ? __bcopy_power7
>> : __bcopy_ppc);
>> diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
>> index 6a88536..9040bbc 100644
>> --- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
>> +++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
>> @@ -51,6 +51,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
>> #ifdef SHARED
>> /* Support sysdeps/powerpc/powerpc64/multiarch/memcpy.c. */
>> IFUNC_IMPL (i, name, memcpy,
>> + IFUNC_IMPL_ADD (array, i, memcpy, hwcap2 & PPC_FEATURE2_ARCH_3_00,
>> + __memcpy_power9)
>> IFUNC_IMPL_ADD (array, i, memcpy, hwcap & PPC_FEATURE_HAS_VSX,
>> __memcpy_power7)
>> IFUNC_IMPL_ADD (array, i, memcpy, hwcap & PPC_FEATURE_ARCH_2_06,
>> @@ -65,6 +67,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
>>
>> /* Support sysdeps/powerpc/powerpc64/multiarch/memmove.c. */
>> IFUNC_IMPL (i, name, memmove,
>> + IFUNC_IMPL_ADD (array, i, memmove, hwcap2 & PPC_FEATURE2_ARCH_3_00,
>> + __memmove_power9)
>> IFUNC_IMPL_ADD (array, i, memmove, hwcap & PPC_FEATURE_HAS_VSX,
>> __memmove_power7)
>> IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_ppc))
>> @@ -168,6 +172,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
>>
>> /* Support sysdeps/powerpc/powerpc64/multiarch/bcopy.c. */
>> IFUNC_IMPL (i, name, bcopy,
>> + IFUNC_IMPL_ADD (array, i, bcopy, hwcap2 & PPC_FEATURE2_ARCH_3_00,
>> + __bcopy_power9)
>> IFUNC_IMPL_ADD (array, i, bcopy, hwcap & PPC_FEATURE_HAS_VSX,
>> __bcopy_power7)
>> IFUNC_IMPL_ADD (array, i, bcopy, 1, __bcopy_ppc))
>
> I'm concerned that this needs *another* change to recognize post-DD2.1
> POWER 9 hardware which has the fix (or perhaps OpenPOWER implementations
> which never had the bug).
>
> Is there any other way you could select the workaround that is more
> specific?
In userspace, fast and simple to use in an IFUNC resolver, no.
AFAIK, it would require to parse /proc/cpuinfo.
> Or is the performance hit from avoiding the affected vector
> instructions not so severe that it would be an immediate concern for
> future silicon?
The instructions Raji chose to use here do not impact the performance on
current silicon and is not expected to impact future silicon as well.
On 10/19/2017 05:56 PM, Tulio Magno Quites Machado Filho wrote:
>> Or is the performance hit from avoiding the affected vector
>> instructions not so severe that it would be an immediate concern for
>> future silicon?
> The instructions Raji chose to use here do not impact the performance on
> current silicon and is not expected to impact future silicon as well.
Okay, this addresses my concern. Unfortunately, I don't feel confident
to review the new implementation myself.
Thanks,
Florian
@@ -1,6 +1,6 @@
ifeq ($(subdir),string)
-sysdep_routines += memcpy-power7 memcpy-a2 memcpy-power6 memcpy-cell \
- memcpy-power4 memcpy-ppc64 \
+sysdep_routines += memcpy-power9 memcpy-power7 memcpy-a2 memcpy-power6 \
+ memcpy-cell memcpy-power4 memcpy-ppc64 \
memcmp-power8 memcmp-power7 memcmp-power4 memcmp-ppc64 \
memset-power7 memset-power6 memset-power4 \
memset-ppc64 memset-power8 \
@@ -24,7 +24,8 @@ sysdep_routines += memcpy-power7 memcpy-a2 memcpy-power6 memcpy-cell \
stpncpy-power8 stpncpy-power7 stpncpy-ppc64 \
strcmp-power9 strcmp-power8 strcmp-power7 strcmp-ppc64 \
strcat-power8 strcat-power7 strcat-ppc64 \
- memmove-power7 memmove-ppc64 wordcopy-ppc64 bcopy-ppc64 \
+ memmove-power9 memmove-power7 memmove-ppc64 \
+ wordcopy-ppc64 bcopy-ppc64 \
strncpy-power8 strstr-power7 strstr-ppc64 \
strspn-power8 strspn-ppc64 strcspn-power8 strcspn-ppc64 \
strlen-power8 strcasestr-power8 strcasestr-ppc64 \
@@ -22,8 +22,12 @@
extern __typeof (bcopy) __bcopy_ppc attribute_hidden;
/* __bcopy_power7 symbol is implemented at memmove-power7.S */
extern __typeof (bcopy) __bcopy_power7 attribute_hidden;
+/* __bcopy_power9 symbol is implemented at memmove-power9.S. */
+extern __typeof (bcopy) __bcopy_power9 attribute_hidden;
libc_ifunc (bcopy,
- (hwcap & PPC_FEATURE_HAS_VSX)
+ (hwcap2 & PPC_FEATURE2_ARCH_3_00)
+ ? __bcopy_power9
+ : (hwcap & PPC_FEATURE_HAS_VSX)
? __bcopy_power7
: __bcopy_ppc);
@@ -51,6 +51,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
#ifdef SHARED
/* Support sysdeps/powerpc/powerpc64/multiarch/memcpy.c. */
IFUNC_IMPL (i, name, memcpy,
+ IFUNC_IMPL_ADD (array, i, memcpy, hwcap2 & PPC_FEATURE2_ARCH_3_00,
+ __memcpy_power9)
IFUNC_IMPL_ADD (array, i, memcpy, hwcap & PPC_FEATURE_HAS_VSX,
__memcpy_power7)
IFUNC_IMPL_ADD (array, i, memcpy, hwcap & PPC_FEATURE_ARCH_2_06,
@@ -65,6 +67,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/powerpc/powerpc64/multiarch/memmove.c. */
IFUNC_IMPL (i, name, memmove,
+ IFUNC_IMPL_ADD (array, i, memmove, hwcap2 & PPC_FEATURE2_ARCH_3_00,
+ __memmove_power9)
IFUNC_IMPL_ADD (array, i, memmove, hwcap & PPC_FEATURE_HAS_VSX,
__memmove_power7)
IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_ppc))
@@ -168,6 +172,8 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
/* Support sysdeps/powerpc/powerpc64/multiarch/bcopy.c. */
IFUNC_IMPL (i, name, bcopy,
+ IFUNC_IMPL_ADD (array, i, bcopy, hwcap2 & PPC_FEATURE2_ARCH_3_00,
+ __bcopy_power9)
IFUNC_IMPL_ADD (array, i, bcopy, hwcap & PPC_FEATURE_HAS_VSX,
__bcopy_power7)
IFUNC_IMPL_ADD (array, i, bcopy, 1, __bcopy_ppc))
new file mode 100644
@@ -0,0 +1,26 @@
+/* Optimized memcpy implementation for PowerPC/POWER9.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+#define MEMCPY __memcpy_power9
+
+#undef libc_hidden_builtin_def
+#define libc_hidden_builtin_def(name)
+
+#include <sysdeps/powerpc/powerpc64/power9/memcpy.S>
@@ -35,8 +35,11 @@ extern __typeof (__redirect_memcpy) __memcpy_cell attribute_hidden;
extern __typeof (__redirect_memcpy) __memcpy_power6 attribute_hidden;
extern __typeof (__redirect_memcpy) __memcpy_a2 attribute_hidden;
extern __typeof (__redirect_memcpy) __memcpy_power7 attribute_hidden;
+extern __typeof (__redirect_memcpy) __memcpy_power9 attribute_hidden;
libc_ifunc (__libc_memcpy,
+ (hwcap2 & PPC_FEATURE2_ARCH_3_00)
+ ? __memcpy_power9 :
(hwcap & PPC_FEATURE_HAS_VSX)
? __memcpy_power7 :
(hwcap & PPC_FEATURE_ARCH_2_06)
@@ -23,7 +23,7 @@
#undef libc_hidden_builtin_def
#define libc_hidden_builtin_def(name)
-#undef bcopy
-#define bcopy __bcopy_power7
+#undef __bcopy
+#define __bcopy __bcopy_power7
#include <sysdeps/powerpc/powerpc64/power7/memmove.S>
new file mode 100644
@@ -0,0 +1,29 @@
+/* Optimized memmove implementation for PowerPC64/POWER7.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+#define MEMMOVE __memmove_power9
+
+#undef libc_hidden_builtin_def
+#define libc_hidden_builtin_def(name)
+
+#undef __bcopy
+#define __bcopy __bcopy_power9
+
+#include <sysdeps/powerpc/powerpc64/power9/memmove.S>
@@ -31,9 +31,12 @@ extern __typeof (__redirect_memmove) __libc_memmove;
extern __typeof (__redirect_memmove) __memmove_ppc attribute_hidden;
extern __typeof (__redirect_memmove) __memmove_power7 attribute_hidden;
+extern __typeof (__redirect_memmove) __memmove_power9 attribute_hidden;
libc_ifunc (__libc_memmove,
- (hwcap & PPC_FEATURE_HAS_VSX)
+ (hwcap2 & PPC_FEATURE2_ARCH_3_00)
+ ? __memmove_power9
+ : (hwcap & PPC_FEATURE_HAS_VSX)
? __memmove_power7
: __memmove_ppc);
@@ -832,4 +832,6 @@ ENTRY_TOCLESS (__bcopy)
mr r4,r6
b L(_memmove)
END (__bcopy)
+#ifndef __bcopy
weak_alias (__bcopy, bcopy)
+#endif
new file mode 100644
@@ -0,0 +1,429 @@
+/* Optimized memcpy implementation for PowerPC64/POWER9.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+
+/* void * [r3] memcpy (void *dst [r3], void *src [r4], size_t len [r5]);
+ Returns 'dst'. */
+
+#ifndef MEMCPY
+# define MEMCPY memcpy
+#endif
+
+#define dst 11 /* Use r11 so r3 kept unchanged. */
+#define src 4
+#define cnt 5
+
+ .machine power7
+ENTRY_TOCLESS (MEMCPY, 5)
+ CALL_MCOUNT 3
+
+ cmpldi cr1,cnt,31
+ neg 0,3
+ ble cr1, L(copy_LT_32) /* If move < 32 bytes use short move
+ code. */
+
+/* Align copies using VSX instructions to quadword. It is to avoid alignment
+ traps when memcpy is used on non-cacheable memory (for instance, memory
+ mapped I/O). */
+ andi. 10,3,15
+ clrldi 11,4,60
+ cmpld cr6,10,11 /* SRC and DST alignments match? */
+
+ mr dst,3
+ bne cr6,L(copy_GE_32_unaligned)
+ beq L(aligned_copy)
+
+ mtocrf 0x01,0
+ clrldi 0,0,60
+
+/* Get the DST and SRC aligned to 16 bytes. */
+1:
+ bf 31,2f
+ lbz 6,0(src)
+ addi src,src,1
+ stb 6,0(dst)
+ addi dst,dst,1
+2:
+ bf 30,4f
+ lhz 6,0(src)
+ addi src,src,2
+ sth 6,0(dst)
+ addi dst,dst,2
+4:
+ bf 29,8f
+ lwz 6,0(src)
+ addi src,src,4
+ stw 6,0(dst)
+ addi dst,dst,4
+8:
+ bf 28,16f
+ ld 6,0(src)
+ addi src,src,8
+ std 6,0(dst)
+ addi dst,dst,8
+16:
+ subf cnt,0,cnt
+
+/* Main aligned copy loop. Copies 128 bytes at a time. */
+L(aligned_copy):
+ li 6,16
+ li 7,32
+ li 8,48
+ mtocrf 0x02,cnt
+ srdi 12,cnt,7
+ cmpdi 12,0
+ beq L(aligned_tail)
+ lvx 6,0,src
+ lvx 7,src,6
+ mtctr 12
+ b L(aligned_128loop)
+
+ .align 4
+L(aligned_128head):
+ /* for the 2nd + iteration of this loop. */
+ lvx 6,0,src
+ lvx 7,src,6
+L(aligned_128loop):
+ lvx 8,src,7
+ lvx 9,src,8
+ stvx 6,0,dst
+ addi src,src,64
+ stvx 7,dst,6
+ stvx 8,dst,7
+ stvx 9,dst,8
+ lvx 6,0,src
+ lvx 7,src,6
+ addi dst,dst,64
+ lvx 8,src,7
+ lvx 9,src,8
+ addi src,src,64
+ stvx 6,0,dst
+ stvx 7,dst,6
+ stvx 8,dst,7
+ stvx 9,dst,8
+ addi dst,dst,64
+ bdnz L(aligned_128head)
+
+L(aligned_tail):
+ mtocrf 0x01,cnt
+ bf 25,32f
+ lvx 6,0,src
+ lvx 7,src,6
+ lvx 8,src,7
+ lvx 9,src,8
+ addi src,src,64
+ stvx 6,0,dst
+ stvx 7,dst,6
+ stvx 8,dst,7
+ stvx 9,dst,8
+ addi dst,dst,64
+32:
+ bf 26,16f
+ lvx 6,0,src
+ lvx 7,src,6
+ addi src,src,32
+ stvx 6,0,dst
+ stvx 7,dst,6
+ addi dst,dst,32
+16:
+ bf 27,8f
+ lvx 6,0,src
+ addi src,src,16
+ stvx 6,0,dst
+ addi dst,dst,16
+8:
+ bf 28,4f
+ ld 6,0(src)
+ addi src,src,8
+ std 6,0(dst)
+ addi dst,dst,8
+4: /* Copies 4~7 bytes. */
+ bf 29,L(tail2)
+ lwz 6,0(src)
+ stw 6,0(dst)
+ bf 30,L(tail5)
+ lhz 7,4(src)
+ sth 7,4(dst)
+ bflr 31
+ lbz 8,6(src)
+ stb 8,6(dst)
+ /* Return original DST pointer. */
+ blr
+
+
+/* Handle copies of 0~31 bytes. */
+ .align 4
+L(copy_LT_32):
+ mr dst,3
+ cmpldi cr6,cnt,8
+ mtocrf 0x01,cnt
+ ble cr6,L(copy_LE_8)
+
+ /* At least 9 bytes to go. */
+ neg 8,4
+ andi. 0,8,3
+ cmpldi cr1,cnt,16
+ beq L(copy_LT_32_aligned)
+
+ /* Force 4-byte alignment for SRC. */
+ mtocrf 0x01,0
+ subf cnt,0,cnt
+2:
+ bf 30,1f
+ lhz 6,0(src)
+ addi src,src,2
+ sth 6,0(dst)
+ addi dst,dst,2
+1:
+ bf 31,L(end_4bytes_alignment)
+ lbz 6,0(src)
+ addi src,src,1
+ stb 6,0(dst)
+ addi dst,dst,1
+
+ .align 4
+L(end_4bytes_alignment):
+ cmpldi cr1,cnt,16
+ mtocrf 0x01,cnt
+
+L(copy_LT_32_aligned):
+ /* At least 6 bytes to go, and SRC is word-aligned. */
+ blt cr1,8f
+
+ /* Copy 16 bytes. */
+ lwz 6,0(src)
+ lwz 7,4(src)
+ stw 6,0(dst)
+ lwz 8,8(src)
+ stw 7,4(dst)
+ lwz 6,12(src)
+ addi src,src,16
+ stw 8,8(dst)
+ stw 6,12(dst)
+ addi dst,dst,16
+8: /* Copy 8 bytes. */
+ bf 28,L(tail4)
+ lwz 6,0(src)
+ lwz 7,4(src)
+ addi src,src,8
+ stw 6,0(dst)
+ stw 7,4(dst)
+ addi dst,dst,8
+
+ .align 4
+/* Copies 4~7 bytes. */
+L(tail4):
+ bf 29,L(tail2)
+ lwz 6,0(src)
+ stw 6,0(dst)
+ bf 30,L(tail5)
+ lhz 7,4(src)
+ sth 7,4(dst)
+ bflr 31
+ lbz 8,6(src)
+ stb 8,6(dst)
+ /* Return original DST pointer. */
+ blr
+
+ .align 4
+/* Copies 2~3 bytes. */
+L(tail2):
+ bf 30,1f
+ lhz 6,0(src)
+ sth 6,0(dst)
+ bflr 31
+ lbz 7,2(src)
+ stb 7,2(dst)
+ blr
+
+ .align 4
+L(tail5):
+ bflr 31
+ lbz 6,4(src)
+ stb 6,4(dst)
+ blr
+
+ .align 4
+1:
+ bflr 31
+ lbz 6,0(src)
+ stb 6,0(dst)
+ /* Return original DST pointer. */
+ blr
+
+
+/* Handles copies of 0~8 bytes. */
+ .align 4
+L(copy_LE_8):
+ bne cr6,L(tail4)
+
+ /* Though we could've used ld/std here, they are still
+ slow for unaligned cases. */
+
+ lwz 6,0(src)
+ lwz 7,4(src)
+ stw 6,0(dst)
+ stw 7,4(dst)
+ blr
+
+
+/* Handle copies of 32+ bytes where DST is aligned (to quadword) but
+ SRC is not. Use aligned quadword loads from SRC, shifted to realign
+ the data, allowing for aligned DST stores. */
+ .align 4
+L(copy_GE_32_unaligned):
+ clrldi 0,0,60 /* Number of bytes until the 1st dst quadword. */
+ srdi 9,cnt,4 /* Number of full quadwords remaining. */
+
+ beq L(copy_GE_32_unaligned_cont)
+
+ /* DST is not quadword aligned, get it aligned. */
+
+ mtocrf 0x01,0
+ subf cnt,0,cnt
+
+ /* Vector instructions work best when proper alignment (16-bytes)
+ is present. Move 0~15 bytes as needed to get DST quadword-aligned. */
+1:
+ bf 31,2f
+ lbz 6,0(src)
+ addi src,src,1
+ stb 6,0(dst)
+ addi dst,dst,1
+2:
+ bf 30,4f
+ lhz 6,0(src)
+ addi src,src,2
+ sth 6,0(dst)
+ addi dst,dst,2
+4:
+ bf 29,8f
+ lwz 6,0(src)
+ addi src,src,4
+ stw 6,0(dst)
+ addi dst,dst,4
+8:
+ bf 28,0f
+ ld 6,0(src)
+ addi src,src,8
+ std 6,0(dst)
+ addi dst,dst,8
+0:
+ srdi 9,cnt,4 /* Number of full quadwords remaining. */
+
+ /* The proper alignment is present, it is OK to copy the bytes now. */
+L(copy_GE_32_unaligned_cont):
+
+ /* Setup two indexes to speed up the indexed vector operations. */
+ clrldi 10,cnt,60
+ li 6,16 /* Index for 16-bytes offsets. */
+ li 7,32 /* Index for 32-bytes offsets. */
+ cmpldi cr1,10,0
+ srdi 8,cnt,5 /* Setup the loop counter. */
+ mtocrf 0x01,9
+ cmpldi cr6,9,1
+#ifdef __LITTLE_ENDIAN__
+ lvsr 5,0,src
+#else
+ lvsl 5,0,src
+#endif
+ lvx 3,0,src
+ li 0,0
+ bf 31,L(setup_unaligned_loop)
+
+ /* Copy another 16 bytes to align to 32-bytes due to the loop. */
+ lvx 4,src,6
+#ifdef __LITTLE_ENDIAN__
+ vperm 6,4,3,5
+#else
+ vperm 6,3,4,5
+#endif
+ addi src,src,16
+ stvx 6,0,dst
+ addi dst,dst,16
+ vor 3,4,4
+ clrrdi 0,src,60
+
+L(setup_unaligned_loop):
+ mtctr 8
+ ble cr6,L(end_unaligned_loop)
+
+ /* Copy 32 bytes at a time using vector instructions. */
+ .align 4
+L(unaligned_loop):
+
+ /* Note: vr6/vr10 may contain data that was already copied,
+ but in order to get proper alignment, we may have to copy
+ some portions again. This is faster than having unaligned
+ vector instructions though. */
+
+ lvx 4,src,6
+#ifdef __LITTLE_ENDIAN__
+ vperm 6,4,3,5
+#else
+ vperm 6,3,4,5
+#endif
+ lvx 3,src,7
+#ifdef __LITTLE_ENDIAN__
+ vperm 10,3,4,5
+#else
+ vperm 10,4,3,5
+#endif
+ addi src,src,32
+ stvx 6,0,dst
+ stvx 10,dst,6
+ addi dst,dst,32
+ bdnz L(unaligned_loop)
+
+ clrrdi 0,src,60
+
+ .align 4
+L(end_unaligned_loop):
+
+ /* Check for tail bytes. */
+ mtocrf 0x01,cnt
+ beqlr cr1
+
+ add src,src,0
+
+ /* We have 1~15 tail bytes to copy, and DST is quadword aligned. */
+ /* Copy 8 bytes. */
+ bf 28,4f
+ lwz 6,0(src)
+ lwz 7,4(src)
+ addi src,src,8
+ stw 6,0(dst)
+ stw 7,4(dst)
+ addi dst,dst,8
+4: /* Copy 4~7 bytes. */
+ bf 29,L(tail2)
+ lwz 6,0(src)
+ stw 6,0(dst)
+ bf 30,L(tail5)
+ lhz 7,4(src)
+ sth 7,4(dst)
+ bflr 31
+ lbz 8,6(src)
+ stb 8,6(dst)
+ /* Return original DST pointer. */
+ blr
+
+END_GEN_TB (MEMCPY,TB_TOCLESS)
+libc_hidden_builtin_def (memcpy)
new file mode 100644
@@ -0,0 +1,837 @@
+/* Optimized memmove implementation for PowerPC64/POWER7.
+ Copyright (C) 2017 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <sysdep.h>
+
+
+/* void* [r3] memmove (void *dest [r3], const void *src [r4], size_t len [r5])
+
+ This optimization check if memory 'dest' overlaps with 'src'. If it does
+ not then it calls an optimized memcpy call (similar to memcpy for POWER7,
+ embedded here to gain some cycles).
+ If source and destiny overlaps, a optimized backwards memcpy is used
+ instead. */
+
+#ifndef MEMMOVE
+# define MEMMOVE memmove
+#endif
+ .machine power7
+ENTRY_TOCLESS (MEMMOVE, 5)
+ CALL_MCOUNT 3
+
+L(_memmove):
+ subf r9,r4,r3
+ cmpld cr7,r9,r5
+ blt cr7,L(memmove_bwd)
+
+ cmpldi cr1,r5,31
+ neg 0,3
+ ble cr1, L(copy_LT_32) /* If move < 32 bytes use short move
+ code. */
+
+ andi. 10,3,15
+ clrldi 11,4,60
+ cmpld cr6,10,11 /* SRC and DST alignments match? */
+
+ mr r11,3
+ bne cr6,L(copy_GE_32_unaligned)
+ beq L(aligned_copy)
+
+ mtocrf 0x01,0
+ clrldi 0,0,60
+
+/* Get the DST and SRC aligned to 8 bytes (16 for little-endian). */
+1:
+ bf 31,2f
+ lbz 6,0(r4)
+ addi r4,r4,1
+ stb 6,0(r11)
+ addi r11,r11,1
+2:
+ bf 30,4f
+ lhz 6,0(r4)
+ addi r4,r4,2
+ sth 6,0(r11)
+ addi r11,r11,2
+4:
+ bf 29,8f
+ lwz 6,0(r4)
+ addi r4,r4,4
+ stw 6,0(r11)
+ addi r11,r11,4
+8:
+ bf 28,16f
+ ld 6,0(r4)
+ addi r4,r4,8
+ std 6,0(r11)
+ addi r11,r11,8
+16:
+ subf r5,0,r5
+
+/* Main aligned copy loop. Copies 128 bytes at a time. */
+L(aligned_copy):
+ li 6,16
+ li 7,32
+ li 8,48
+ mtocrf 0x02,r5
+ srdi 12,r5,7
+ cmpdi 12,0
+ beq L(aligned_tail)
+ lvx 6,0,r4
+ lvx 7,r4,6
+ mtctr 12
+ b L(aligned_128loop)
+
+ .align 4
+L(aligned_128head):
+ /* for the 2nd + iteration of this loop. */
+ lvx 6,0,r4
+ lvx 7,r4,6
+L(aligned_128loop):
+ lvx 8,r4,7
+ lvx 9,r4,8
+ stvx 6,0,r11
+ addi r4,r4,64
+ stvx 7,r11,6
+ stvx 8,r11,7
+ stvx 9,r11,8
+ lvx 6,0,r4
+ lvx 7,r4,6
+ addi r11,r11,64
+ lvx 8,r4,7
+ lvx 9,r4,8
+ addi r4,r4,64
+ stvx 6,0,r11
+ stvx 7,r11,6
+ stvx 8,r11,7
+ stvx 9,r11,8
+ addi r11,r11,64
+ bdnz L(aligned_128head)
+
+L(aligned_tail):
+ mtocrf 0x01,r5
+ bf 25,32f
+ lvx 6,0,r4
+ lvx 7,r4,6
+ lvx 8,r4,7
+ lvx 9,r4,8
+ addi r4,r4,64
+ stvx 6,0,r11
+ stvx 7,r11,6
+ stvx 8,r11,7
+ stvx 9,r11,8
+ addi r11,r11,64
+32:
+ bf 26,16f
+ lvx 6,0,r4
+ lvx 7,r4,6
+ addi r4,r4,32
+ stvx 6,0,r11
+ stvx 7,r11,6
+ addi r11,r11,32
+16:
+ bf 27,8f
+ lvx 6,0,r4
+ addi r4,r4,16
+ stvx 6,0,r11
+ addi r11,r11,16
+8:
+ bf 28,4f
+ ld 6,0(r4)
+ addi r4,r4,8
+ std 6,0(r11)
+ addi r11,r11,8
+4: /* Copies 4~7 bytes. */
+ bf 29,L(tail2)
+ lwz 6,0(r4)
+ stw 6,0(r11)
+ bf 30,L(tail5)
+ lhz 7,4(r4)
+ sth 7,4(r11)
+ bflr 31
+ lbz 8,6(r4)
+ stb 8,6(r11)
+ /* Return original DST pointer. */
+ blr
+
+/* Handle copies of 0~31 bytes. */
+ .align 4
+L(copy_LT_32):
+ mr r11,3
+ cmpldi cr6,r5,8
+ mtocrf 0x01,r5
+ ble cr6,L(copy_LE_8)
+
+ /* At least 9 bytes to go. */
+ neg 8,4
+ andi. 0,8,3
+ cmpldi cr1,r5,16
+ beq L(copy_LT_32_aligned)
+
+ /* Force 4-byte alignment for SRC. */
+ mtocrf 0x01,0
+ subf r5,0,r5
+2:
+ bf 30,1f
+ lhz 6,0(r4)
+ addi r4,r4,2
+ sth 6,0(r11)
+ addi r11,r11,2
+1:
+ bf 31,L(end_4bytes_alignment)
+ lbz 6,0(r4)
+ addi r4,r4,1
+ stb 6,0(r11)
+ addi r11,r11,1
+
+ .align 4
+L(end_4bytes_alignment):
+ cmpldi cr1,r5,16
+ mtocrf 0x01,r5
+
+L(copy_LT_32_aligned):
+ /* At least 6 bytes to go, and SRC is word-aligned. */
+ blt cr1,8f
+
+ /* Copy 16 bytes. */
+ lwz 6,0(r4)
+ lwz 7,4(r4)
+ stw 6,0(r11)
+ lwz 8,8(r4)
+ stw 7,4(r11)
+ lwz 6,12(r4)
+ addi r4,r4,16
+ stw 8,8(r11)
+ stw 6,12(r11)
+ addi r11,r11,16
+8: /* Copy 8 bytes. */
+ bf 28,L(tail4)
+ lwz 6,0(r4)
+ lwz 7,4(r4)
+ addi r4,r4,8
+ stw 6,0(r11)
+ stw 7,4(r11)
+ addi r11,r11,8
+
+ .align 4
+/* Copies 4~7 bytes. */
+L(tail4):
+ bf 29,L(tail2)
+ lwz 6,0(r4)
+ stw 6,0(r11)
+ bf 30,L(tail5)
+ lhz 7,4(r4)
+ sth 7,4(r11)
+ bflr 31
+ lbz 8,6(r4)
+ stb 8,6(r11)
+ /* Return original DST pointer. */
+ blr
+
+ .align 4
+/* Copies 2~3 bytes. */
+L(tail2):
+ bf 30,1f
+ lhz 6,0(r4)
+ sth 6,0(r11)
+ bflr 31
+ lbz 7,2(r4)
+ stb 7,2(r11)
+ blr
+
+ .align 4
+L(tail5):
+ bflr 31
+ lbz 6,4(r4)
+ stb 6,4(r11)
+ blr
+
+ .align 4
+1:
+ bflr 31
+ lbz 6,0(r4)
+ stb 6,0(r11)
+ /* Return original DST pointer. */
+ blr
+
+/* Handles copies of 0~8 bytes. */
+ .align 4
+L(copy_LE_8):
+ bne cr6,L(tail4)
+
+ /* Though we could've used ld/std here, they are still
+ slow for unaligned cases. */
+
+ lwz 6,0(r4)
+ lwz 7,4(r4)
+ stw 6,0(r11)
+ stw 7,4(r11)
+ blr
+
+
+/* Handle copies of 32+ bytes where DST is aligned (to quadword) but
+ SRC is not. Use aligned quadword loads from SRC, shifted to realign
+ the data, allowing for aligned DST stores. */
+ .align 4
+L(copy_GE_32_unaligned):
+ clrldi 0,0,60 /* Number of bytes until the 1st r11 quadword. */
+ srdi 9,r5,4 /* Number of full quadwords remaining. */
+
+ beq L(copy_GE_32_unaligned_cont)
+
+ /* DST is not quadword aligned, get it aligned. */
+
+ mtocrf 0x01,0
+ subf r5,0,r5
+
+ /* Vector instructions work best when proper alignment (16-bytes)
+ is present. Move 0~15 bytes as needed to get DST quadword-aligned. */
+1:
+ bf 31,2f
+ lbz 6,0(r4)
+ addi r4,r4,1
+ stb 6,0(r11)
+ addi r11,r11,1
+2:
+ bf 30,4f
+ lhz 6,0(r4)
+ addi r4,r4,2
+ sth 6,0(r11)
+ addi r11,r11,2
+4:
+ bf 29,8f
+ lwz 6,0(r4)
+ addi r4,r4,4
+ stw 6,0(r11)
+ addi r11,r11,4
+8:
+ bf 28,0f
+ ld 6,0(r4)
+ addi r4,r4,8
+ std 6,0(r11)
+ addi r11,r11,8
+0:
+ srdi 9,r5,4 /* Number of full quadwords remaining. */
+
+ /* The proper alignment is present, it is OK to copy the bytes now. */
+L(copy_GE_32_unaligned_cont):
+
+ /* Setup two indexes to speed up the indexed vector operations. */
+ clrldi 10,r5,60
+ li 6,16 /* Index for 16-bytes offsets. */
+ li 7,32 /* Index for 32-bytes offsets. */
+ cmpldi cr1,10,0
+ srdi 8,r5,5 /* Setup the loop counter. */
+ mtocrf 0x01,9
+ cmpldi cr6,9,1
+#ifdef __LITTLE_ENDIAN__
+ lvsr 5,0,r4
+#else
+ lvsl 5,0,r4
+#endif
+ lvx 3,0,r4
+ li 0,0
+ bf 31,L(setup_unaligned_loop)
+
+ /* Copy another 16 bytes to align to 32-bytes due to the loop. */
+ lvx 4,r4,6
+#ifdef __LITTLE_ENDIAN__
+ vperm 6,4,3,5
+#else
+ vperm 6,3,4,5
+#endif
+ addi r4,r4,16
+ stvx 6,0,r11
+ addi r11,r11,16
+ vor 3,4,4
+ clrrdi 0,r4,60
+
+L(setup_unaligned_loop):
+ mtctr 8
+ ble cr6,L(end_unaligned_loop)
+
+ /* Copy 32 bytes at a time using vector instructions. */
+ .align 4
+L(unaligned_loop):
+
+ /* Note: vr6/vr10 may contain data that was already copied,
+ but in order to get proper alignment, we may have to copy
+ some portions again. This is faster than having unaligned
+ vector instructions though. */
+
+ lvx 4,r4,6
+#ifdef __LITTLE_ENDIAN__
+ vperm 6,4,3,5
+#else
+ vperm 6,3,4,5
+#endif
+ lvx 3,r4,7
+#ifdef __LITTLE_ENDIAN__
+ vperm 10,3,4,5
+#else
+ vperm 10,4,3,5
+#endif
+ addi r4,r4,32
+ stvx 6,0,r11
+ stvx 10,r11,6
+ addi r11,r11,32
+ bdnz L(unaligned_loop)
+
+ clrrdi 0,r4,60
+
+ .align 4
+L(end_unaligned_loop):
+
+ /* Check for tail bytes. */
+ mtocrf 0x01,r5
+ beqlr cr1
+
+ add r4,r4,0
+
+ /* We have 1~15 tail bytes to copy, and DST is quadword aligned. */
+ /* Copy 8 bytes. */
+ bf 28,4f
+ lwz 6,0(r4)
+ lwz 7,4(r4)
+ addi r4,r4,8
+ stw 6,0(r11)
+ stw 7,4(r11)
+ addi r11,r11,8
+4: /* Copy 4~7 bytes. */
+ bf 29,L(tail2)
+ lwz 6,0(r4)
+ stw 6,0(r11)
+ bf 30,L(tail5)
+ lhz 7,4(r4)
+ sth 7,4(r11)
+ bflr 31
+ lbz 8,6(r4)
+ stb 8,6(r11)
+ /* Return original DST pointer. */
+ blr
+
+ /* Start to memcpy backward implementation: the algorith first check if
+ src and dest have the same alignment and if it does align both to 16
+ bytes and copy using VSX instructions.
+ If does not, align dest to 16 bytes and use VMX (altivec) instruction
+ to read two 16 bytes at time, shift/permute the bytes read and write
+ aligned to dest. */
+L(memmove_bwd):
+ cmpldi cr1,r5,31
+ /* Copy is done backwards: update the pointers and check alignment. */
+ add r11,r3,r5
+ add r4,r4,r5
+ mr r0,r11
+ ble cr1, L(copy_LT_32_bwd) /* If move < 32 bytes use short move
+ code. */
+
+ andi. r10,r11,15 /* Check if r11 is aligned to 16 bytes */
+ clrldi r9,r4,60 /* Check if r4 is aligned to 16 bytes */
+ cmpld cr6,r10,r9 /* SRC and DST alignments match? */
+
+ bne cr6,L(copy_GE_32_unaligned_bwd)
+ beq L(aligned_copy_bwd)
+
+ mtocrf 0x01,r0
+ clrldi r0,r0,60
+
+/* Get the DST and SRC aligned to 16 bytes. */
+1:
+ bf 31,2f
+ lbz r6,-1(r4)
+ subi r4,r4,1
+ stb r6,-1(r11)
+ subi r11,r11,1
+2:
+ bf 30,4f
+ lhz r6,-2(r4)
+ subi r4,r4,2
+ sth r6,-2(r11)
+ subi r11,r11,2
+4:
+ bf 29,8f
+ lwz r6,-4(r4)
+ subi r4,r4,4
+ stw r6,-4(r11)
+ subi r11,r11,4
+8:
+ bf 28,16f
+ ld r6,-8(r4)
+ subi r4,r4,8
+ std r6,-8(r11)
+ subi r11,r11,8
+16:
+ subf r5,0,r5
+
+/* Main aligned copy loop. Copies 128 bytes at a time. */
+L(aligned_copy_bwd):
+ li r6,-16
+ li r7,-32
+ li r8,-48
+ li r9,-64
+ mtocrf 0x02,r5
+ srdi r12,r5,7
+ cmpdi r12,0
+ beq L(aligned_tail_bwd)
+ lvx v6,r4,r6
+ lvx v7,r4,r7
+ mtctr 12
+ b L(aligned_128loop_bwd)
+
+ .align 4
+L(aligned_128head_bwd):
+ /* for the 2nd + iteration of this loop. */
+ lvx v6,r4,r6
+ lvx v7,r4,r7
+L(aligned_128loop_bwd):
+ lvx v8,r4,r8
+ lvx v9,r4,r9
+ stvx v6,r11,r6
+ subi r4,r4,64
+ stvx v7,r11,r7
+ stvx v8,r11,r8
+ stvx v9,r11,r9
+ lvx v6,r4,r6
+ lvx v7,r4,7
+ subi r11,r11,64
+ lvx v8,r4,r8
+ lvx v9,r4,r9
+ subi r4,r4,64
+ stvx v6,r11,r6
+ stvx v7,r11,r7
+ stvx v8,r11,r8
+ stvx v9,r11,r9
+ subi r11,r11,64
+ bdnz L(aligned_128head_bwd)
+
+L(aligned_tail_bwd):
+ mtocrf 0x01,r5
+ bf 25,32f
+ lvx v6,r4,r6
+ lvx v7,r4,r7
+ lvx v8,r4,r8
+ lvx v9,r4,r9
+ subi r4,r4,64
+ stvx v6,r11,r6
+ stvx v7,r11,r7
+ stvx v8,r11,r8
+ stvx v9,r11,r9
+ subi r11,r11,64
+32:
+ bf 26,16f
+ lvx v6,r4,r6
+ lvx v7,r4,r7
+ subi r4,r4,32
+ stvx v6,r11,r6
+ stvx v7,r11,r7
+ subi r11,r11,32
+16:
+ bf 27,8f
+ lvx v6,r4,r6
+ subi r4,r4,16
+ stvx v6,r11,r6
+ subi r11,r11,16
+8:
+ bf 28,4f
+ ld r6,-8(r4)
+ subi r4,r4,8
+ std r6,-8(r11)
+ subi r11,r11,8
+4: /* Copies 4~7 bytes. */
+ bf 29,L(tail2_bwd)
+ lwz r6,-4(r4)
+ stw r6,-4(r11)
+ bf 30,L(tail5_bwd)
+ lhz r7,-6(r4)
+ sth r7,-6(r11)
+ bflr 31
+ lbz r8,-7(r4)
+ stb r8,-7(r11)
+ /* Return original DST pointer. */
+ blr
+
+/* Handle copies of 0~31 bytes. */
+ .align 4
+L(copy_LT_32_bwd):
+ cmpldi cr6,r5,8
+ mtocrf 0x01,r5
+ ble cr6,L(copy_LE_8_bwd)
+
+ /* At least 9 bytes to go. */
+ neg r8,r4
+ andi. r0,r8,3
+ cmpldi cr1,r5,16
+ beq L(copy_LT_32_aligned_bwd)
+
+ /* Force 4-byte alignment for SRC. */
+ mtocrf 0x01,0
+ subf r5,0,r5
+2:
+ bf 30,1f
+ lhz r6,-2(r4)
+ subi r4,r4,2
+ sth r6,-2(r11)
+ subi r11,r11,2
+1:
+ bf 31,L(end_4bytes_alignment_bwd)
+ lbz 6,-1(r4)
+ subi r4,r4,1
+ stb 6,-1(r11)
+ subi r11,r11,1
+
+ .align 4
+L(end_4bytes_alignment_bwd):
+ cmpldi cr1,r5,16
+ mtocrf 0x01,r5
+
+L(copy_LT_32_aligned_bwd):
+ /* At least 6 bytes to go, and SRC is word-aligned. */
+ blt cr1,8f
+
+ /* Copy 16 bytes. */
+ lwz r6,-4(r4)
+ lwz r7,-8(r4)
+ stw r6,-4(r11)
+ lwz r8,-12(r4)
+ stw r7,-8(r11)
+ lwz r6,-16(r4)
+ subi r4,r4,16
+ stw r8,-12(r11)
+ stw r6,-16(r11)
+ subi r11,r11,16
+8: /* Copy 8 bytes. */
+ bf 28,L(tail4_bwd)
+ lwz r6,-4(r4)
+ lwz r7,-8(r4)
+ subi r4,r4,8
+ stw r6,-4(r11)
+ stw r7,-8(r11)
+ subi r11,r11,8
+
+ .align 4
+/* Copies 4~7 bytes. */
+L(tail4_bwd):
+ bf 29,L(tail2_bwd)
+ lwz 6,-4(r4)
+ stw 6,-4(r11)
+ bf 30,L(tail5_bwd)
+ lhz 7,-6(r4)
+ sth 7,-6(r11)
+ bflr 31
+ lbz 8,-7(r4)
+ stb 8,-7(r11)
+ /* Return original DST pointer. */
+ blr
+
+ .align 4
+/* Copies 2~3 bytes. */
+L(tail2_bwd):
+ bf 30,1f
+ lhz 6,-2(r4)
+ sth 6,-2(r11)
+ bflr 31
+ lbz 7,-3(r4)
+ stb 7,-3(r11)
+ blr
+
+ .align 4
+L(tail5_bwd):
+ bflr 31
+ lbz 6,-5(r4)
+ stb 6,-5(r11)
+ blr
+
+ .align 4
+1:
+ bflr 31
+ lbz 6,-1(r4)
+ stb 6,-1(r11)
+ /* Return original DST pointer. */
+ blr
+
+
+/* Handles copies of 0~8 bytes. */
+ .align 4
+L(copy_LE_8_bwd):
+ bne cr6,L(tail4_bwd)
+
+ /* Though we could've used ld/std here, they are still
+ slow for unaligned cases. */
+ lwz 6,-8(r4)
+ lwz 7,-4(r4)
+ stw 6,-8(r11)
+ stw 7,-4(r11)
+ blr
+
+
+/* Handle copies of 32+ bytes where DST is aligned (to quadword) but
+ SRC is not. Use aligned quadword loads from SRC, shifted to realign
+ the data, allowing for aligned DST stores. */
+ .align 4
+L(copy_GE_32_unaligned_bwd):
+ andi. r10,r11,15 /* Check alignment of DST against 16 bytes.. */
+ srdi r9,r5,4 /* Number of full quadwords remaining. */
+
+ beq L(copy_GE_32_unaligned_cont_bwd)
+
+ /* DST is not quadword aligned and r10 holds the address masked to
+ compare alignments. */
+ mtocrf 0x01,r10
+ subf r5,r10,r5
+
+ /* Vector instructions work best when proper alignment (16-bytes)
+ is present. Move 0~15 bytes as needed to get DST quadword-aligned. */
+1:
+ bf 31,2f
+ lbz r6,-1(r4)
+ subi r4,r4,1
+ stb r6,-1(r11)
+ subi r11,r11,1
+2:
+ bf 30,4f
+ lhz r6,-2(r4)
+ subi r4,r4,2
+ sth r6,-2(r11)
+ subi r11,r11,2
+4:
+ bf 29,8f
+ lwz r6,-4(r4)
+ subi r4,r4,4
+ stw r6,-4(r11)
+ subi r11,r11,4
+8:
+ bf 28,0f
+ ld r6,-8(r4)
+ subi r4,r4,8
+ std r6,-8(r11)
+ subi r11,r11,8
+0:
+ srdi r9,r5,4 /* Number of full quadwords remaining. */
+
+ /* The proper alignment is present, it is OK to copy the bytes now. */
+L(copy_GE_32_unaligned_cont_bwd):
+
+ /* Setup two indexes to speed up the indexed vector operations. */
+ clrldi r10,r5,60
+ li r6,-16 /* Index for 16-bytes offsets. */
+ li r7,-32 /* Index for 32-bytes offsets. */
+ cmpldi cr1,10,0
+ srdi r8,r5,5 /* Setup the loop counter. */
+ mtocrf 0x01,9
+ cmpldi cr6,r9,1
+#ifdef __LITTLE_ENDIAN__
+ lvsr v5,r0,r4
+#else
+ lvsl v5,r0,r4
+#endif
+ lvx v3,0,r4
+ li r0,0
+ bf 31,L(setup_unaligned_loop_bwd)
+
+ /* Copy another 16 bytes to align to 32-bytes due to the loop. */
+ lvx v4,r4,r6
+#ifdef __LITTLE_ENDIAN__
+ vperm v6,v3,v4,v5
+#else
+ vperm v6,v4,v3,v5
+#endif
+ subi r4,r4,16
+ stvx v6,r11,r6
+ subi r11,r11,16
+ vor v3,v4,v4
+ clrrdi r0,r4,60
+
+L(setup_unaligned_loop_bwd):
+ mtctr r8
+ ble cr6,L(end_unaligned_loop_bwd)
+
+ /* Copy 32 bytes at a time using vector instructions. */
+ .align 4
+L(unaligned_loop_bwd):
+
+ /* Note: vr6/vr10 may contain data that was already copied,
+ but in order to get proper alignment, we may have to copy
+ some portions again. This is faster than having unaligned
+ vector instructions though. */
+
+ lvx v4,r4,r6
+#ifdef __LITTLE_ENDIAN__
+ vperm v6,v3,v4,v5
+#else
+ vperm v6,v4,v3,v5
+#endif
+ lvx v3,r4,r7
+#ifdef __LITTLE_ENDIAN__
+ vperm v10,v4,v3,v5
+#else
+ vperm v10,v3,v4,v5
+#endif
+ subi r4,r4,32
+ stvx v6,r11,r6
+ stvx v10,r11,r7
+ subi r11,r11,32
+ bdnz L(unaligned_loop_bwd)
+
+ clrrdi r0,r4,60
+
+ .align 4
+L(end_unaligned_loop_bwd):
+
+ /* Check for tail bytes. */
+ mtocrf 0x01,r5
+ beqlr cr1
+
+ add r4,r4,0
+
+ /* We have 1~15 tail bytes to copy, and DST is quadword aligned. */
+ /* Copy 8 bytes. */
+ bf 28,4f
+ lwz r6,-4(r4)
+ lwz r7,-8(r4)
+ subi r4,r4,8
+ stw r6,-4(r11)
+ stw r7,-8(r11)
+ subi r11,r11,8
+4: /* Copy 4~7 bytes. */
+ bf 29,L(tail2_bwd)
+ lwz r6,-4(r4)
+ stw r6,-4(r11)
+ bf 30,L(tail5_bwd)
+ lhz r7,-6(r4)
+ sth r7,-6(r11)
+ bflr 31
+ lbz r8,-7(r4)
+ stb r8,-7(r11)
+ /* Return original DST pointer. */
+ blr
+END_GEN_TB (MEMMOVE, TB_TOCLESS)
+libc_hidden_builtin_def (memmove)
+
+
+/* void bcopy(const void *src [r3], void *dest [r4], size_t n [r5])
+ Implemented in this file to avoid linker create a stub function call
+ in the branch to '_memmove'. */
+ENTRY_TOCLESS (__bcopy)
+ mr r6,r3
+ mr r3,r4
+ mr r4,r6
+ b L(_memmove)
+END (__bcopy)
+#ifndef __bcopy
+weak_alias (__bcopy, bcopy)
+#endif