From patchwork Fri Mar 25 15:14:54 2016 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Lu, Hongjiu" X-Patchwork-Id: 11519 Received: (qmail 32108 invoked by alias); 25 Mar 2016 15:15:07 -0000 Mailing-List: contact libc-alpha-help@sourceware.org; run by ezmlm Precedence: bulk List-Id: List-Unsubscribe: List-Subscribe: List-Archive: List-Post: List-Help: , Sender: libc-alpha-owner@sourceware.org Delivered-To: mailing list libc-alpha@sourceware.org Received: (qmail 32088 invoked by uid 89); 25 Mar 2016 15:15:06 -0000 Authentication-Results: sourceware.org; auth=none X-Virus-Found: No X-Spam-SWARE-Status: No, score=2.0 required=5.0 tests=AWL, BAYES_99, KAM_LAZY_DOMAIN_SECURITY, NO_DNS_FOR_FROM, RP_MATCHES_RCVD autolearn=no version=3.3.2 spammy=rcx, jae, sk:__mempc, MEMCPY X-HELO: mga04.intel.com X-ExtLoop1: 1 Date: Fri, 25 Mar 2016 08:14:54 -0700 From: "H.J. Lu" To: GNU C Library Cc: Andrew Senkevich Subject: [PATCH] Make __memcpy_avx512_no_vzeroupper an alias Message-ID: <20160325151454.GA331@intel.com> Reply-To: "H.J. Lu" MIME-Version: 1.0 Content-Disposition: inline User-Agent: Mutt/1.5.24 (2015-08-30) Since x86-64 memcpy-avx512-no-vzeroupper.S implements memmove, we can make __memcpy_avx512_no_vzeroupper an alias of __memmove_avx512_no_vzeroupper to reduce code size of libc.so. Tested on AVX512 machine. Comments? Feedbacks? H.J. --- * sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Remove memcpy-avx512-no-vzeroupper. * sysdeps/x86_64/multiarch/memcpy-avx512-no-vzeroupper.S: Renamed to ... * sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S: This. (MEMCPY): Don't define. (MEMCPY_CHK): Likewise. (MEMPCPY): Likewise. (MEMPCPY_CHK): Likewise. (MEMPCPY_CHK): Renamed to ... (__mempcpy_chk_avx512_no_vzeroupper): This. (MEMPCPY_CHK): Renamed to ... (__mempcpy_chk_avx512_no_vzeroupper): This. (MEMCPY_CHK): Renamed to ... (__memmove_chk_avx512_no_vzeroupper): This. (MEMCPY): Renamed to ... (__memmove_avx512_no_vzeroupper): This. (__memcpy_avx512_no_vzeroupper): New alias. (__memcpy_chk_avx512_no_vzeroupper): Likewise. --- sysdeps/x86_64/multiarch/Makefile | 2 +- .../x86_64/multiarch/memcpy-avx512-no-vzeroupper.S | 424 --------------------- .../multiarch/memmove-avx512-no-vzeroupper.S | 408 +++++++++++++++++++- 3 files changed, 404 insertions(+), 430 deletions(-) delete mode 100644 sysdeps/x86_64/multiarch/memcpy-avx512-no-vzeroupper.S diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile index 39c0905..7fc89c2 100644 --- a/sysdeps/x86_64/multiarch/Makefile +++ b/sysdeps/x86_64/multiarch/Makefile @@ -8,7 +8,7 @@ ifeq ($(subdir),string) sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \ strcmp-sse2-unaligned strncmp-ssse3 \ memcmp-sse4 memcpy-ssse3 memcpy-sse2-unaligned \ - memcpy-avx512-no-vzeroupper memmove-ssse3 \ + memmove-ssse3 \ memcpy-ssse3-back memmove-avx-unaligned \ memcpy-avx-unaligned \ memmove-ssse3-back \ diff --git a/sysdeps/x86_64/multiarch/memcpy-avx512-no-vzeroupper.S b/sysdeps/x86_64/multiarch/memcpy-avx512-no-vzeroupper.S deleted file mode 100644 index 285bb83..0000000 --- a/sysdeps/x86_64/multiarch/memcpy-avx512-no-vzeroupper.S +++ /dev/null @@ -1,424 +0,0 @@ -/* memcpy optimized with AVX512 for KNL hardware. - Copyright (C) 2016 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include - -#if defined HAVE_AVX512_ASM_SUPPORT && IS_IN (libc) \ - && (defined SHARED \ - || defined USE_AS_MEMMOVE \ - || !defined USE_MULTIARCH) - -#include "asm-syntax.h" -#ifndef MEMCPY -# define MEMCPY __memcpy_avx512_no_vzeroupper -# define MEMCPY_CHK __memcpy_chk_avx512_no_vzeroupper -# define MEMPCPY __mempcpy_avx512_no_vzeroupper -# define MEMPCPY_CHK __mempcpy_chk_avx512_no_vzeroupper -#endif - - .section .text.avx512,"ax",@progbits -#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE -ENTRY (MEMPCPY_CHK) - cmpq %rdx, %rcx - jb HIDDEN_JUMPTARGET (__chk_fail) -END (MEMPCPY_CHK) - -ENTRY (MEMPCPY) - movq %rdi, %rax - addq %rdx, %rax - jmp L(start) -END (MEMPCPY) -#endif - -#if !defined USE_AS_BCOPY -ENTRY (MEMCPY_CHK) - cmpq %rdx, %rcx - jb HIDDEN_JUMPTARGET (__chk_fail) -END (MEMCPY_CHK) -#endif - -ENTRY (MEMCPY) - mov %rdi, %rax -#ifdef USE_AS_MEMPCPY - add %rdx, %rax -#endif -L(start): - lea (%rsi, %rdx), %rcx - lea (%rdi, %rdx), %r9 - cmp $512, %rdx - ja L(512bytesormore) - -L(check): - cmp $16, %rdx - jbe L(less_16bytes) - cmp $256, %rdx - jb L(less_256bytes) - vmovups (%rsi), %zmm0 - vmovups 0x40(%rsi), %zmm1 - vmovups 0x80(%rsi), %zmm2 - vmovups 0xC0(%rsi), %zmm3 - vmovups -0x100(%rcx), %zmm4 - vmovups -0xC0(%rcx), %zmm5 - vmovups -0x80(%rcx), %zmm6 - vmovups -0x40(%rcx), %zmm7 - vmovups %zmm0, (%rdi) - vmovups %zmm1, 0x40(%rdi) - vmovups %zmm2, 0x80(%rdi) - vmovups %zmm3, 0xC0(%rdi) - vmovups %zmm4, -0x100(%r9) - vmovups %zmm5, -0xC0(%r9) - vmovups %zmm6, -0x80(%r9) - vmovups %zmm7, -0x40(%r9) - ret - -L(less_256bytes): - cmp $128, %dl - jb L(less_128bytes) - vmovups (%rsi), %zmm0 - vmovups 0x40(%rsi), %zmm1 - vmovups -0x80(%rcx), %zmm2 - vmovups -0x40(%rcx), %zmm3 - vmovups %zmm0, (%rdi) - vmovups %zmm1, 0x40(%rdi) - vmovups %zmm2, -0x80(%r9) - vmovups %zmm3, -0x40(%r9) - ret - -L(less_128bytes): - cmp $64, %dl - jb L(less_64bytes) - vmovdqu (%rsi), %ymm0 - vmovdqu 0x20(%rsi), %ymm1 - vmovdqu -0x40(%rcx), %ymm2 - vmovdqu -0x20(%rcx), %ymm3 - vmovdqu %ymm0, (%rdi) - vmovdqu %ymm1, 0x20(%rdi) - vmovdqu %ymm2, -0x40(%r9) - vmovdqu %ymm3, -0x20(%r9) - ret - -L(less_64bytes): - cmp $32, %dl - jb L(less_32bytes) - vmovdqu (%rsi), %ymm0 - vmovdqu -0x20(%rcx), %ymm1 - vmovdqu %ymm0, (%rdi) - vmovdqu %ymm1, -0x20(%r9) - ret - -L(less_32bytes): - vmovdqu (%rsi), %xmm0 - vmovdqu -0x10(%rcx), %xmm1 - vmovdqu %xmm0, (%rdi) - vmovdqu %xmm1, -0x10(%r9) - ret - -L(less_16bytes): - cmp $8, %dl - jb L(less_8bytes) - movq (%rsi), %rsi - movq -0x8(%rcx), %rcx - movq %rsi, (%rdi) - movq %rcx, -0x8(%r9) - ret - -L(less_8bytes): - cmp $4, %dl - jb L(less_4bytes) - mov (%rsi), %esi - mov -0x4(%rcx), %ecx - mov %esi, (%rdi) - mov %ecx, -0x4(%r9) - ret - -L(less_4bytes): - cmp $2, %dl - jb L(less_2bytes) - mov (%rsi), %si - mov -0x2(%rcx), %cx - mov %si, (%rdi) - mov %cx, -0x2(%r9) - ret - -L(less_2bytes): - cmp $1, %dl - jb L(less_1bytes) - mov (%rsi), %cl - mov %cl, (%rdi) -L(less_1bytes): - ret - -L(512bytesormore): -#ifdef SHARED_CACHE_SIZE_HALF - mov $SHARED_CACHE_SIZE_HALF, %r8 -#else - mov __x86_shared_cache_size_half(%rip), %r8 -#endif - cmp %r8, %rdx - jae L(preloop_large) - cmp $1024, %rdx - ja L(1024bytesormore) - prefetcht1 (%rsi) - prefetcht1 0x40(%rsi) - prefetcht1 0x80(%rsi) - prefetcht1 0xC0(%rsi) - prefetcht1 0x100(%rsi) - prefetcht1 0x140(%rsi) - prefetcht1 0x180(%rsi) - prefetcht1 0x1C0(%rsi) - prefetcht1 -0x200(%rcx) - prefetcht1 -0x1C0(%rcx) - prefetcht1 -0x180(%rcx) - prefetcht1 -0x140(%rcx) - prefetcht1 -0x100(%rcx) - prefetcht1 -0xC0(%rcx) - prefetcht1 -0x80(%rcx) - prefetcht1 -0x40(%rcx) - vmovups (%rsi), %zmm0 - vmovups 0x40(%rsi), %zmm1 - vmovups 0x80(%rsi), %zmm2 - vmovups 0xC0(%rsi), %zmm3 - vmovups 0x100(%rsi), %zmm4 - vmovups 0x140(%rsi), %zmm5 - vmovups 0x180(%rsi), %zmm6 - vmovups 0x1C0(%rsi), %zmm7 - vmovups -0x200(%rcx), %zmm8 - vmovups -0x1C0(%rcx), %zmm9 - vmovups -0x180(%rcx), %zmm10 - vmovups -0x140(%rcx), %zmm11 - vmovups -0x100(%rcx), %zmm12 - vmovups -0xC0(%rcx), %zmm13 - vmovups -0x80(%rcx), %zmm14 - vmovups -0x40(%rcx), %zmm15 - vmovups %zmm0, (%rdi) - vmovups %zmm1, 0x40(%rdi) - vmovups %zmm2, 0x80(%rdi) - vmovups %zmm3, 0xC0(%rdi) - vmovups %zmm4, 0x100(%rdi) - vmovups %zmm5, 0x140(%rdi) - vmovups %zmm6, 0x180(%rdi) - vmovups %zmm7, 0x1C0(%rdi) - vmovups %zmm8, -0x200(%r9) - vmovups %zmm9, -0x1C0(%r9) - vmovups %zmm10, -0x180(%r9) - vmovups %zmm11, -0x140(%r9) - vmovups %zmm12, -0x100(%r9) - vmovups %zmm13, -0xC0(%r9) - vmovups %zmm14, -0x80(%r9) - vmovups %zmm15, -0x40(%r9) - ret - -L(1024bytesormore): - cmp %rsi, %rdi - ja L(1024bytesormore_bkw) - sub $512, %r9 - vmovups -0x200(%rcx), %zmm8 - vmovups -0x1C0(%rcx), %zmm9 - vmovups -0x180(%rcx), %zmm10 - vmovups -0x140(%rcx), %zmm11 - vmovups -0x100(%rcx), %zmm12 - vmovups -0xC0(%rcx), %zmm13 - vmovups -0x80(%rcx), %zmm14 - vmovups -0x40(%rcx), %zmm15 - prefetcht1 (%rsi) - prefetcht1 0x40(%rsi) - prefetcht1 0x80(%rsi) - prefetcht1 0xC0(%rsi) - prefetcht1 0x100(%rsi) - prefetcht1 0x140(%rsi) - prefetcht1 0x180(%rsi) - prefetcht1 0x1C0(%rsi) - -/* Loop with unaligned memory access. */ -L(gobble_512bytes_loop): - vmovups (%rsi), %zmm0 - vmovups 0x40(%rsi), %zmm1 - vmovups 0x80(%rsi), %zmm2 - vmovups 0xC0(%rsi), %zmm3 - vmovups 0x100(%rsi), %zmm4 - vmovups 0x140(%rsi), %zmm5 - vmovups 0x180(%rsi), %zmm6 - vmovups 0x1C0(%rsi), %zmm7 - add $512, %rsi - prefetcht1 (%rsi) - prefetcht1 0x40(%rsi) - prefetcht1 0x80(%rsi) - prefetcht1 0xC0(%rsi) - prefetcht1 0x100(%rsi) - prefetcht1 0x140(%rsi) - prefetcht1 0x180(%rsi) - prefetcht1 0x1C0(%rsi) - vmovups %zmm0, (%rdi) - vmovups %zmm1, 0x40(%rdi) - vmovups %zmm2, 0x80(%rdi) - vmovups %zmm3, 0xC0(%rdi) - vmovups %zmm4, 0x100(%rdi) - vmovups %zmm5, 0x140(%rdi) - vmovups %zmm6, 0x180(%rdi) - vmovups %zmm7, 0x1C0(%rdi) - add $512, %rdi - cmp %r9, %rdi - jb L(gobble_512bytes_loop) - vmovups %zmm8, (%r9) - vmovups %zmm9, 0x40(%r9) - vmovups %zmm10, 0x80(%r9) - vmovups %zmm11, 0xC0(%r9) - vmovups %zmm12, 0x100(%r9) - vmovups %zmm13, 0x140(%r9) - vmovups %zmm14, 0x180(%r9) - vmovups %zmm15, 0x1C0(%r9) - ret - -L(1024bytesormore_bkw): - add $512, %rdi - vmovups 0x1C0(%rsi), %zmm8 - vmovups 0x180(%rsi), %zmm9 - vmovups 0x140(%rsi), %zmm10 - vmovups 0x100(%rsi), %zmm11 - vmovups 0xC0(%rsi), %zmm12 - vmovups 0x80(%rsi), %zmm13 - vmovups 0x40(%rsi), %zmm14 - vmovups (%rsi), %zmm15 - prefetcht1 -0x40(%rcx) - prefetcht1 -0x80(%rcx) - prefetcht1 -0xC0(%rcx) - prefetcht1 -0x100(%rcx) - prefetcht1 -0x140(%rcx) - prefetcht1 -0x180(%rcx) - prefetcht1 -0x1C0(%rcx) - prefetcht1 -0x200(%rcx) - -/* Backward loop with unaligned memory access. */ -L(gobble_512bytes_loop_bkw): - vmovups -0x40(%rcx), %zmm0 - vmovups -0x80(%rcx), %zmm1 - vmovups -0xC0(%rcx), %zmm2 - vmovups -0x100(%rcx), %zmm3 - vmovups -0x140(%rcx), %zmm4 - vmovups -0x180(%rcx), %zmm5 - vmovups -0x1C0(%rcx), %zmm6 - vmovups -0x200(%rcx), %zmm7 - sub $512, %rcx - prefetcht1 -0x40(%rcx) - prefetcht1 -0x80(%rcx) - prefetcht1 -0xC0(%rcx) - prefetcht1 -0x100(%rcx) - prefetcht1 -0x140(%rcx) - prefetcht1 -0x180(%rcx) - prefetcht1 -0x1C0(%rcx) - prefetcht1 -0x200(%rcx) - vmovups %zmm0, -0x40(%r9) - vmovups %zmm1, -0x80(%r9) - vmovups %zmm2, -0xC0(%r9) - vmovups %zmm3, -0x100(%r9) - vmovups %zmm4, -0x140(%r9) - vmovups %zmm5, -0x180(%r9) - vmovups %zmm6, -0x1C0(%r9) - vmovups %zmm7, -0x200(%r9) - sub $512, %r9 - cmp %rdi, %r9 - ja L(gobble_512bytes_loop_bkw) - vmovups %zmm8, -0x40(%rdi) - vmovups %zmm9, -0x80(%rdi) - vmovups %zmm10, -0xC0(%rdi) - vmovups %zmm11, -0x100(%rdi) - vmovups %zmm12, -0x140(%rdi) - vmovups %zmm13, -0x180(%rdi) - vmovups %zmm14, -0x1C0(%rdi) - vmovups %zmm15, -0x200(%rdi) - ret - -L(preloop_large): - cmp %rsi, %rdi - ja L(preloop_large_bkw) - vmovups (%rsi), %zmm4 - vmovups 0x40(%rsi), %zmm5 - -/* Align destination for access with non-temporal stores in the loop. */ - mov %rdi, %r8 - and $-0x80, %rdi - add $0x80, %rdi - sub %rdi, %r8 - sub %r8, %rsi - add %r8, %rdx -L(gobble_256bytes_nt_loop): - prefetcht1 0x200(%rsi) - prefetcht1 0x240(%rsi) - prefetcht1 0x280(%rsi) - prefetcht1 0x2C0(%rsi) - prefetcht1 0x300(%rsi) - prefetcht1 0x340(%rsi) - prefetcht1 0x380(%rsi) - prefetcht1 0x3C0(%rsi) - vmovdqu64 (%rsi), %zmm0 - vmovdqu64 0x40(%rsi), %zmm1 - vmovdqu64 0x80(%rsi), %zmm2 - vmovdqu64 0xC0(%rsi), %zmm3 - vmovntdq %zmm0, (%rdi) - vmovntdq %zmm1, 0x40(%rdi) - vmovntdq %zmm2, 0x80(%rdi) - vmovntdq %zmm3, 0xC0(%rdi) - sub $256, %rdx - add $256, %rsi - add $256, %rdi - cmp $256, %rdx - ja L(gobble_256bytes_nt_loop) - sfence - vmovups %zmm4, (%rax) - vmovups %zmm5, 0x40(%rax) - jmp L(check) - -L(preloop_large_bkw): - vmovups -0x80(%rcx), %zmm4 - vmovups -0x40(%rcx), %zmm5 - -/* Align end of destination for access with non-temporal stores. */ - mov %r9, %r8 - and $-0x80, %r9 - sub %r9, %r8 - sub %r8, %rcx - sub %r8, %rdx - add %r9, %r8 -L(gobble_256bytes_nt_loop_bkw): - prefetcht1 -0x400(%rcx) - prefetcht1 -0x3C0(%rcx) - prefetcht1 -0x380(%rcx) - prefetcht1 -0x340(%rcx) - prefetcht1 -0x300(%rcx) - prefetcht1 -0x2C0(%rcx) - prefetcht1 -0x280(%rcx) - prefetcht1 -0x240(%rcx) - vmovdqu64 -0x100(%rcx), %zmm0 - vmovdqu64 -0xC0(%rcx), %zmm1 - vmovdqu64 -0x80(%rcx), %zmm2 - vmovdqu64 -0x40(%rcx), %zmm3 - vmovntdq %zmm0, -0x100(%r9) - vmovntdq %zmm1, -0xC0(%r9) - vmovntdq %zmm2, -0x80(%r9) - vmovntdq %zmm3, -0x40(%r9) - sub $256, %rdx - sub $256, %rcx - sub $256, %r9 - cmp $256, %rdx - ja L(gobble_256bytes_nt_loop_bkw) - sfence - vmovups %zmm4, -0x80(%r8) - vmovups %zmm5, -0x40(%r8) - jmp L(check) -END (MEMCPY) -#endif diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S b/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S index 518d1fe..5b8ff57 100644 --- a/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S +++ b/sysdeps/x86_64/multiarch/memmove-avx512-no-vzeroupper.S @@ -1,4 +1,4 @@ -/* memmove optimized with AVX512 for KNL hardware. +/* memmove/memcpy/mempcpy optimized with AVX512 for KNL hardware. Copyright (C) 2016 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,7 +16,405 @@ License along with the GNU C Library; if not, see . */ -#define USE_AS_MEMMOVE -#define MEMCPY __memmove_avx512_no_vzeroupper -#define MEMCPY_CHK __memmove_chk_avx512_no_vzeroupper -#include "memcpy-avx512-no-vzeroupper.S" +#include + +#if defined HAVE_AVX512_ASM_SUPPORT && IS_IN (libc) + +# include "asm-syntax.h" + + .section .text.avx512,"ax",@progbits +# if defined SHARED && !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE +ENTRY (__mempcpy_chk_avx512_no_vzeroupper) + cmpq %rdx, %rcx + jb HIDDEN_JUMPTARGET (__chk_fail) +END (__mempcpy_chk_avx512_no_vzeroupper) + +ENTRY (__mempcpy_avx512_no_vzeroupper) + movq %rdi, %rax + addq %rdx, %rax + jmp L(start) +END (__mempcpy_avx512_no_vzeroupper) +# endif + +# ifdef SHARED +ENTRY (__memmove_chk_avx512_no_vzeroupper) + cmpq %rdx, %rcx + jb HIDDEN_JUMPTARGET (__chk_fail) +END (__memmove_chk_avx512_no_vzeroupper) +# endif + +ENTRY (__memmove_avx512_no_vzeroupper) + mov %rdi, %rax +# ifdef USE_AS_MEMPCPY + add %rdx, %rax +# endif +L(start): + lea (%rsi, %rdx), %rcx + lea (%rdi, %rdx), %r9 + cmp $512, %rdx + ja L(512bytesormore) + +L(check): + cmp $16, %rdx + jbe L(less_16bytes) + cmp $256, %rdx + jb L(less_256bytes) + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups 0x80(%rsi), %zmm2 + vmovups 0xC0(%rsi), %zmm3 + vmovups -0x100(%rcx), %zmm4 + vmovups -0xC0(%rcx), %zmm5 + vmovups -0x80(%rcx), %zmm6 + vmovups -0x40(%rcx), %zmm7 + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, 0x80(%rdi) + vmovups %zmm3, 0xC0(%rdi) + vmovups %zmm4, -0x100(%r9) + vmovups %zmm5, -0xC0(%r9) + vmovups %zmm6, -0x80(%r9) + vmovups %zmm7, -0x40(%r9) + ret + +L(less_256bytes): + cmp $128, %dl + jb L(less_128bytes) + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups -0x80(%rcx), %zmm2 + vmovups -0x40(%rcx), %zmm3 + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, -0x80(%r9) + vmovups %zmm3, -0x40(%r9) + ret + +L(less_128bytes): + cmp $64, %dl + jb L(less_64bytes) + vmovdqu (%rsi), %ymm0 + vmovdqu 0x20(%rsi), %ymm1 + vmovdqu -0x40(%rcx), %ymm2 + vmovdqu -0x20(%rcx), %ymm3 + vmovdqu %ymm0, (%rdi) + vmovdqu %ymm1, 0x20(%rdi) + vmovdqu %ymm2, -0x40(%r9) + vmovdqu %ymm3, -0x20(%r9) + ret + +L(less_64bytes): + cmp $32, %dl + jb L(less_32bytes) + vmovdqu (%rsi), %ymm0 + vmovdqu -0x20(%rcx), %ymm1 + vmovdqu %ymm0, (%rdi) + vmovdqu %ymm1, -0x20(%r9) + ret + +L(less_32bytes): + vmovdqu (%rsi), %xmm0 + vmovdqu -0x10(%rcx), %xmm1 + vmovdqu %xmm0, (%rdi) + vmovdqu %xmm1, -0x10(%r9) + ret + +L(less_16bytes): + cmp $8, %dl + jb L(less_8bytes) + movq (%rsi), %rsi + movq -0x8(%rcx), %rcx + movq %rsi, (%rdi) + movq %rcx, -0x8(%r9) + ret + +L(less_8bytes): + cmp $4, %dl + jb L(less_4bytes) + mov (%rsi), %esi + mov -0x4(%rcx), %ecx + mov %esi, (%rdi) + mov %ecx, -0x4(%r9) + ret + +L(less_4bytes): + cmp $2, %dl + jb L(less_2bytes) + mov (%rsi), %si + mov -0x2(%rcx), %cx + mov %si, (%rdi) + mov %cx, -0x2(%r9) + ret + +L(less_2bytes): + cmp $1, %dl + jb L(less_1bytes) + mov (%rsi), %cl + mov %cl, (%rdi) +L(less_1bytes): + ret + +L(512bytesormore): +# ifdef SHARED_CACHE_SIZE_HALF + mov $SHARED_CACHE_SIZE_HALF, %r8 +# else + mov __x86_shared_cache_size_half(%rip), %r8 +# endif + cmp %r8, %rdx + jae L(preloop_large) + cmp $1024, %rdx + ja L(1024bytesormore) + prefetcht1 (%rsi) + prefetcht1 0x40(%rsi) + prefetcht1 0x80(%rsi) + prefetcht1 0xC0(%rsi) + prefetcht1 0x100(%rsi) + prefetcht1 0x140(%rsi) + prefetcht1 0x180(%rsi) + prefetcht1 0x1C0(%rsi) + prefetcht1 -0x200(%rcx) + prefetcht1 -0x1C0(%rcx) + prefetcht1 -0x180(%rcx) + prefetcht1 -0x140(%rcx) + prefetcht1 -0x100(%rcx) + prefetcht1 -0xC0(%rcx) + prefetcht1 -0x80(%rcx) + prefetcht1 -0x40(%rcx) + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups 0x80(%rsi), %zmm2 + vmovups 0xC0(%rsi), %zmm3 + vmovups 0x100(%rsi), %zmm4 + vmovups 0x140(%rsi), %zmm5 + vmovups 0x180(%rsi), %zmm6 + vmovups 0x1C0(%rsi), %zmm7 + vmovups -0x200(%rcx), %zmm8 + vmovups -0x1C0(%rcx), %zmm9 + vmovups -0x180(%rcx), %zmm10 + vmovups -0x140(%rcx), %zmm11 + vmovups -0x100(%rcx), %zmm12 + vmovups -0xC0(%rcx), %zmm13 + vmovups -0x80(%rcx), %zmm14 + vmovups -0x40(%rcx), %zmm15 + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, 0x80(%rdi) + vmovups %zmm3, 0xC0(%rdi) + vmovups %zmm4, 0x100(%rdi) + vmovups %zmm5, 0x140(%rdi) + vmovups %zmm6, 0x180(%rdi) + vmovups %zmm7, 0x1C0(%rdi) + vmovups %zmm8, -0x200(%r9) + vmovups %zmm9, -0x1C0(%r9) + vmovups %zmm10, -0x180(%r9) + vmovups %zmm11, -0x140(%r9) + vmovups %zmm12, -0x100(%r9) + vmovups %zmm13, -0xC0(%r9) + vmovups %zmm14, -0x80(%r9) + vmovups %zmm15, -0x40(%r9) + ret + +L(1024bytesormore): + cmp %rsi, %rdi + ja L(1024bytesormore_bkw) + sub $512, %r9 + vmovups -0x200(%rcx), %zmm8 + vmovups -0x1C0(%rcx), %zmm9 + vmovups -0x180(%rcx), %zmm10 + vmovups -0x140(%rcx), %zmm11 + vmovups -0x100(%rcx), %zmm12 + vmovups -0xC0(%rcx), %zmm13 + vmovups -0x80(%rcx), %zmm14 + vmovups -0x40(%rcx), %zmm15 + prefetcht1 (%rsi) + prefetcht1 0x40(%rsi) + prefetcht1 0x80(%rsi) + prefetcht1 0xC0(%rsi) + prefetcht1 0x100(%rsi) + prefetcht1 0x140(%rsi) + prefetcht1 0x180(%rsi) + prefetcht1 0x1C0(%rsi) + +/* Loop with unaligned memory access. */ +L(gobble_512bytes_loop): + vmovups (%rsi), %zmm0 + vmovups 0x40(%rsi), %zmm1 + vmovups 0x80(%rsi), %zmm2 + vmovups 0xC0(%rsi), %zmm3 + vmovups 0x100(%rsi), %zmm4 + vmovups 0x140(%rsi), %zmm5 + vmovups 0x180(%rsi), %zmm6 + vmovups 0x1C0(%rsi), %zmm7 + add $512, %rsi + prefetcht1 (%rsi) + prefetcht1 0x40(%rsi) + prefetcht1 0x80(%rsi) + prefetcht1 0xC0(%rsi) + prefetcht1 0x100(%rsi) + prefetcht1 0x140(%rsi) + prefetcht1 0x180(%rsi) + prefetcht1 0x1C0(%rsi) + vmovups %zmm0, (%rdi) + vmovups %zmm1, 0x40(%rdi) + vmovups %zmm2, 0x80(%rdi) + vmovups %zmm3, 0xC0(%rdi) + vmovups %zmm4, 0x100(%rdi) + vmovups %zmm5, 0x140(%rdi) + vmovups %zmm6, 0x180(%rdi) + vmovups %zmm7, 0x1C0(%rdi) + add $512, %rdi + cmp %r9, %rdi + jb L(gobble_512bytes_loop) + vmovups %zmm8, (%r9) + vmovups %zmm9, 0x40(%r9) + vmovups %zmm10, 0x80(%r9) + vmovups %zmm11, 0xC0(%r9) + vmovups %zmm12, 0x100(%r9) + vmovups %zmm13, 0x140(%r9) + vmovups %zmm14, 0x180(%r9) + vmovups %zmm15, 0x1C0(%r9) + ret + +L(1024bytesormore_bkw): + add $512, %rdi + vmovups 0x1C0(%rsi), %zmm8 + vmovups 0x180(%rsi), %zmm9 + vmovups 0x140(%rsi), %zmm10 + vmovups 0x100(%rsi), %zmm11 + vmovups 0xC0(%rsi), %zmm12 + vmovups 0x80(%rsi), %zmm13 + vmovups 0x40(%rsi), %zmm14 + vmovups (%rsi), %zmm15 + prefetcht1 -0x40(%rcx) + prefetcht1 -0x80(%rcx) + prefetcht1 -0xC0(%rcx) + prefetcht1 -0x100(%rcx) + prefetcht1 -0x140(%rcx) + prefetcht1 -0x180(%rcx) + prefetcht1 -0x1C0(%rcx) + prefetcht1 -0x200(%rcx) + +/* Backward loop with unaligned memory access. */ +L(gobble_512bytes_loop_bkw): + vmovups -0x40(%rcx), %zmm0 + vmovups -0x80(%rcx), %zmm1 + vmovups -0xC0(%rcx), %zmm2 + vmovups -0x100(%rcx), %zmm3 + vmovups -0x140(%rcx), %zmm4 + vmovups -0x180(%rcx), %zmm5 + vmovups -0x1C0(%rcx), %zmm6 + vmovups -0x200(%rcx), %zmm7 + sub $512, %rcx + prefetcht1 -0x40(%rcx) + prefetcht1 -0x80(%rcx) + prefetcht1 -0xC0(%rcx) + prefetcht1 -0x100(%rcx) + prefetcht1 -0x140(%rcx) + prefetcht1 -0x180(%rcx) + prefetcht1 -0x1C0(%rcx) + prefetcht1 -0x200(%rcx) + vmovups %zmm0, -0x40(%r9) + vmovups %zmm1, -0x80(%r9) + vmovups %zmm2, -0xC0(%r9) + vmovups %zmm3, -0x100(%r9) + vmovups %zmm4, -0x140(%r9) + vmovups %zmm5, -0x180(%r9) + vmovups %zmm6, -0x1C0(%r9) + vmovups %zmm7, -0x200(%r9) + sub $512, %r9 + cmp %rdi, %r9 + ja L(gobble_512bytes_loop_bkw) + vmovups %zmm8, -0x40(%rdi) + vmovups %zmm9, -0x80(%rdi) + vmovups %zmm10, -0xC0(%rdi) + vmovups %zmm11, -0x100(%rdi) + vmovups %zmm12, -0x140(%rdi) + vmovups %zmm13, -0x180(%rdi) + vmovups %zmm14, -0x1C0(%rdi) + vmovups %zmm15, -0x200(%rdi) + ret + +L(preloop_large): + cmp %rsi, %rdi + ja L(preloop_large_bkw) + vmovups (%rsi), %zmm4 + vmovups 0x40(%rsi), %zmm5 + +/* Align destination for access with non-temporal stores in the loop. */ + mov %rdi, %r8 + and $-0x80, %rdi + add $0x80, %rdi + sub %rdi, %r8 + sub %r8, %rsi + add %r8, %rdx +L(gobble_256bytes_nt_loop): + prefetcht1 0x200(%rsi) + prefetcht1 0x240(%rsi) + prefetcht1 0x280(%rsi) + prefetcht1 0x2C0(%rsi) + prefetcht1 0x300(%rsi) + prefetcht1 0x340(%rsi) + prefetcht1 0x380(%rsi) + prefetcht1 0x3C0(%rsi) + vmovdqu64 (%rsi), %zmm0 + vmovdqu64 0x40(%rsi), %zmm1 + vmovdqu64 0x80(%rsi), %zmm2 + vmovdqu64 0xC0(%rsi), %zmm3 + vmovntdq %zmm0, (%rdi) + vmovntdq %zmm1, 0x40(%rdi) + vmovntdq %zmm2, 0x80(%rdi) + vmovntdq %zmm3, 0xC0(%rdi) + sub $256, %rdx + add $256, %rsi + add $256, %rdi + cmp $256, %rdx + ja L(gobble_256bytes_nt_loop) + sfence + vmovups %zmm4, (%rax) + vmovups %zmm5, 0x40(%rax) + jmp L(check) + +L(preloop_large_bkw): + vmovups -0x80(%rcx), %zmm4 + vmovups -0x40(%rcx), %zmm5 + +/* Align end of destination for access with non-temporal stores. */ + mov %r9, %r8 + and $-0x80, %r9 + sub %r9, %r8 + sub %r8, %rcx + sub %r8, %rdx + add %r9, %r8 +L(gobble_256bytes_nt_loop_bkw): + prefetcht1 -0x400(%rcx) + prefetcht1 -0x3C0(%rcx) + prefetcht1 -0x380(%rcx) + prefetcht1 -0x340(%rcx) + prefetcht1 -0x300(%rcx) + prefetcht1 -0x2C0(%rcx) + prefetcht1 -0x280(%rcx) + prefetcht1 -0x240(%rcx) + vmovdqu64 -0x100(%rcx), %zmm0 + vmovdqu64 -0xC0(%rcx), %zmm1 + vmovdqu64 -0x80(%rcx), %zmm2 + vmovdqu64 -0x40(%rcx), %zmm3 + vmovntdq %zmm0, -0x100(%r9) + vmovntdq %zmm1, -0xC0(%r9) + vmovntdq %zmm2, -0x80(%r9) + vmovntdq %zmm3, -0x40(%r9) + sub $256, %rdx + sub $256, %rcx + sub $256, %r9 + cmp $256, %rdx + ja L(gobble_256bytes_nt_loop_bkw) + sfence + vmovups %zmm4, -0x80(%r8) + vmovups %zmm5, -0x40(%r8) + jmp L(check) +END (__memmove_avx512_no_vzeroupper) + +# ifdef SHARED +strong_alias (__memmove_avx512_no_vzeroupper, __memcpy_avx512_no_vzeroupper) +strong_alias (__memmove_chk_avx512_no_vzeroupper, __memcpy_chk_avx512_no_vzeroupper) +# endif +#endif