[v9,4/6] x86: Update memset to use new VEC macros

Message ID 20221015002100.129511-4-goldstein.w.n@gmail.com
State Committed
Commit a6784653f77e77c1f91dedcb524dcb3851d49fc9
Headers
Series [v9,1/6] x86: Update VEC macros to complete API for evex/evex512 impls |

Checks

Context Check Description
dj/TryBot-apply_patch success Patch applied to master at the time it was sent

Commit Message

Noah Goldstein Oct. 15, 2022, 12:20 a.m. UTC
  Replace %VEC(n) -> %VMM(n)

This commit does not change libc.so

Tested build on x86-64
---
 .../memset-avx2-unaligned-erms-rtm.S          |  8 +--
 .../multiarch/memset-avx2-unaligned-erms.S    | 14 +---
 .../multiarch/memset-avx512-unaligned-erms.S  | 20 +-----
 .../multiarch/memset-evex-unaligned-erms.S    | 20 +-----
 .../multiarch/memset-sse2-unaligned-erms.S    | 10 +--
 .../multiarch/memset-vec-unaligned-erms.S     | 70 ++++++++-----------
 6 files changed, 43 insertions(+), 99 deletions(-)
  

Comments

H.J. Lu Oct. 15, 2022, 2:53 a.m. UTC | #1
On Fri, Oct 14, 2022 at 5:21 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> Replace %VEC(n) -> %VMM(n)
>
> This commit does not change libc.so
>
> Tested build on x86-64
> ---
>  .../memset-avx2-unaligned-erms-rtm.S          |  8 +--
>  .../multiarch/memset-avx2-unaligned-erms.S    | 14 +---
>  .../multiarch/memset-avx512-unaligned-erms.S  | 20 +-----
>  .../multiarch/memset-evex-unaligned-erms.S    | 20 +-----
>  .../multiarch/memset-sse2-unaligned-erms.S    | 10 +--
>  .../multiarch/memset-vec-unaligned-erms.S     | 70 ++++++++-----------
>  6 files changed, 43 insertions(+), 99 deletions(-)
>
> diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S
> index 8ac3e479bb..bc8605faf3 100644
> --- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S
> +++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S
> @@ -1,10 +1,6 @@
> -#define ZERO_UPPER_VEC_REGISTERS_RETURN \
> -  ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
> +#include "x86-avx-rtm-vecs.h"
>
> -#define VZEROUPPER_RETURN jmp   L(return)
> -
> -#define SECTION(p) p##.avx.rtm
>  #define MEMSET_SYMBOL(p,s)     p##_avx2_##s##_rtm
>  #define WMEMSET_SYMBOL(p,s)    p##_avx2_##s##_rtm
>
> -#include "memset-avx2-unaligned-erms.S"
> +# include "memset-avx2-unaligned-erms.S"
> diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S
> index a9054a9122..47cf5072a4 100644
> --- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S
> +++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S
> @@ -4,14 +4,9 @@
>
>  # define USE_WITH_AVX2 1
>
> -# define VEC_SIZE      32
> -# define MOV_SIZE      4
> -# define RET_SIZE      4
> -
> -# define VEC(i)                ymm##i
> -
> -# define VMOVU     vmovdqu
> -# define VMOVA     vmovdqa
> +# ifndef VEC_SIZE
> +#  include "x86-avx-vecs.h"
> +# endif
>
>  # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
>    vmovd d, %xmm0; \
> @@ -26,9 +21,6 @@
>  # define WMEMSET_VDUP_TO_VEC0_HIGH() vpbroadcastd %xmm0, %ymm0
>  # define WMEMSET_VDUP_TO_VEC0_LOW() vpbroadcastd %xmm0, %xmm0
>
> -# ifndef SECTION
> -#  define SECTION(p)           p##.avx
> -# endif
>  # ifndef MEMSET_SYMBOL
>  #  define MEMSET_SYMBOL(p,s)   p##_avx2_##s
>  # endif
> diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
> index 47623b8ee8..84145b6c27 100644
> --- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
> +++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
> @@ -4,26 +4,14 @@
>
>  # define USE_WITH_AVX512       1
>
> -# define VEC_SIZE      64
> -# define MOV_SIZE      6
> -# define RET_SIZE      1
> -
> -# define XMM0          xmm16
> -# define YMM0          ymm16
> -# define VEC0          zmm16
> -# define VEC(i)                VEC##i
> -
> -# define VMOVU     vmovdqu64
> -# define VMOVA     vmovdqa64
> -
> -# define VZEROUPPER
> +# include "x86-evex512-vecs.h"
>
>  # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
> -  vpbroadcastb d, %VEC0; \
> +  vpbroadcastb d, %VMM(0); \
>    movq r, %rax
>
>  # define WMEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
> -  vpbroadcastd d, %VEC0; \
> +  vpbroadcastd d, %VMM(0); \
>    movq r, %rax
>
>  # define MEMSET_VDUP_TO_VEC0_HIGH()
> @@ -32,8 +20,6 @@
>  # define WMEMSET_VDUP_TO_VEC0_HIGH()
>  # define WMEMSET_VDUP_TO_VEC0_LOW()
>
> -# define SECTION(p)            p##.evex512
> -
>  #ifndef MEMSET_SYMBOL
>  # define MEMSET_SYMBOL(p,s)    p##_avx512_##s
>  #endif
> diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
> index ac4b2d2d50..1f03b26bf8 100644
> --- a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
> +++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
> @@ -4,26 +4,14 @@
>
>  # define USE_WITH_EVEX 1
>
> -# define VEC_SIZE      32
> -# define MOV_SIZE      6
> -# define RET_SIZE      1
> -
> -# define XMM0          xmm16
> -# define YMM0          ymm16
> -# define VEC0          ymm16
> -# define VEC(i)                VEC##i
> -
> -# define VMOVU     vmovdqu64
> -# define VMOVA     vmovdqa64
> -
> -# define VZEROUPPER
> +# include "x86-evex256-vecs.h"
>
>  # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
> -  vpbroadcastb d, %VEC0; \
> +  vpbroadcastb d, %VMM(0); \
>    movq r, %rax
>
>  # define WMEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
> -  vpbroadcastd d, %VEC0; \
> +  vpbroadcastd d, %VMM(0); \
>    movq r, %rax
>
>  # define MEMSET_VDUP_TO_VEC0_HIGH()
> @@ -32,8 +20,6 @@
>  # define WMEMSET_VDUP_TO_VEC0_HIGH()
>  # define WMEMSET_VDUP_TO_VEC0_LOW()
>
> -# define SECTION(p)            p##.evex
> -
>  #ifndef MEMSET_SYMBOL
>  # define MEMSET_SYMBOL(p,s)    p##_evex_##s
>  #endif
> diff --git a/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
> index 44f9b8888b..34b245d8ca 100644
> --- a/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
> +++ b/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
> @@ -26,13 +26,7 @@
>  # include <sysdep.h>
>  # define USE_WITH_SSE2 1
>
> -# define VEC_SIZE      16
> -# define MOV_SIZE      3
> -# define RET_SIZE      1
> -
> -# define VEC(i)                xmm##i
> -# define VMOVU     movups
> -# define VMOVA     movaps
> +# include "x86-sse2-vecs.h"
>
>  # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
>    movd d, %xmm0; \
> @@ -52,8 +46,6 @@
>  # define WMEMSET_VDUP_TO_VEC0_HIGH()
>  # define WMEMSET_VDUP_TO_VEC0_LOW()
>
> -# define SECTION(p)            p
> -
>  # ifndef MEMSET_SYMBOL
>  #  define MEMSET_SYMBOL(p,s)   p##_sse2_##s
>  # endif
> diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
> index 905d0fa464..03de0ab907 100644
> --- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
> +++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
> @@ -34,14 +34,6 @@
>  # define WMEMSET_CHK_SYMBOL(p,s)       WMEMSET_SYMBOL(p, s)
>  #endif
>
> -#ifndef XMM0
> -# define XMM0                          xmm0
> -#endif
> -
> -#ifndef YMM0
> -# define YMM0                          ymm0
> -#endif
> -
>  #ifndef VZEROUPPER
>  # if VEC_SIZE > 16
>  #  define VZEROUPPER                   vzeroupper
> @@ -150,8 +142,8 @@ L(entry_from_wmemset):
>         cmpq    $(VEC_SIZE * 2), %rdx
>         ja      L(more_2x_vec)
>         /* From VEC and to 2 * VEC.  No branch when size == VEC_SIZE.  */
> -       VMOVU   %VEC(0), -VEC_SIZE(%rdi,%rdx)
> -       VMOVU   %VEC(0), (%rdi)
> +       VMOVU   %VMM(0), -VEC_SIZE(%rdi,%rdx)
> +       VMOVU   %VMM(0), (%rdi)
>         VZEROUPPER_RETURN
>  #if defined USE_MULTIARCH && IS_IN (libc)
>  END (MEMSET_SYMBOL (__memset, unaligned))
> @@ -175,19 +167,19 @@ ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6)
>         cmp     $(VEC_SIZE * 2), %RDX_LP
>         ja      L(stosb_more_2x_vec)
>         /* From VEC and to 2 * VEC.  No branch when size == VEC_SIZE.  */
> -       VMOVU   %VEC(0), (%rdi)
> -       VMOVU   %VEC(0), (VEC_SIZE * -1)(%rdi, %rdx)
> +       VMOVU   %VMM(0), (%rdi)
> +       VMOVU   %VMM(0), (VEC_SIZE * -1)(%rdi, %rdx)
>         VZEROUPPER_RETURN
>  #endif
>
>         .p2align 4,, 4
>  L(last_2x_vec):
>  #ifdef USE_LESS_VEC_MASK_STORE
> -       VMOVU   %VEC(0), (VEC_SIZE * -2)(%rdi, %rdx)
> -       VMOVU   %VEC(0), (VEC_SIZE * -1)(%rdi, %rdx)
> +       VMOVU   %VMM(0), (VEC_SIZE * -2)(%rdi, %rdx)
> +       VMOVU   %VMM(0), (VEC_SIZE * -1)(%rdi, %rdx)
>  #else
> -       VMOVU   %VEC(0), (VEC_SIZE * -2)(%rdi)
> -       VMOVU   %VEC(0), (VEC_SIZE * -1)(%rdi)
> +       VMOVU   %VMM(0), (VEC_SIZE * -2)(%rdi)
> +       VMOVU   %VMM(0), (VEC_SIZE * -1)(%rdi)
>  #endif
>         VZEROUPPER_RETURN
>
> @@ -221,7 +213,7 @@ L(less_vec_from_wmemset):
>         bzhil   %edx, %ecx, %ecx
>         kmovd   %ecx, %k1
>  # endif
> -       vmovdqu8 %VEC(0), (%rax){%k1}
> +       vmovdqu8 %VMM(0), (%rax){%k1}
>         VZEROUPPER_RETURN
>
>  # if defined USE_MULTIARCH && IS_IN (libc)
> @@ -249,8 +241,8 @@ L(stosb_more_2x_vec):
>            and (4x, 8x] jump to target.  */
>  L(more_2x_vec):
>         /* Store next 2x vec regardless.  */
> -       VMOVU   %VEC(0), (%rdi)
> -       VMOVU   %VEC(0), (VEC_SIZE * 1)(%rdi)
> +       VMOVU   %VMM(0), (%rdi)
> +       VMOVU   %VMM(0), (VEC_SIZE * 1)(%rdi)
>
>
>         /* Two different methods of setting up pointers / compare. The two
> @@ -278,8 +270,8 @@ L(more_2x_vec):
>  #endif
>
>         /* Store next 2x vec regardless.  */
> -       VMOVU   %VEC(0), (VEC_SIZE * 2)(%rax)
> -       VMOVU   %VEC(0), (VEC_SIZE * 3)(%rax)
> +       VMOVU   %VMM(0), (VEC_SIZE * 2)(%rax)
> +       VMOVU   %VMM(0), (VEC_SIZE * 3)(%rax)
>
>
>  #if defined USE_WITH_EVEX || defined USE_WITH_AVX512
> @@ -304,20 +296,20 @@ L(more_2x_vec):
>         andq    $(VEC_SIZE * -2), %LOOP_REG
>         .p2align 4
>  L(loop):
> -       VMOVA   %VEC(0), LOOP_4X_OFFSET(%LOOP_REG)
> -       VMOVA   %VEC(0), (VEC_SIZE + LOOP_4X_OFFSET)(%LOOP_REG)
> -       VMOVA   %VEC(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%LOOP_REG)
> -       VMOVA   %VEC(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%LOOP_REG)
> +       VMOVA   %VMM(0), LOOP_4X_OFFSET(%LOOP_REG)
> +       VMOVA   %VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%LOOP_REG)
> +       VMOVA   %VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%LOOP_REG)
> +       VMOVA   %VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%LOOP_REG)
>         subq    $-(VEC_SIZE * 4), %LOOP_REG
>         cmpq    %END_REG, %LOOP_REG
>         jb      L(loop)
>         .p2align 4,, MOV_SIZE
>  L(last_4x_vec):
> -       VMOVU   %VEC(0), LOOP_4X_OFFSET(%END_REG)
> -       VMOVU   %VEC(0), (VEC_SIZE + LOOP_4X_OFFSET)(%END_REG)
> -       VMOVU   %VEC(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%END_REG)
> -       VMOVU   %VEC(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%END_REG)
> -L(return):
> +       VMOVU   %VMM(0), LOOP_4X_OFFSET(%END_REG)
> +       VMOVU   %VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%END_REG)
> +       VMOVU   %VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%END_REG)
> +       VMOVU   %VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%END_REG)
> +L(return_vzeroupper):
>  #if VEC_SIZE > 16
>         ZERO_UPPER_VEC_REGISTERS_RETURN
>  #else
> @@ -355,7 +347,7 @@ L(cross_page):
>         jge     L(between_16_31)
>  #endif
>  #ifndef USE_XMM_LESS_VEC
> -       MOVQ    %XMM0, %SET_REG64
> +       MOVQ    %VMM_128(0), %SET_REG64
>  #endif
>         cmpl    $8, %edx
>         jge     L(between_8_15)
> @@ -374,8 +366,8 @@ L(between_0_0):
>         .p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, RET_SIZE)
>         /* From 32 to 63.  No branch when size == 32.  */
>  L(between_32_63):
> -       VMOVU   %YMM0, (%LESS_VEC_REG)
> -       VMOVU   %YMM0, -32(%LESS_VEC_REG, %rdx)
> +       VMOVU   %VMM_256(0), (%LESS_VEC_REG)
> +       VMOVU   %VMM_256(0), -32(%LESS_VEC_REG, %rdx)
>         VZEROUPPER_RETURN
>  #endif
>
> @@ -383,8 +375,8 @@ L(between_32_63):
>         .p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, 1)
>  L(between_16_31):
>         /* From 16 to 31.  No branch when size == 16.  */
> -       VMOVU   %XMM0, (%LESS_VEC_REG)
> -       VMOVU   %XMM0, -16(%LESS_VEC_REG, %rdx)
> +       VMOVU   %VMM_128(0), (%LESS_VEC_REG)
> +       VMOVU   %VMM_128(0), -16(%LESS_VEC_REG, %rdx)
>         ret
>  #endif
>
> @@ -394,8 +386,8 @@ L(between_16_31):
>  L(between_8_15):
>         /* From 8 to 15.  No branch when size == 8.  */
>  #ifdef USE_XMM_LESS_VEC
> -       MOVQ    %XMM0, (%rdi)
> -       MOVQ    %XMM0, -8(%rdi, %rdx)
> +       MOVQ    %VMM_128(0), (%rdi)
> +       MOVQ    %VMM_128(0), -8(%rdi, %rdx)
>  #else
>         movq    %SET_REG64, (%LESS_VEC_REG)
>         movq    %SET_REG64, -8(%LESS_VEC_REG, %rdx)
> @@ -408,8 +400,8 @@ L(between_8_15):
>  L(between_4_7):
>         /* From 4 to 7.  No branch when size == 4.  */
>  #ifdef USE_XMM_LESS_VEC
> -       MOVD    %XMM0, (%rdi)
> -       MOVD    %XMM0, -4(%rdi, %rdx)
> +       MOVD    %VMM_128(0), (%rdi)
> +       MOVD    %VMM_128(0), -4(%rdi, %rdx)
>  #else
>         movl    %SET_REG32, (%LESS_VEC_REG)
>         movl    %SET_REG32, -4(%LESS_VEC_REG, %rdx)
> --
> 2.34.1
>

LGTM.

Thanks.
  

Patch

diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S
index 8ac3e479bb..bc8605faf3 100644
--- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S
+++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S
@@ -1,10 +1,6 @@ 
-#define ZERO_UPPER_VEC_REGISTERS_RETURN \
-  ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
+#include "x86-avx-rtm-vecs.h"
 
-#define VZEROUPPER_RETURN jmp	 L(return)
-
-#define SECTION(p) p##.avx.rtm
 #define MEMSET_SYMBOL(p,s)	p##_avx2_##s##_rtm
 #define WMEMSET_SYMBOL(p,s)	p##_avx2_##s##_rtm
 
-#include "memset-avx2-unaligned-erms.S"
+# include "memset-avx2-unaligned-erms.S"
diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S
index a9054a9122..47cf5072a4 100644
--- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S
@@ -4,14 +4,9 @@ 
 
 # define USE_WITH_AVX2	1
 
-# define VEC_SIZE	32
-# define MOV_SIZE	4
-# define RET_SIZE	4
-
-# define VEC(i)		ymm##i
-
-# define VMOVU     vmovdqu
-# define VMOVA     vmovdqa
+# ifndef VEC_SIZE
+#  include "x86-avx-vecs.h"
+# endif
 
 # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
   vmovd d, %xmm0; \
@@ -26,9 +21,6 @@ 
 # define WMEMSET_VDUP_TO_VEC0_HIGH() vpbroadcastd %xmm0, %ymm0
 # define WMEMSET_VDUP_TO_VEC0_LOW() vpbroadcastd %xmm0, %xmm0
 
-# ifndef SECTION
-#  define SECTION(p)		p##.avx
-# endif
 # ifndef MEMSET_SYMBOL
 #  define MEMSET_SYMBOL(p,s)	p##_avx2_##s
 # endif
diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
index 47623b8ee8..84145b6c27 100644
--- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
@@ -4,26 +4,14 @@ 
 
 # define USE_WITH_AVX512	1
 
-# define VEC_SIZE	64
-# define MOV_SIZE	6
-# define RET_SIZE	1
-
-# define XMM0		xmm16
-# define YMM0		ymm16
-# define VEC0		zmm16
-# define VEC(i)		VEC##i
-
-# define VMOVU     vmovdqu64
-# define VMOVA     vmovdqa64
-
-# define VZEROUPPER
+# include "x86-evex512-vecs.h"
 
 # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
-  vpbroadcastb d, %VEC0; \
+  vpbroadcastb d, %VMM(0); \
   movq r, %rax
 
 # define WMEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
-  vpbroadcastd d, %VEC0; \
+  vpbroadcastd d, %VMM(0); \
   movq r, %rax
 
 # define MEMSET_VDUP_TO_VEC0_HIGH()
@@ -32,8 +20,6 @@ 
 # define WMEMSET_VDUP_TO_VEC0_HIGH()
 # define WMEMSET_VDUP_TO_VEC0_LOW()
 
-# define SECTION(p)		p##.evex512
-
 #ifndef MEMSET_SYMBOL
 # define MEMSET_SYMBOL(p,s)	p##_avx512_##s
 #endif
diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
index ac4b2d2d50..1f03b26bf8 100644
--- a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
@@ -4,26 +4,14 @@ 
 
 # define USE_WITH_EVEX	1
 
-# define VEC_SIZE	32
-# define MOV_SIZE	6
-# define RET_SIZE	1
-
-# define XMM0		xmm16
-# define YMM0		ymm16
-# define VEC0		ymm16
-# define VEC(i)		VEC##i
-
-# define VMOVU     vmovdqu64
-# define VMOVA     vmovdqa64
-
-# define VZEROUPPER
+# include "x86-evex256-vecs.h"
 
 # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
-  vpbroadcastb d, %VEC0; \
+  vpbroadcastb d, %VMM(0); \
   movq r, %rax
 
 # define WMEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
-  vpbroadcastd d, %VEC0; \
+  vpbroadcastd d, %VMM(0); \
   movq r, %rax
 
 # define MEMSET_VDUP_TO_VEC0_HIGH()
@@ -32,8 +20,6 @@ 
 # define WMEMSET_VDUP_TO_VEC0_HIGH()
 # define WMEMSET_VDUP_TO_VEC0_LOW()
 
-# define SECTION(p)		p##.evex
-
 #ifndef MEMSET_SYMBOL
 # define MEMSET_SYMBOL(p,s)	p##_evex_##s
 #endif
diff --git a/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
index 44f9b8888b..34b245d8ca 100644
--- a/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
@@ -26,13 +26,7 @@ 
 # include <sysdep.h>
 # define USE_WITH_SSE2	1
 
-# define VEC_SIZE	16
-# define MOV_SIZE	3
-# define RET_SIZE	1
-
-# define VEC(i)		xmm##i
-# define VMOVU     movups
-# define VMOVA     movaps
+# include "x86-sse2-vecs.h"
 
 # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
   movd d, %xmm0; \
@@ -52,8 +46,6 @@ 
 # define WMEMSET_VDUP_TO_VEC0_HIGH()
 # define WMEMSET_VDUP_TO_VEC0_LOW()
 
-# define SECTION(p)		p
-
 # ifndef MEMSET_SYMBOL
 #  define MEMSET_SYMBOL(p,s)	p##_sse2_##s
 # endif
diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
index 905d0fa464..03de0ab907 100644
--- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
+++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
@@ -34,14 +34,6 @@ 
 # define WMEMSET_CHK_SYMBOL(p,s)	WMEMSET_SYMBOL(p, s)
 #endif
 
-#ifndef XMM0
-# define XMM0				xmm0
-#endif
-
-#ifndef YMM0
-# define YMM0				ymm0
-#endif
-
 #ifndef VZEROUPPER
 # if VEC_SIZE > 16
 #  define VZEROUPPER			vzeroupper
@@ -150,8 +142,8 @@  L(entry_from_wmemset):
 	cmpq	$(VEC_SIZE * 2), %rdx
 	ja	L(more_2x_vec)
 	/* From VEC and to 2 * VEC.  No branch when size == VEC_SIZE.  */
-	VMOVU	%VEC(0), -VEC_SIZE(%rdi,%rdx)
-	VMOVU	%VEC(0), (%rdi)
+	VMOVU	%VMM(0), -VEC_SIZE(%rdi,%rdx)
+	VMOVU	%VMM(0), (%rdi)
 	VZEROUPPER_RETURN
 #if defined USE_MULTIARCH && IS_IN (libc)
 END (MEMSET_SYMBOL (__memset, unaligned))
@@ -175,19 +167,19 @@  ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6)
 	cmp	$(VEC_SIZE * 2), %RDX_LP
 	ja	L(stosb_more_2x_vec)
 	/* From VEC and to 2 * VEC.  No branch when size == VEC_SIZE.  */
-	VMOVU	%VEC(0), (%rdi)
-	VMOVU	%VEC(0), (VEC_SIZE * -1)(%rdi, %rdx)
+	VMOVU	%VMM(0), (%rdi)
+	VMOVU	%VMM(0), (VEC_SIZE * -1)(%rdi, %rdx)
 	VZEROUPPER_RETURN
 #endif
 
 	.p2align 4,, 4
 L(last_2x_vec):
 #ifdef USE_LESS_VEC_MASK_STORE
-	VMOVU	%VEC(0), (VEC_SIZE * -2)(%rdi, %rdx)
-	VMOVU	%VEC(0), (VEC_SIZE * -1)(%rdi, %rdx)
+	VMOVU	%VMM(0), (VEC_SIZE * -2)(%rdi, %rdx)
+	VMOVU	%VMM(0), (VEC_SIZE * -1)(%rdi, %rdx)
 #else
-	VMOVU	%VEC(0), (VEC_SIZE * -2)(%rdi)
-	VMOVU	%VEC(0), (VEC_SIZE * -1)(%rdi)
+	VMOVU	%VMM(0), (VEC_SIZE * -2)(%rdi)
+	VMOVU	%VMM(0), (VEC_SIZE * -1)(%rdi)
 #endif
 	VZEROUPPER_RETURN
 
@@ -221,7 +213,7 @@  L(less_vec_from_wmemset):
 	bzhil	%edx, %ecx, %ecx
 	kmovd	%ecx, %k1
 # endif
-	vmovdqu8 %VEC(0), (%rax){%k1}
+	vmovdqu8 %VMM(0), (%rax){%k1}
 	VZEROUPPER_RETURN
 
 # if defined USE_MULTIARCH && IS_IN (libc)
@@ -249,8 +241,8 @@  L(stosb_more_2x_vec):
 	   and (4x, 8x] jump to target.  */
 L(more_2x_vec):
 	/* Store next 2x vec regardless.  */
-	VMOVU	%VEC(0), (%rdi)
-	VMOVU	%VEC(0), (VEC_SIZE * 1)(%rdi)
+	VMOVU	%VMM(0), (%rdi)
+	VMOVU	%VMM(0), (VEC_SIZE * 1)(%rdi)
 
 
 	/* Two different methods of setting up pointers / compare. The two
@@ -278,8 +270,8 @@  L(more_2x_vec):
 #endif
 
 	/* Store next 2x vec regardless.  */
-	VMOVU	%VEC(0), (VEC_SIZE * 2)(%rax)
-	VMOVU	%VEC(0), (VEC_SIZE * 3)(%rax)
+	VMOVU	%VMM(0), (VEC_SIZE * 2)(%rax)
+	VMOVU	%VMM(0), (VEC_SIZE * 3)(%rax)
 
 
 #if defined USE_WITH_EVEX || defined USE_WITH_AVX512
@@ -304,20 +296,20 @@  L(more_2x_vec):
 	andq	$(VEC_SIZE * -2), %LOOP_REG
 	.p2align 4
 L(loop):
-	VMOVA	%VEC(0), LOOP_4X_OFFSET(%LOOP_REG)
-	VMOVA	%VEC(0), (VEC_SIZE + LOOP_4X_OFFSET)(%LOOP_REG)
-	VMOVA	%VEC(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%LOOP_REG)
-	VMOVA	%VEC(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%LOOP_REG)
+	VMOVA	%VMM(0), LOOP_4X_OFFSET(%LOOP_REG)
+	VMOVA	%VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%LOOP_REG)
+	VMOVA	%VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%LOOP_REG)
+	VMOVA	%VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%LOOP_REG)
 	subq	$-(VEC_SIZE * 4), %LOOP_REG
 	cmpq	%END_REG, %LOOP_REG
 	jb	L(loop)
 	.p2align 4,, MOV_SIZE
 L(last_4x_vec):
-	VMOVU	%VEC(0), LOOP_4X_OFFSET(%END_REG)
-	VMOVU	%VEC(0), (VEC_SIZE + LOOP_4X_OFFSET)(%END_REG)
-	VMOVU	%VEC(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%END_REG)
-	VMOVU	%VEC(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%END_REG)
-L(return):
+	VMOVU	%VMM(0), LOOP_4X_OFFSET(%END_REG)
+	VMOVU	%VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%END_REG)
+	VMOVU	%VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%END_REG)
+	VMOVU	%VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%END_REG)
+L(return_vzeroupper):
 #if VEC_SIZE > 16
 	ZERO_UPPER_VEC_REGISTERS_RETURN
 #else
@@ -355,7 +347,7 @@  L(cross_page):
 	jge	L(between_16_31)
 #endif
 #ifndef USE_XMM_LESS_VEC
-	MOVQ	%XMM0, %SET_REG64
+	MOVQ	%VMM_128(0), %SET_REG64
 #endif
 	cmpl	$8, %edx
 	jge	L(between_8_15)
@@ -374,8 +366,8 @@  L(between_0_0):
 	.p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, RET_SIZE)
 	/* From 32 to 63.  No branch when size == 32.  */
 L(between_32_63):
-	VMOVU	%YMM0, (%LESS_VEC_REG)
-	VMOVU	%YMM0, -32(%LESS_VEC_REG, %rdx)
+	VMOVU	%VMM_256(0), (%LESS_VEC_REG)
+	VMOVU	%VMM_256(0), -32(%LESS_VEC_REG, %rdx)
 	VZEROUPPER_RETURN
 #endif
 
@@ -383,8 +375,8 @@  L(between_32_63):
 	.p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, 1)
 L(between_16_31):
 	/* From 16 to 31.  No branch when size == 16.  */
-	VMOVU	%XMM0, (%LESS_VEC_REG)
-	VMOVU	%XMM0, -16(%LESS_VEC_REG, %rdx)
+	VMOVU	%VMM_128(0), (%LESS_VEC_REG)
+	VMOVU	%VMM_128(0), -16(%LESS_VEC_REG, %rdx)
 	ret
 #endif
 
@@ -394,8 +386,8 @@  L(between_16_31):
 L(between_8_15):
 	/* From 8 to 15.  No branch when size == 8.  */
 #ifdef USE_XMM_LESS_VEC
-	MOVQ	%XMM0, (%rdi)
-	MOVQ	%XMM0, -8(%rdi, %rdx)
+	MOVQ	%VMM_128(0), (%rdi)
+	MOVQ	%VMM_128(0), -8(%rdi, %rdx)
 #else
 	movq	%SET_REG64, (%LESS_VEC_REG)
 	movq	%SET_REG64, -8(%LESS_VEC_REG, %rdx)
@@ -408,8 +400,8 @@  L(between_8_15):
 L(between_4_7):
 	/* From 4 to 7.  No branch when size == 4.  */
 #ifdef USE_XMM_LESS_VEC
-	MOVD	%XMM0, (%rdi)
-	MOVD	%XMM0, -4(%rdi, %rdx)
+	MOVD	%VMM_128(0), (%rdi)
+	MOVD	%VMM_128(0), -4(%rdi, %rdx)
 #else
 	movl	%SET_REG32, (%LESS_VEC_REG)
 	movl	%SET_REG32, -4(%LESS_VEC_REG, %rdx)