diff mbox series

Improve V16QI vector initialization on x86 SSE.

Message ID 002f01dd3aeb$f867ca20$e9375e60$@nextmovesoftware.com
State New
Headers
Series Improve V16QI vector initialization on x86 SSE. |

Checks

Context Check Description
linaro-tcwg-bot/tcwg_simplebootstrap_build--master-aarch64-bootstrap success Build passed

Commit Message

Roger Sayle Sept. 2, 2026, 3:01 p.m. UTC 
  As promised, this is the next instalment in the patch series for
improving vector initialization on x86, this time for V16QImode.
As an example of the improvements, consider these motivating examples:

typedef char v16qi __attribute__ ((__vector_size__ (16)));
v16qi foo (char a, char b) {
  return (v16qi){a,b,a,b,a,b,a,b,a,b,a,b,a,b,a,b};
}
v16qi bar (char a) {
  return (v16qi){a,1,2,3,4,5,6,7,a,1,2,3,4,5,6,7};
}

Currently with -O2 -mavx2, GCC 17 generates

foo:    movzbl  %dil, %eax
        vmovd   %eax, %xmm0
        vpinsrb $1, %esi, %xmm0, %xmm0
        vpinsrb $2, %edi, %xmm0, %xmm0
        vpinsrb $3, %esi, %xmm0, %xmm0
        vpinsrb $4, %edi, %xmm0, %xmm0
        vpinsrb $5, %esi, %xmm0, %xmm0
        vpinsrb $6, %edi, %xmm0, %xmm0
        vpinsrb $7, %esi, %xmm0, %xmm0
        vpinsrb $8, %edi, %xmm0, %xmm0
        vpinsrb $9, %esi, %xmm0, %xmm0
        vpinsrb $10, %edi, %xmm0, %xmm0
        vpinsrb $11, %esi, %xmm0, %xmm0
        vpinsrb $12, %edi, %xmm0, %xmm0
        vpinsrb $13, %esi, %xmm0, %xmm0
        vpinsrb $14, %edi, %xmm0, %xmm0
        vpinsrb $15, %esi, %xmm0, %xmm0
        ret

bar:    movzbl  %dil, %eax
        vmovd   %eax, %xmm0
        vpinsrb $8, %edi, %xmm0, %xmm0
        vpor    .LC0(%rip), %xmm0, %xmm0
        ret

where GCC 16 previously generated:

foo:    vmovd   xmm0, edi
        vpinsrb xmm0, xmm0, esi, 1
        vpunpcklwd      xmm0, xmm0, xmm0
        vpunpckldq      xmm0, xmm0, xmm0
        vpunpcklqdq     xmm0, xmm0, xmm0
        ret

bar:    vmovd   xmm0, edi
        mov     eax, 1
        vpinsrb xmm0, xmm0, eax, 1
        mov     eax, 2
        vmovd   xmm3, eax
        mov     eax, 4
        vmovd   xmm1, eax
        mov     eax, 6
        vmovd   xmm2, eax
        mov     eax, 3
        vpinsrb xmm3, xmm3, eax, 1
        mov     eax, 5
        vpinsrb xmm1, xmm1, eax, 1
        mov     eax, 7
        vpunpcklwd      xmm0, xmm0, xmm3
        vpinsrb xmm2, xmm2, eax, 1
        vpunpcklwd      xmm1, xmm1, xmm2
        vpunpckldq      xmm0, xmm0, xmm1
        vpunpcklqdq     xmm0, xmm0, xmm0
        ret

Showing that bar is an improvement, but foo is a regression.
With this patch we now generate (even better for both):

foo:    vmovd   %edi, %xmm0
        vpinsrb $1, %esi, %xmm0, %xmm0
        vpbroadcastw    %xmm0, %xmm0
        ret

bar:    vmovdqa .LC0(%rip), %xmm0
        vpinsrb $0, %edi, %xmm0, %xmm0
        vpinsrb $8, %edi, %xmm0, %xmm0
        ret

There's still more that can be done/improved, but I stopped here
(1) because the patch is already quite large and (2) I'm not sure
if the final bar above is optimal (yes in number of instructions),
or if (v2di){0x12345678abcdefLL,0} is better written as movabs+movq?

This patch has been tested on x86_64-pc-linux-gnu with make bootstrap
and make -k check, both with and without --target_board=unix{-m32}
with no new failures.  Ok for mainline?

2026-09-02  Roger Sayle  <roger@nextmovesoftware.com>

gcc/ChangeLog
        * config/i386/i386-expand.cc (ix86_expand_vector_init_insert)
        <case E_V16QImode>: We don't need to zero extend var, if the
        high bytes will be overwritten.
        <case E_V8HImode>: Likewise for V8HImode.  Whitespace fix.
        (ix86_expand_vector_init_qi4): New helper function to effectively
        load a V4QImode vector into an SImode register or constant.
        (ix86_expand_vector_init_qi8): Likewise, to effectively load
        a V8QImode vector into a DImode register or constant.
        (ix86_expand_vector_init_v16qi): Change return type to void,
        and never fail.  Fallback to V2DI on TARGET64 and V4SI otherwise
        using the above helper functions.
        (ix86_expand_vector_init_general): Update call to
        ix86_expand_vecot_init_v16qi now that it can't fail.

gcc/testsuite/ChangeLog
        * gcc.target/i386/avx2-init-v16qi-1.c: New test case.
        * gcc.target/i386/avx-init-v16qi-1.c: Add more test vectors.
        * gcc.target/i386/sse-init-v16qi-1.c: Likewise.
        * gcc.target/i386/sse2-init-v16qi-2.c: Likewise.
        * gcc.target/i386/sse2-init-v16qi-3.c: Likewise.
        * gcc.target/i386/sse4_1-init-v16qi-1.c: Likewise.


Thanks again,
Roger
--
diff mbox series

Patch

diff --git a/gcc/config/i386/i386-expand.cc b/gcc/config/i386/i386-expand.cc
index 34c9599f928..620aff799aa 100644
--- a/gcc/config/i386/i386-expand.cc
+++ b/gcc/config/i386/i386-expand.cc
@@ -18910,7 +18910,13 @@  ix86_expand_vector_init_insert (machine_mode mode, rtx target,
     case E_V16QImode:
       if (var != const0_rtx)
 	{
-	  var = convert_modes (SImode, QImode, var, true);
+	  if (REG_P (var)
+	      && ops[1] != const0_rtx
+	      && ops[2] != const0_rtx
+	      && ops[3] != const0_rtx)
+	    var = gen_lowpart (SImode, var);
+	  else
+	    var = convert_modes (SImode, QImode, var, true);
 	  var = force_reg (SImode, var);
 	  x = gen_reg_rtx (V4SImode);
 	  emit_insn (gen_vec_setv4si_0 (x, CONST0_RTX (V4SImode), var));
@@ -18923,7 +18929,11 @@  ix86_expand_vector_init_insert (machine_mode mode, rtx target,
     case E_V8HImode:
       if (var != const0_rtx)
 	{
-	  var = convert_modes (SImode, HImode, var, true);
+	  if (REG_P (var)
+	      && ops[1] != const0_rtx)
+	    var = gen_lowpart (SImode, var);
+	  else
+	    var = convert_modes (SImode, HImode, var, true);
 	  var = force_reg (SImode, var);
 	  x = gen_reg_rtx (V4SImode);
 	  emit_insn (gen_vec_setv4si_0 (x, CONST0_RTX (V4SImode), var));
@@ -19392,7 +19402,7 @@  ix86_expand_vector_init_v8hi (rtx target, rtx *ops)
 	  }
       rtx tmp1 = gen_reg_rtx (V8HImode);
       if (!ix86_expand_vector_init_v8hi (tmp1, vars))
-        gcc_unreachable ();
+	gcc_unreachable ();
       rtx tmp2 = gen_reg_rtx (V8HImode);
       rtx vec = gen_rtx_CONST_VECTOR (V8HImode, gen_rtvec_v (8, csts));
       emit_move_insn (tmp2, vec);
@@ -19408,12 +19418,243 @@  ix86_expand_vector_init_v8hi (rtx target, rtx *ops)
   return false;
 }
 
+/* A subroutine of ix86_expand_vector_init_v16qi.
+   Place OPS[0..3] in an SImode REG or CONST_INT.  */
+
+static rtx
+ix86_expand_vector_init_qi4 (rtx *ops)
+{
+  rtx vars[4];
+  rtx word;
+  int i;
+
+  if (CONST_INT_P (ops[0])
+      && CONST_INT_P (ops[1])
+      && CONST_INT_P (ops[2])
+      && CONST_INT_P (ops[3]))
+    {
+      HOST_WIDE_INT val = (UINTVAL (ops[0]) & 0xff)
+			  + ((UINTVAL (ops[1]) & 0xff) << 8)
+			  + ((UINTVAL (ops[2]) & 0xff) << 16)
+			  + ((UINTVAL (ops[3]) & 0xff) << 24);
+      return gen_int_mode (val, SImode);
+    }
+
+  if (ops[1] == const0_rtx
+      && ops[2] == const0_rtx
+      && ops[3] == const0_rtx)
+    return convert_modes (SImode, QImode, ops[0], true);
+
+
+  if (rtx_equal_p (ops[0], ops[2])
+      && rtx_equal_p (ops[1], ops[3]))
+    {
+      vars[0] = ops[0];
+      vars[1] = ops[1];
+      vars[2] = const0_rtx;
+      vars[3] = const0_rtx;
+      rtx tmp1 = ix86_expand_vector_init_qi4 (vars);
+      rtx tmp2 = expand_simple_binop (SImode, ASHIFT, tmp1, GEN_INT (16),
+				      NULL_RTX, 1, OPTAB_LIB_WIDEN);
+      return expand_simple_binop (SImode, PLUS, tmp2, tmp1,
+				  NULL_RTX, 1, OPTAB_LIB_WIDEN);
+    }
+
+  HOST_WIDE_INT val = 0;
+  for (i = 0; i < 4; i++)
+    if (CONST_INT_P (ops[i]) && ops[i] != const0_rtx)
+      {
+	val += (UINTVAL (ops[0]) & 0xff) << (i*8);
+	vars[i] = const0_rtx;
+      }
+    else
+      vars[i] = ops[i];
+
+  if (val != 0)
+    {
+      rtx tmp = ix86_expand_vector_init_qi4 (vars);
+      return expand_simple_binop (SImode, IOR, tmp,
+				  gen_int_mode (val, SImode),
+				  NULL_RTX, 1, OPTAB_LIB_WIDEN);
+    }
+
+  if (ops[0] == const0_rtx && ops[1] == const0_rtx)
+    {
+      if (ops[2] == const0_rtx)
+	{
+	  word = convert_modes (SImode, QImode, ops[3], true);
+	  return expand_simple_binop (SImode, ASHIFT, word, GEN_INT (24),
+				      NULL_RTX, 1, OPTAB_LIB_WIDEN);
+	}
+      vars[0] = ops[2];
+      vars[1] = ops[3];
+      vars[2] = const0_rtx;
+      vars[3] = const0_rtx;
+      word = ix86_expand_vector_init_qi4 (vars);
+      return expand_simple_binop (SImode, ASHIFT, word, GEN_INT (16),
+				  NULL_RTX, 1, OPTAB_LIB_WIDEN);
+    }
+
+  vars[0] = ops[1];
+  vars[1] = ops[2];
+  vars[2] = ops[3];
+  vars[3] = const0_rtx;
+  word = ix86_expand_vector_init_qi4 (vars);
+  word = expand_simple_binop (SImode, ASHIFT, word, GEN_INT (8),
+			      NULL_RTX, 1, OPTAB_LIB_WIDEN);
+  if (ops[0] != const0_rtx)
+    {
+      rtx elt = convert_modes (SImode, QImode, ops[0], true);
+      word = expand_simple_binop (SImode, PLUS, word, elt,
+				  NULL_RTX, 1, OPTAB_LIB_WIDEN);
+    }
+  return word;
+}
+
+/* A subroutine of ix86_expand_vector_init_v16qi.
+   Place OPS[0..7] in an DImode REG or CONST_INT.  */
+
+static rtx
+ix86_expand_vector_init_qi8 (rtx *ops)
+{
+  rtx vars[8];
+  rtx word;
+  int i;
+
+  if (CONST_INT_P (ops[0])
+      && CONST_INT_P (ops[1])
+      && CONST_INT_P (ops[2])
+      && CONST_INT_P (ops[3])
+      && CONST_INT_P (ops[4])
+      && CONST_INT_P (ops[5])
+      && CONST_INT_P (ops[6])
+      && CONST_INT_P (ops[7]))
+    {
+      HOST_WIDE_INT val = (UINTVAL (ops[0]) & 0xff)
+			  + ((UINTVAL (ops[1]) & 0xff) << 8)
+			  + ((UINTVAL (ops[2]) & 0xff) << 16)
+			  + ((UINTVAL (ops[3]) & 0xff) << 24)
+			  + ((UINTVAL (ops[4]) & 0xff) << 32)
+			  + ((UINTVAL (ops[5]) & 0xff) << 40)
+			  + ((UINTVAL (ops[6]) & 0xff) << 48)
+			  + ((UINTVAL (ops[7]) & 0xff) << 56);
+      return gen_int_mode (val, DImode);
+    }
+
+  if (ops[1] == const0_rtx
+      && ops[2] == const0_rtx
+      && ops[3] == const0_rtx
+      && ops[4] == const0_rtx
+      && ops[5] == const0_rtx
+      && ops[6] == const0_rtx
+      && ops[7] == const0_rtx)
+    return convert_modes (DImode, QImode, ops[0], true);
+
+  if (rtx_equal_p (ops[0], ops[4])
+      && rtx_equal_p (ops[1], ops[5])
+      && rtx_equal_p (ops[2], ops[6])
+      && rtx_equal_p (ops[3], ops[7]))
+    {
+      vars[0] = ops[0];
+      vars[1] = ops[1];
+      vars[2] = ops[2];
+      vars[3] = ops[3];
+      vars[4] = const0_rtx;
+      vars[5] = const0_rtx;
+      vars[6] = const0_rtx;
+      vars[7] = const0_rtx;
+      rtx tmp1 = ix86_expand_vector_init_qi8 (vars);
+      rtx tmp2 = expand_simple_binop (DImode, ASHIFT, tmp1, GEN_INT (32),
+				      NULL_RTX, 1, OPTAB_LIB_WIDEN);
+      return expand_simple_binop (DImode, PLUS, tmp2, tmp1,
+				  NULL_RTX, 1, OPTAB_LIB_WIDEN);
+    }
+
+  HOST_WIDE_INT val = 0;
+  for (i = 0; i < 8; i++)
+    if (CONST_INT_P (ops[i]) && ops[i] != const0_rtx)
+      {
+	val += (UINTVAL (ops[i]) & 0xff) << (i*8);
+	vars[i] = const0_rtx;
+      }
+    else
+      vars[i] = ops[i];
+
+  if (val != 0)
+    {
+      rtx tmp = ix86_expand_vector_init_qi8 (vars);
+      return expand_simple_binop (DImode, IOR, tmp,
+				  gen_int_mode (val, DImode),
+				  NULL_RTX, 1, OPTAB_LIB_WIDEN);
+    }
+
+  if (rtx_equal_p (ops[0], ops[2])
+      && rtx_equal_p (ops[1], ops[3])
+      && ops[4] == const0_rtx
+      && ops[5] == const0_rtx
+      && ops[6] == const0_rtx
+      && ops[7] == const0_rtx)
+    {
+      vars[0] = ops[0];
+      vars[1] = ops[1];
+      vars[2] = const0_rtx;
+      vars[3] = const0_rtx;
+      vars[4] = const0_rtx;
+      vars[5] = const0_rtx;
+      vars[6] = const0_rtx;
+      vars[7] = const0_rtx;
+      rtx tmp1 = ix86_expand_vector_init_qi8 (vars);
+      rtx tmp2 = expand_simple_binop (DImode, ASHIFT, tmp1, GEN_INT (16),
+				      NULL_RTX, 1, OPTAB_LIB_WIDEN);
+      return expand_simple_binop (DImode, PLUS, tmp2, tmp1,
+				  NULL_RTX, 1, OPTAB_LIB_WIDEN);
+    }
+
+  if (ops[0] == const0_rtx && ops[1] == const0_rtx)
+    {
+      i = 2;
+      while (ops[i] == const0_rtx)
+	i++;
+      vars[0] = ops[i];
+      vars[1] = i + 1 < 8 ? ops[i + 1] : const0_rtx;
+      vars[2] = i + 2 < 8 ? ops[i + 2] : const0_rtx;
+      vars[3] = i + 3 < 8 ? ops[i + 3] : const0_rtx;
+      vars[4] = i + 4 < 8 ? ops[i + 4] : const0_rtx;
+      vars[5] = i + 5 < 8 ? ops[i + 5] : const0_rtx;
+      vars[6] = const0_rtx;
+      vars[7] = const0_rtx;
+      word = ix86_expand_vector_init_qi8 (vars);
+      return expand_simple_binop (DImode, ASHIFT, word, GEN_INT (i * 8),
+				  NULL_RTX, 1, OPTAB_LIB_WIDEN);
+    }
+
+  vars[0] = ops[1];
+  vars[1] = ops[2];
+  vars[2] = ops[3];
+  vars[3] = ops[4];
+  vars[4] = ops[5];
+  vars[5] = ops[6];
+  vars[6] = ops[7];
+  vars[7] = const0_rtx;
+  word = ix86_expand_vector_init_qi8 (vars);
+  word = expand_simple_binop (DImode, ASHIFT, word, GEN_INT (8),
+			      NULL_RTX, 1, OPTAB_LIB_WIDEN);
+  if (ops[0] != const0_rtx)
+    {
+      rtx elt = convert_modes (DImode, QImode, ops[0], true);
+      word = expand_simple_binop (DImode, PLUS, word, elt,
+				  NULL_RTX, 1, OPTAB_LIB_WIDEN);
+    }
+  return word;
+}
+
 /* A subroutine of ix86_expand_vector_init for V16QImode.  */
 
-static bool
+static void
 ix86_expand_vector_init_v16qi (rtx target, rtx *ops)
 {
   rtx vars[16];
+  bool ok;
   int i;
 
   bool all_zero_p = true;
@@ -19426,7 +19667,66 @@  ix86_expand_vector_init_v16qi (rtx target, rtx *ops)
   if (all_zero_p)
     {
       emit_move_insn (target, CONST0_RTX (V16QImode));
-      return true;
+      return;
+    }
+
+  bool all_same_p = true;
+  for (i = 1; i < 16; i++)
+    if (!rtx_equal_p (ops[i], ops[0]))
+      {
+	all_same_p = false;
+	break;
+      }
+  if (all_same_p)
+    {
+      ok = ix86_expand_vector_init_duplicate (false, V16QImode,
+					      target, ops[0]);
+      gcc_assert (ok);
+      return;
+    }
+
+  /* abababababababab */
+  if (TARGET_AVX2
+      && rtx_equal_p (ops[0], ops[2])
+      && rtx_equal_p (ops[1], ops[3])
+      && rtx_equal_p (ops[0], ops[4])
+      && rtx_equal_p (ops[1], ops[5])
+      && rtx_equal_p (ops[0], ops[6])
+      && rtx_equal_p (ops[1], ops[7])
+      && rtx_equal_p (ops[0], ops[8])
+      && rtx_equal_p (ops[1], ops[9])
+      && rtx_equal_p (ops[0], ops[10])
+      && rtx_equal_p (ops[1], ops[11])
+      && rtx_equal_p (ops[0], ops[12])
+      && rtx_equal_p (ops[1], ops[13])
+      && rtx_equal_p (ops[0], ops[14])
+      && rtx_equal_p (ops[1], ops[15]))
+    {
+      rtx tmp1 = gen_reg_rtx (V16QImode);
+      if (REG_P (ops[0]))
+	{
+	  rtx op0 = force_reg (SImode, gen_lowpart (SImode, ops[0]));
+	  rtx tmp = gen_reg_rtx (V4SImode);
+	  emit_insn (gen_vec_setv4si_0 (tmp, CONST0_RTX (V4SImode), op0));
+	  emit_move_insn (tmp1, gen_lowpart (V16QImode, tmp));
+	  rtx op1 = ops[1];
+	  if (!REG_P (op1) && !MEM_P (op1))
+	    op1 = force_reg (QImode, op1);
+	  emit_insn (gen_sse4_1_pinsrb (tmp1, tmp1, op1, GEN_INT (2)));
+	}
+      else
+	{
+	  vars[0] = ops[0];
+	  vars[1] = ops[1];
+	  for (i = 2; i < 16; i++)
+	    vars[i] = const0_rtx;
+	  ix86_expand_vector_init_v16qi (tmp1, vars);
+	}
+      tmp1 = gen_lowpart (V8HImode, tmp1);
+      rtx tmp2 = gen_reg_rtx (V8HImode);
+      emit_insn (gen_avx2_pbroadcastv8hi (tmp2, tmp1));
+      emit_move_insn (target, gen_lowpart (V16QImode, tmp2));
+      return;
     }
 
   bool all_const_p = true;
@@ -19438,15 +19738,31 @@  ix86_expand_vector_init_v16qi (rtx target, rtx *ops)
       }
   if (all_const_p)
     {
+      int last_nonzero = 15;
+      while (ops[last_nonzero] == const0_rtx)
+	last_nonzero--;
+      if (last_nonzero < 4)
+	{
+	  HOST_WIDE_INT val = (UINTVAL (ops[0]) & 0xff)
+			      + ((UINTVAL (ops[1]) & 0xff) << 8)
+			      + ((UINTVAL (ops[2]) & 0xff) << 16)
+			      + ((UINTVAL (ops[3]) & 0xff) << 24);
+	  rtx tmp1 = force_reg (SImode, gen_int_mode (val, SImode));
+	  rtx tmp2 = gen_reg_rtx (V4SImode);
+	  emit_insn (gen_vec_setv4si_0 (tmp2, CONST0_RTX (V4SImode), tmp1));
+	  emit_move_insn (target, gen_lowpart (V16QImode, tmp2));
+	  return;
+	}
       rtx vec = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, ops));
       emit_move_insn (target, vec);
-      return true;
+      return;
     }
 
   if (TARGET_SSE4_1
       && nonzero_int_const_count (ops, 16) >= 2)
     {
       rtx csts[16];
+      int count = 0;
       for (i = 0; i < 16; i++)
 	if (CONST_INT_P (ops[i]))
 	  {
@@ -19457,23 +19773,93 @@  ix86_expand_vector_init_v16qi (rtx target, rtx *ops)
 	  {
 	    csts[i] = const0_rtx;
 	    vars[i] = ops[i];
+	    count++;
 	  }
       rtx tmp1 = gen_reg_rtx (V16QImode);
-      if (!ix86_expand_vector_init_v16qi (tmp1, vars))
-	gcc_unreachable ();
+      if (count <= 3)
+	{
+	  ix86_expand_vector_init_v16qi (tmp1, csts);
+	  for (i=0; i<16; i++)
+	    if (vars[i] != const0_rtx)
+	      {
+		rtx elt = vars[i];
+		if (!REG_P (elt) && !MEM_P (elt))
+		  elt = force_reg (QImode, elt);
+		emit_insn (gen_sse4_1_pinsrb (tmp1, tmp1, elt,
+					      GEN_INT (1 << i)));
+	      }
+	  emit_move_insn (target, tmp1);
+	  return;
+	}
+      ix86_expand_vector_init_v16qi (tmp1, vars);
       rtx tmp2 = gen_reg_rtx (V16QImode);
       rtx vec = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, csts));
       emit_move_insn (tmp2, vec);
       emit_insn (gen_rtx_SET (target, gen_rtx_IOR (V16QImode, tmp1, tmp2)));
-      return true;
     }
-
-  if (TARGET_SSE4_1)
+  else if (TARGET_SSE4_1)
+    ix86_expand_vector_init_insert (V16QImode, target, ops, 16);
+  else if (TARGET_64BIT)
     {
-      ix86_expand_vector_init_insert (V16QImode, target, ops, 16);
-      return true;
+      rtx tmp = gen_reg_rtx (V2DImode);
+      vars[0] = ix86_expand_vector_init_qi8 (ops);
+      if (rtx_equal_p (ops[0], ops[8])
+	  && rtx_equal_p (ops[1], ops[9])
+	  && rtx_equal_p (ops[2], ops[10])
+	  && rtx_equal_p (ops[3], ops[11])
+	  && rtx_equal_p (ops[4], ops[12])
+	  && rtx_equal_p (ops[5], ops[13])
+	  && rtx_equal_p (ops[6], ops[14])
+	  && rtx_equal_p (ops[7], ops[15]))
+	vars[1] = vars[0];
+      else
+	vars[1] = ix86_expand_vector_init_qi8 (ops + 8);
+      ix86_expand_vector_init_v2di (tmp, vars);
+      emit_move_insn (target, gen_lowpart (V16QImode, tmp));
+    }
+  else
+    {
+      rtx tmp = gen_reg_rtx (V4SImode);
+      vars[0] = ix86_expand_vector_init_qi4 (ops);
+      if (rtx_equal_p (ops[0], ops[4])
+	  && rtx_equal_p (ops[1], ops[5])
+	  && rtx_equal_p (ops[2], ops[6])
+	  && rtx_equal_p (ops[3], ops[7]))
+	vars[1] = vars[0];
+      else
+	vars[1] = ix86_expand_vector_init_qi4 (ops + 4);
+      if (rtx_equal_p (ops[0], ops[8])
+	  && rtx_equal_p (ops[1], ops[9])
+	  && rtx_equal_p (ops[2], ops[10])
+	  && rtx_equal_p (ops[3], ops[11]))
+	vars[2] = vars[0];
+      else if (rtx_equal_p (ops[4], ops[8])
+	       && rtx_equal_p (ops[5], ops[9])
+	       && rtx_equal_p (ops[6], ops[10])
+	       && rtx_equal_p (ops[7], ops[11]))
+	vars[2] = vars[1];
+      else
+	vars[2] = ix86_expand_vector_init_qi4 (ops + 8);
+      if (rtx_equal_p (ops[0], ops[12])
+	  && rtx_equal_p (ops[1], ops[13])
+	  && rtx_equal_p (ops[2], ops[14])
+	  && rtx_equal_p (ops[3], ops[15]))
+	vars[3] = vars[0];
+      else if (rtx_equal_p (ops[4], ops[12])
+	       && rtx_equal_p (ops[5], ops[13])
+	       && rtx_equal_p (ops[6], ops[14])
+	       && rtx_equal_p (ops[7], ops[15]))
+	vars[3] = vars[1];
+      else if (rtx_equal_p (ops[8], ops[12])
+	       && rtx_equal_p (ops[9], ops[13])
+	       && rtx_equal_p (ops[10], ops[14])
+	       && rtx_equal_p (ops[11], ops[15]))
+	vars[3] = vars[2];
+      else
+	vars[3] = ix86_expand_vector_init_qi4 (ops + 12);
+      ix86_expand_vector_init_v4si (tmp, vars);
+      emit_move_insn (target, gen_lowpart (V16QImode, tmp));
     }
-  return false;
 }
 
 /* A subroutine of ix86_expand_vector_init for V4DImode.  */
@@ -19953,9 +20339,8 @@  ix86_expand_vector_init_general (bool mmx_ok, machine_mode mode,
     case E_V16QImode:
       for (i = 0; i < 16; i++)
 	ops[i] = XVECEXP (vals, 0, i);
-      if (ix86_expand_vector_init_v16qi (target, ops))
-	return;
-      break;
+      ix86_expand_vector_init_v16qi (target, ops);
+      return;
 
     case E_V4DImode:
       for (i = 0; i < 4; i++)
diff --git a/gcc/testsuite/gcc.target/i386/avx-init-v16qi-1.c b/gcc/testsuite/gcc.target/i386/avx-init-v16qi-1.c
index f608d094074..20893761818 100644
--- a/gcc/testsuite/gcc.target/i386/avx-init-v16qi-1.c
+++ b/gcc/testsuite/gcc.target/i386/avx-init-v16qi-1.c
@@ -22,9 +22,23 @@  v16qi f0000000000000a00() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,a,0,0}; }
 v16qi f00000000000000a0() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,a,0}; }
 v16qi f000000000000000a() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,a}; }
 
+v16qi faa00000000000000() { return (v16qi){a,a,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0a0000000000000() { return (v16qi){a,0,a,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa000a00000000000() { return (v16qi){a,0,0,0,a,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0000000a0000000() { return (v16qi){a,0,0,0,0,0,0,0,a,0,0,0,0,0,0,0}; }
+
+v16qi faaaaaaaaaaaaaaaa() { return (v16qi){a,a,a,a,a,a,a,a,a,a,a,a,a,a,a,a}; }
+v16qi faaaaaaaabbbbbbbb() { return (v16qi){a,a,a,a,a,a,a,a,b,b,b,b,b,b,b,b}; }
+v16qi faaaabbbbaaaabbbb() { return (v16qi){a,a,a,a,b,b,b,b,a,a,a,a,b,b,b,b}; }
+v16qi fabababababababab() { return (v16qi){a,b,a,b,a,b,a,b,a,b,a,b,a,b,a,b}; }
+
+v16qi fabcd000000000000() { return (v16qi){a,b,c,d,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fabcdefgh00000000() { return (v16qi){a,b,c,d,e,f,g,h,0,0,0,0,0,0,0,0}; }
 v16qi fabcdefghijklmnop() { return (v16qi){a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}; }
 
-/* { dg-final { scan-assembler-times "vpxor" 16 } } */
-/* { dg-final { scan-assembler-times "vpinsrb" 31 } } */
-/* { dg-final { scan-assembler-times "movzbl" 1 } } */
-/* { dg-final { scan-assembler-times "vmovd" 1 } } */
+/* { dg-final { scan-assembler-not "movaps" } } */
+/* { dg-final { scan-assembler-times "movzbl" 14 } } */
+/* { dg-final { scan-assembler-times "vmovd" 11 } } */
+/* { dg-final { scan-assembler-times "vpinsrb" 90 } } */
+/* { dg-final { scan-assembler-times "vpshufb" 1 } } */
+/* { dg-final { scan-assembler-times "vpxor" 17 } } */
diff --git a/gcc/testsuite/gcc.target/i386/avx2-init-v16qi-1.c b/gcc/testsuite/gcc.target/i386/avx2-init-v16qi-1.c
new file mode 100644
index 00000000000..de1d61f4835
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx2-init-v16qi-1.c
@@ -0,0 +1,46 @@ 
+/* { dg-do compile } */
+/* { dg-options "-O2 -mavx2 -mno-avx512vl" } */
+
+typedef char v16qi __attribute__ ((__vector_size__ (16)));
+
+char a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p;
+
+v16qi fa000000000000000() { return (v16qi){a,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi f0a00000000000000() { return (v16qi){0,a,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi f00a0000000000000() { return (v16qi){0,0,a,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi f000a000000000000() { return (v16qi){0,0,0,a,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi f0000a00000000000() { return (v16qi){0,0,0,0,a,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi f00000a0000000000() { return (v16qi){0,0,0,0,0,a,0,0,0,0,0,0,0,0,0,0}; }
+v16qi f000000a000000000() { return (v16qi){0,0,0,0,0,0,a,0,0,0,0,0,0,0,0,0}; }
+v16qi f0000000a00000000() { return (v16qi){0,0,0,0,0,0,0,a,0,0,0,0,0,0,0,0}; }
+v16qi f00000000a0000000() { return (v16qi){0,0,0,0,0,0,0,0,a,0,0,0,0,0,0,0}; }
+v16qi f000000000a000000() { return (v16qi){0,0,0,0,0,0,0,0,0,a,0,0,0,0,0,0}; }
+v16qi f0000000000a00000() { return (v16qi){0,0,0,0,0,0,0,0,0,0,a,0,0,0,0,0}; }
+v16qi f00000000000a0000() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,a,0,0,0,0}; }
+v16qi f000000000000a000() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,a,0,0,0}; }
+v16qi f0000000000000a00() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,a,0,0}; }
+v16qi f00000000000000a0() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,a,0}; }
+v16qi f000000000000000a() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,a}; }
+
+v16qi faa00000000000000() { return (v16qi){a,a,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0a0000000000000() { return (v16qi){a,0,a,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa000a00000000000() { return (v16qi){a,0,0,0,a,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0000000a0000000() { return (v16qi){a,0,0,0,0,0,0,0,a,0,0,0,0,0,0,0}; }
+
+v16qi faaaaaaaaaaaaaaaa() { return (v16qi){a,a,a,a,a,a,a,a,a,a,a,a,a,a,a,a}; }
+v16qi faaaaaaaabbbbbbbb() { return (v16qi){a,a,a,a,a,a,a,a,b,b,b,b,b,b,b,b}; }
+v16qi faaaabbbbaaaabbbb() { return (v16qi){a,a,a,a,b,b,b,b,a,a,a,a,b,b,b,b}; }
+v16qi fabababababababab() { return (v16qi){a,b,a,b,a,b,a,b,a,b,a,b,a,b,a,b}; }
+
+v16qi fabcd000000000000() { return (v16qi){a,b,c,d,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fabcdefgh00000000() { return (v16qi){a,b,c,d,e,f,g,h,0,0,0,0,0,0,0,0}; }
+v16qi fabcdefghijklmnop() { return (v16qi){a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}; }
+
+/* { dg-final { scan-assembler-not "movaps" } } */
+/* { dg-final { scan-assembler-times "movzbl" 12 } } */
+/* { dg-final { scan-assembler-times "vmovd" 10 } } */
+/* { dg-final { scan-assembler-times "vpbroadcastb" 1 } } */
+/* { dg-final { scan-assembler-times "vpbroadcastw" 1 } } */
+/* { dg-final { scan-assembler-times "vpinsrb" 76 } } */
+/* { dg-final { scan-assembler-not "vpshufb" } } */
+/* { dg-final { scan-assembler-times "vpxor" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/sse-init-v16qi-1.c b/gcc/testsuite/gcc.target/i386/sse-init-v16qi-1.c
index 244dda3e59a..ade96f0287e 100644
--- a/gcc/testsuite/gcc.target/i386/sse-init-v16qi-1.c
+++ b/gcc/testsuite/gcc.target/i386/sse-init-v16qi-1.c
@@ -22,6 +22,18 @@  v16qi f0000000000000a00() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,a,0,0}; }
 v16qi f00000000000000a0() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,a,0}; }
 v16qi f000000000000000a() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,a}; }
 
+v16qi faa00000000000000() { return (v16qi){a,a,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0a0000000000000() { return (v16qi){a,0,a,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa000a00000000000() { return (v16qi){a,0,0,0,a,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0000000a0000000() { return (v16qi){a,0,0,0,0,0,0,0,a,0,0,0,0,0,0,0}; }
+
+v16qi faaaaaaaaaaaaaaaa() { return (v16qi){a,a,a,a,a,a,a,a,a,a,a,a,a,a,a,a}; }
+v16qi faaaaaaaabbbbbbbb() { return (v16qi){a,a,a,a,a,a,a,a,b,b,b,b,b,b,b,b}; }
+v16qi faaaabbbbaaaabbbb() { return (v16qi){a,a,a,a,b,b,b,b,a,a,a,a,b,b,b,b}; }
+v16qi fabababababababab() { return (v16qi){a,b,a,b,a,b,a,b,a,b,a,b,a,b,a,b}; }
+
+v16qi fabcd000000000000() { return (v16qi){a,b,c,d,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fabcdefgh00000000() { return (v16qi){a,b,c,d,e,f,g,h,0,0,0,0,0,0,0,0}; }
 v16qi fabcdefghijklmnop() { return (v16qi){a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}; }
 
-/* { dg-final { scan-assembler-times "movaps" 17 } } */
+/* { dg-final { scan-assembler-times "movaps" 27 } } */
diff --git a/gcc/testsuite/gcc.target/i386/sse2-init-v16qi-2.c b/gcc/testsuite/gcc.target/i386/sse2-init-v16qi-2.c
index b448ecfe01b..ee2ff892133 100644
--- a/gcc/testsuite/gcc.target/i386/sse2-init-v16qi-2.c
+++ b/gcc/testsuite/gcc.target/i386/sse2-init-v16qi-2.c
@@ -22,11 +22,29 @@  v16qi f0000000000000a00() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,a,0,0}; }
 v16qi f00000000000000a0() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,a,0}; }
 v16qi f000000000000000a() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,a}; }
 
+v16qi faa00000000000000() { return (v16qi){a,a,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0a0000000000000() { return (v16qi){a,0,a,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa000a00000000000() { return (v16qi){a,0,0,0,a,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0000000a0000000() { return (v16qi){a,0,0,0,0,0,0,0,a,0,0,0,0,0,0,0}; }
+
+v16qi faaaaaaaaaaaaaaaa() { return (v16qi){a,a,a,a,a,a,a,a,a,a,a,a,a,a,a,a}; }
+v16qi faaaaaaaabbbbbbbb() { return (v16qi){a,a,a,a,a,a,a,a,b,b,b,b,b,b,b,b}; }
+v16qi faaaabbbbaaaabbbb() { return (v16qi){a,a,a,a,b,b,b,b,a,a,a,a,b,b,b,b}; }
+v16qi fabababababababab() { return (v16qi){a,b,a,b,a,b,a,b,a,b,a,b,a,b,a,b}; }
+
+v16qi fabcd000000000000() { return (v16qi){a,b,c,d,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fabcdefgh00000000() { return (v16qi){a,b,c,d,e,f,g,h,0,0,0,0,0,0,0,0}; }
 v16qi fabcdefghijklmnop() { return (v16qi){a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}; }
 
-/* { dg-final { scan-assembler-times "movzbl" 32 } } */
-/* { dg-final { scan-assembler-times "movd" 16 } } */
-/* { dg-final { scan-assembler-times "sall" 3 } } */
+/* { dg-final { scan-assembler-times "addq" 42 } } */
+/* { dg-final { scan-assembler-not "movaps" } } */
+/* { dg-final { scan-assembler-times "movd" 17 } } */
+/* { dg-final { scan-assembler-times "movq" 25 } } */
+/* { dg-final { scan-assembler-times "movzbl" 55 } } */
+/* { dg-final { scan-assembler-times "pshufd" 1 } } */
 /* { dg-final { scan-assembler-times "pslldq" 12 } } */
-/* { dg-final { scan-assembler-times "movq" 2 } } */
-/* { dg-final { scan-assembler-times "punpcklqdq" 1 } } */
+/* { dg-final { scan-assembler-times "punpcklbw" 1 } } */
+/* { dg-final { scan-assembler-times "punpcklwd" 1 } } */
+/* { dg-final { scan-assembler-times "punpcklqdq" 5 } } */
+/* { dg-final { scan-assembler-times "sall" 3 } } */
+/* { dg-final { scan-assembler-times "salq" 42 } } */
diff --git a/gcc/testsuite/gcc.target/i386/sse2-init-v16qi-3.c b/gcc/testsuite/gcc.target/i386/sse2-init-v16qi-3.c
index ea741e38bfa..ea79dab7b11 100644
--- a/gcc/testsuite/gcc.target/i386/sse2-init-v16qi-3.c
+++ b/gcc/testsuite/gcc.target/i386/sse2-init-v16qi-3.c
@@ -22,11 +22,29 @@  v16qi f0000000000000a00() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,a,0,0}; }
 v16qi f00000000000000a0() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,a,0}; }
 v16qi f000000000000000a() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,a}; }
 
+v16qi faa00000000000000() { return (v16qi){a,a,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0a0000000000000() { return (v16qi){a,0,a,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa000a00000000000() { return (v16qi){a,0,0,0,a,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0000000a0000000() { return (v16qi){a,0,0,0,0,0,0,0,a,0,0,0,0,0,0,0}; }
+
+v16qi faaaaaaaaaaaaaaaa() { return (v16qi){a,a,a,a,a,a,a,a,a,a,a,a,a,a,a,a}; }
+v16qi faaaaaaaabbbbbbbb() { return (v16qi){a,a,a,a,a,a,a,a,b,b,b,b,b,b,b,b}; }
+v16qi faaaabbbbaaaabbbb() { return (v16qi){a,a,a,a,b,b,b,b,a,a,a,a,b,b,b,b}; }
+v16qi fabababababababab() { return (v16qi){a,b,a,b,a,b,a,b,a,b,a,b,a,b,a,b}; }
+
+v16qi fabcd000000000000() { return (v16qi){a,b,c,d,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fabcdefgh00000000() { return (v16qi){a,b,c,d,e,f,g,h,0,0,0,0,0,0,0,0}; }
 v16qi fabcdefghijklmnop() { return (v16qi){a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}; }
 
-/* { dg-final { scan-assembler-times "movzbl" 32 } } */
-/* { dg-final { scan-assembler-times "movd" 20 } } */
-/* { dg-final { scan-assembler-times "sall" 15 } } */
+/* { dg-final { scan-assembler-times "addl" 33 } } */
+/* { dg-final { scan-assembler-not "movaps" } } */
+/* { dg-final { scan-assembler-times "movd" 33 } } */
+/* { dg-final { scan-assembler-not "movq" } } */
+/* { dg-final { scan-assembler-times "movzbl" 55 } } */
+/* { dg-final { scan-assembler-times "pshufd" 2 } } */
 /* { dg-final { scan-assembler-times "pslldq" 12 } } */
-/* { dg-final { scan-assembler-times "punpckldq" 2 } } */
-/* { dg-final { scan-assembler-times "shufps" 1 } } */
+/* { dg-final { scan-assembler-times "punpcklbw" 1 } } */
+/* { dg-final { scan-assembler-times "punpcklwd" 1 } } */
+/* { dg-final { scan-assembler-times "punpckldq" 5 } } */
+/* { dg-final { scan-assembler-times "sall" 36 } } */
+/* { dg-final { scan-assembler-times "shufps" 5 } } */
diff --git a/gcc/testsuite/gcc.target/i386/sse4_1-init-v16qi-2.c b/gcc/testsuite/gcc.target/i386/sse4_1-init-v16qi-2.c
index bc6748b0e7b..107b5f38ee0 100644
--- a/gcc/testsuite/gcc.target/i386/sse4_1-init-v16qi-2.c
+++ b/gcc/testsuite/gcc.target/i386/sse4_1-init-v16qi-2.c
@@ -22,9 +22,23 @@  v16qi f0000000000000a00() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,a,0,0}; }
 v16qi f00000000000000a0() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,a,0}; }
 v16qi f000000000000000a() { return (v16qi){0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,a}; }
 
+v16qi faa00000000000000() { return (v16qi){a,a,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0a0000000000000() { return (v16qi){a,0,a,0,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa000a00000000000() { return (v16qi){a,0,0,0,a,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fa0000000a0000000() { return (v16qi){a,0,0,0,0,0,0,0,a,0,0,0,0,0,0,0}; }
+
+v16qi faaaaaaaaaaaaaaaa() { return (v16qi){a,a,a,a,a,a,a,a,a,a,a,a,a,a,a,a}; }
+v16qi faaaaaaaabbbbbbbb() { return (v16qi){a,a,a,a,a,a,a,a,b,b,b,b,b,b,b,b}; }
+v16qi faaaabbbbaaaabbbb() { return (v16qi){a,a,a,a,b,b,b,b,a,a,a,a,b,b,b,b}; }
+v16qi fabababababababab() { return (v16qi){a,b,a,b,a,b,a,b,a,b,a,b,a,b,a,b}; }
+
+v16qi fabcd000000000000() { return (v16qi){a,b,c,d,0,0,0,0,0,0,0,0,0,0,0,0}; }
+v16qi fabcdefgh00000000() { return (v16qi){a,b,c,d,e,f,g,h,0,0,0,0,0,0,0,0}; }
 v16qi fabcdefghijklmnop() { return (v16qi){a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p}; }
 
-/* { dg-final { scan-assembler-times "pxor" 16 } } */
-/* { dg-final { scan-assembler-times "pinsrb" 31 } } */
-/* { dg-final { scan-assembler-times "movzbl" 1 } } */
-/* { dg-final { scan-assembler-times "movd" 1 } } */
+/* { dg-final { scan-assembler-not "movaps" } } */
+/* { dg-final { scan-assembler-times "movd" 11 } } */
+/* { dg-final { scan-assembler-times "movzbl" 14 } } */
+/* { dg-final { scan-assembler-times "pinsrb" 90 } } */
+/* { dg-final { scan-assembler-times "pxor" 17 } } */
+