@@ -305,6 +305,7 @@ static machine_mode aarch64_simd_container_mode (scalar_mode, poly_int64);
static bool aarch64_print_address_internal (FILE*, machine_mode, rtx,
aarch64_addr_query_type);
static HOST_WIDE_INT aarch64_clamp_to_uimm12_shift (HOST_WIDE_INT val);
+static void aarch64_expand_vector_init (rtx, rtx_vector_builder&);
/* The processor for which instructions should be scheduled. */
enum aarch64_processor aarch64_tune = cortexa53;
@@ -21804,55 +21805,96 @@ aarch64_simd_make_constant (rtx vals)
return NULL_RTX;
}
+static void
+aarch64_vec_duplicate (rtx target, machine_mode mode, machine_mode element_mode,
+ int narrow_n_elts)
+{
+ poly_uint64 size = narrow_n_elts * GET_MODE_BITSIZE (element_mode);
+ scalar_mode i_mode = int_mode_for_size (size, 0).require ();
+ machine_mode o_mode;
+ if (aarch64_sve_mode_p (mode))
+ o_mode = aarch64_full_sve_mode (i_mode).require ();
+ else
+ o_mode
+ = aarch64_simd_container_mode (i_mode,
+ GET_MODE_BITSIZE (mode));
+ rtx input = simplify_gen_subreg (i_mode, target, mode, 0);
+ rtx output = simplify_gen_subreg (o_mode, target, mode, 0);
+ aarch64_emit_move (output, gen_vec_duplicate (o_mode, input));
+}
+
+
/* Expand a vector initialisation sequence, such that TARGET is
initialised to contain VALS. */
void
aarch64_expand_vector_init (rtx target, rtx vals)
{
- machine_mode mode = GET_MODE (target);
- scalar_mode inner_mode = GET_MODE_INNER (mode);
/* The number of vector elements. */
int n_elts = XVECLEN (vals, 0);
- /* The number of vector elements which are not constant. */
- int n_var = 0;
- rtx any_const = NULL_RTX;
+ machine_mode mode = GET_MODE (target);
+ scalar_mode inner_mode = GET_MODE_INNER (mode);
/* The first element of vals. */
rtx v0 = XVECEXP (vals, 0, 0);
- bool all_same = true;
/* This is a special vec_init<M><N> where N is not an element mode but a
vector mode with half the elements of M. We expect to find two entries
of mode N in VALS and we must put their concatentation into TARGET. */
- if (XVECLEN (vals, 0) == 2 && VECTOR_MODE_P (GET_MODE (XVECEXP (vals, 0, 0))))
+ if (n_elts == 2
+ && VECTOR_MODE_P (GET_MODE (v0)))
{
- machine_mode narrow_mode = GET_MODE (XVECEXP (vals, 0, 0));
+ machine_mode narrow_mode = GET_MODE (v0);
gcc_assert (GET_MODE_INNER (narrow_mode) == inner_mode
&& known_eq (GET_MODE_SIZE (mode),
2 * GET_MODE_SIZE (narrow_mode)));
- emit_insn (gen_aarch64_vec_concat (narrow_mode, target,
- XVECEXP (vals, 0, 0),
+ emit_insn (gen_aarch64_vec_concat (narrow_mode, target, v0,
XVECEXP (vals, 0, 1)));
return;
}
- /* Count the number of variable elements to initialise. */
+ rtx_vector_builder builder (mode, n_elts, 1);
for (int i = 0; i < n_elts; ++i)
+ builder.quick_push (XVECEXP (vals, 0, i));
+ builder.finalize ();
+
+ aarch64_expand_vector_init (target, builder);
+}
+
+static void
+aarch64_expand_vector_init (rtx target, rtx_vector_builder &v)
+{
+ machine_mode mode = GET_MODE (target);
+ scalar_mode inner_mode = GET_MODE_INNER (mode);
+ /* The number of vector elements which are not constant. */
+ unsigned n_var = 0;
+ rtx any_const = NULL_RTX;
+ /* The first element of vals. */
+ rtx v0 = v.elt (0);
+ /* Get the number of elements to insert into an Advanced SIMD vector.
+ If we have more than one element per pattern then we use the constant
+ number of elements in a full vector.
+ If we only have one element per pattern we use the number of patterns as
+ this may be lower than the number of elements in a full vector, which
+ means they repeat and we should use a duplicate of the smaller vector. */
+ unsigned n_elts
+ = v.nelts_per_pattern () == 1 ? v.npatterns ()
+ : v.full_nelts ().coeffs[0];
+
+ /* Count the number of variable elements to initialise. */
+ for (unsigned i = 0; i < n_elts ; ++i)
{
- rtx x = XVECEXP (vals, 0, i);
+ rtx x = v.elt (i);
if (!(CONST_INT_P (x) || CONST_DOUBLE_P (x)))
++n_var;
else
any_const = x;
-
- all_same &= rtx_equal_p (x, v0);
}
/* No variable elements, hand off to aarch64_simd_make_constant which knows
how best to handle this. */
if (n_var == 0)
{
- rtx constant = aarch64_simd_make_constant (vals);
+ rtx constant = aarch64_simd_make_constant (v.build ());
if (constant != NULL_RTX)
{
emit_move_insn (target, constant);
@@ -21861,7 +21903,7 @@ aarch64_expand_vector_init (rtx target, rtx vals)
}
/* Splat a single non-constant element if we can. */
- if (all_same)
+ if (n_elts == 1)
{
rtx x = copy_to_mode_reg (inner_mode, v0);
aarch64_emit_move (target, gen_vec_duplicate (mode, x));
@@ -21879,14 +21921,15 @@ aarch64_expand_vector_init (rtx target, rtx vals)
and matches[X][1] with the count of duplicate elements (if X is the
earliest element which has duplicates). */
- if (n_var == n_elts && n_elts <= 16)
+ if (n_var == n_elts)
{
- int matches[16][2] = {0};
- for (int i = 0; i < n_elts; i++)
+ gcc_assert (n_elts <= 16);
+ unsigned matches[16][2] = {0};
+ for (unsigned i = 0; i < n_elts; i++)
{
- for (int j = 0; j <= i; j++)
+ for (unsigned j = 0; j <= i; j++)
{
- if (rtx_equal_p (XVECEXP (vals, 0, i), XVECEXP (vals, 0, j)))
+ if (rtx_equal_p (v.elt (i), v.elt (j)))
{
matches[i][0] = j;
matches[j][1]++;
@@ -21894,9 +21937,9 @@ aarch64_expand_vector_init (rtx target, rtx vals)
}
}
}
- int maxelement = 0;
- int maxv = 0;
- for (int i = 0; i < n_elts; i++)
+ unsigned maxelement = 0;
+ unsigned maxv = 0;
+ for (unsigned i = 0; i < n_elts; i++)
if (matches[i][1] > maxv)
{
maxelement = i;
@@ -21915,8 +21958,8 @@ aarch64_expand_vector_init (rtx target, rtx vals)
|| inner_mode == E_DFmode))
{
- rtx x0 = XVECEXP (vals, 0, 0);
- rtx x1 = XVECEXP (vals, 0, 1);
+ rtx x0 = v.elt (0);
+ rtx x1 = v.elt (1);
/* Combine can pick up this case, but handling it directly
here leaves clearer RTL.
@@ -21939,24 +21982,26 @@ aarch64_expand_vector_init (rtx target, rtx vals)
vector register. For big-endian we want that position to hold
the last element of VALS. */
maxelement = BYTES_BIG_ENDIAN ? n_elts - 1 : 0;
- rtx x = copy_to_mode_reg (inner_mode, XVECEXP (vals, 0, maxelement));
+ rtx x = copy_to_mode_reg (inner_mode, v.elt (maxelement));
aarch64_emit_move (target, lowpart_subreg (mode, x, inner_mode));
}
else
{
- rtx x = copy_to_mode_reg (inner_mode, XVECEXP (vals, 0, maxelement));
+ rtx x = copy_to_mode_reg (inner_mode, v.elt (maxelement));
aarch64_emit_move (target, gen_vec_duplicate (mode, x));
}
/* Insert the rest. */
- for (int i = 0; i < n_elts; i++)
+ for (unsigned i = 0; i < n_elts; i++)
{
- rtx x = XVECEXP (vals, 0, i);
+ rtx x = v.elt (i);
if (matches[i][0] == maxelement)
continue;
x = copy_to_mode_reg (inner_mode, x);
emit_insn (GEN_FCN (icode) (target, x, GEN_INT (i)));
}
+ if (!known_eq (v.full_nelts (), n_elts))
+ aarch64_vec_duplicate (target, mode, GET_MODE (v0), n_elts);
return;
}
@@ -21965,19 +22010,19 @@ aarch64_expand_vector_init (rtx target, rtx vals)
can. */
if (n_var != n_elts)
{
- rtx copy = copy_rtx (vals);
+ rtx copy = v.build ();
/* Load constant part of vector. We really don't care what goes into the
parts we will overwrite, but we're more likely to be able to load the
constant efficiently if it has fewer, larger, repeating parts
(see aarch64_simd_valid_immediate). */
- for (int i = 0; i < n_elts; i++)
+ for (unsigned i = 0; i < n_elts; i++)
{
- rtx x = XVECEXP (vals, 0, i);
+ rtx x = XVECEXP (copy, 0, i);
if (CONST_INT_P (x) || CONST_DOUBLE_P (x))
continue;
rtx subst = any_const;
- for (int bit = n_elts / 2; bit > 0; bit /= 2)
+ for (unsigned bit = n_elts / 2; bit > 0; bit /= 2)
{
/* Look in the copied vector, as more elements are const. */
rtx test = XVECEXP (copy, 0, i ^ bit);
@@ -21989,18 +22034,21 @@ aarch64_expand_vector_init (rtx target, rtx vals)
}
XVECEXP (copy, 0, i) = subst;
}
+ gcc_assert (GET_MODE (target) == GET_MODE (copy));
aarch64_expand_vector_init (target, copy);
}
/* Insert the variable lanes directly. */
- for (int i = 0; i < n_elts; i++)
+ for (unsigned i = 0; i < n_elts; i++)
{
- rtx x = XVECEXP (vals, 0, i);
+ rtx x = v.elt (i);
if (CONST_INT_P (x) || CONST_DOUBLE_P (x))
continue;
x = copy_to_mode_reg (inner_mode, x);
emit_insn (GEN_FCN (icode) (target, x, GEN_INT (i)));
}
+ if (!known_eq (v.full_nelts (), n_elts))
+ aarch64_vec_duplicate (target, mode, inner_mode, n_elts);
}
/* Emit RTL corresponding to:
@@ -96,8 +96,8 @@ CONS2_FN (4, float);
/*
** cons2_8_float:
-** dup v([0-9]+)\.4s, .*
-** ...
+** ins v0\.s\[1\], v1\.s\[0\]
+** dup v([0-9]+)\.2d, v0\.d\[0\]
** stp q\1, q\1, \[x0\]
** stp q\1, q\1, \[x0, #?32\]
** ret
new file mode 100644
@@ -0,0 +1,144 @@
+#include <arm_neon.h>
+
+/*
+** int32_0:
+** fmov s0, w0
+** ins v0.s\[1\], w1
+** dup v0.2d, v0.d\[0\]
+** ret
+*/
+
+int32x4_t int32_0 (int a, int b)
+{
+ int32x4_t v = {a, b, a, b};
+ return v;
+}
+/*
+** int32_1:
+** dup v0.4s, w0
+** ret
+*/
+
+int32x4_t int32_1 (int a)
+{
+ int32x4_t v = {a, a, a, a};
+ return v;
+}
+
+/*
+** int16_0:
+** sxth w0, w0
+** fmov s0, w0
+** ins v0.h\[1\], w1
+** ins v0.h\[2\], w2
+** ins v0.h\[3\], w3
+** dup v0.2d, v0.d\[0\]
+** ret
+*/
+
+int16x8_t int16_0 (int16_t a, int16_t b, int16_t c, int16_t d)
+{
+ int16x8_t v = {a, b, c, d,
+ a, b, c, d};
+ return v;
+}
+
+/*
+** int16_1:
+** sxth w0, w0
+** fmov s0, w0
+** ins v0.h\[1\], w1
+** dup v0.4s, v0.s\[0\]
+** ret
+*/
+
+int16x8_t int16_1 (int16_t a, int16_t b)
+{
+ int16x8_t v = {a, b, a, b,
+ a, b, a, b};
+ return v;
+}
+
+/*
+** int16_2:
+** dup v0.8h, w0
+** ret
+*/
+
+int16x8_t int16_2 (int16_t a)
+{
+ int16x8_t v = {a, a, a, a,
+ a, a, a, a};
+ return v;
+}
+
+/*
+** int8_0:
+** sxtb w0, w0
+** fmov s0, w0
+** ins v0.b\[1\], w1
+** ins v0.b\[2\], w2
+** ins v0.b\[3\], w3
+** ins v0.b\[4\], w4
+** ins v0.b\[5\], w5
+** ins v0.b\[6\], w6
+** ins v0.b\[7\], w7
+** dup v0.2d, v0.d\[0\]
+** ret
+*/
+
+int8x16_t int8_0 (int8_t a, int8_t b, int8_t c, int8_t d, int8_t e, int8_t f,
+ int8_t g, int8_t h)
+{
+ int8x16_t v = {a, b, c, d, e, f, g, h,
+ a, b, c, d, e, f, g, h};
+ return v;
+}
+
+/*
+** int8_1:
+** sxtb w0, w0
+** fmov s0, w0
+** ins v0.b\[1\], w1
+** ins v0.b\[2\], w2
+** ins v0.b\[3\], w3
+** dup v0.4s, v0.s\[0\]
+** ret
+*/
+
+int8x16_t int8_1 (int8_t a, int8_t b, int8_t c, int8_t d)
+{
+ int8x16_t v = {a, b, c, d, a, b, c, d,
+ a, b, c, d, a, b, c, d};
+ return v;
+}
+
+/*
+** int8_2:
+** sxtb w0, w0
+** fmov s0, w0
+** ins v0.b\[1\], w1
+** dup v0.8h, v0.h\[0\]
+** ret
+*/
+
+int8x16_t int8_2 (int8_t a, int8_t b)
+{
+ int8x16_t v = {a, b, a, b, a, b, a, b,
+ a, b, a, b, a, b, a, b};
+ return v;
+}
+
+/*
+** int8_3:
+** dup v0.16b, w0
+** ret
+*/
+
+int8x16_t int8_3 (int8_t a)
+{
+ int8x16_t v = {a, a, a, a, a, a, a, a,
+ a, a, a, a, a, a, a, a};
+ return v;
+}
+