@@ -149,20 +149,20 @@ (define_insn_and_split "*aarch64_simd_mov<VDMOV:mode>"
&& (register_operand (operands[0], <MODE>mode)
|| aarch64_simd_reg_or_zero (operands[1], <MODE>mode))"
{@ [cons: =0, 1; attrs: type, arch, length]
- [w , m ; neon_load1_1reg<q> , * , *] ldr\t%d0, %1
- [r , m ; load_8 , * , *] ldr\t%x0, %1
- [m , Dz; store_8 , * , *] str\txzr, %0
- [m , w ; neon_store1_1reg<q>, * , *] str\t%d1, %0
- [m , r ; store_8 , * , *] str\t%x1, %0
- [w , w ; neon_logic<q> , simd, *] mov\t%0.<Vbtype>, %1.<Vbtype>
- [w , w ; neon_logic<q> , * , *] fmov\t%d0, %d1
- [?r, w ; neon_to_gp<q> , simd, *] umov\t%0, %1.d[0]
- [?r, w ; neon_to_gp<q> , * , *] fmov\t%x0, %d1
- [?w, r ; f_mcr , * , *] fmov\t%d0, %1
- [?r, r ; mov_reg , * , *] mov\t%0, %1
- [w , Dn; neon_move<q> , simd, *] << aarch64_output_simd_mov_immediate (operands[1], 64);
- [w , Dz; f_mcr , * , *] fmov\t%d0, xzr
- [w , Dx; neon_move , simd, 8] #
+ [w , m ; neon_load1_1reg<q> , * , *] ldr\t%d0, %1
+ [r , m ; load_8 , * , *] ldr\t%x0, %1
+ [m , Dz; store_8 , * , *] str\txzr, %0
+ [m , w ; neon_store1_1reg<q>, * , *] str\t%d1, %0
+ [m , r ; store_8 , * , *] str\t%x1, %0
+ [w , w ; neon_logic<q> , simd , *] mov\t%0.<Vbtype>, %1.<Vbtype>
+ [w , w ; neon_logic<q> , * , *] fmov\t%d0, %d1
+ [?r, w ; neon_to_gp<q> , base_simd, *] umov\t%0, %1.d[0]
+ [?r, w ; neon_to_gp<q> , * , *] fmov\t%x0, %d1
+ [?w, r ; f_mcr , * , *] fmov\t%d0, %1
+ [?r, r ; mov_reg , * , *] mov\t%0, %1
+ [w , Dn; neon_move<q> , simd , *] << aarch64_output_simd_mov_immediate (operands[1], 64);
+ [w , Dz; f_mcr , * , *] fmov\t%d0, xzr
+ [w , Dx; neon_move , simd , 8] #
}
"CONST_INT_P (operands[1])
&& aarch64_simd_special_constant_p (operands[1], <MODE>mode)
@@ -185,6 +185,7 @@ (define_insn_and_split "*aarch64_simd_mov<VQMOV:mode>"
[Umn, Dz; store_16 , * , 4] stp\txzr, xzr, %0
[m , w ; neon_store1_1reg<q>, * , 4] str\t%q1, %0
[w , w ; neon_logic<q> , simd, 4] mov\t%0.<Vbtype>, %1.<Vbtype>
+ [w , w ; * , sve , 4] mov\t%Z0.d, %Z1.d
[?r , w ; multiple , * , 8] #
[?w , r ; multiple , * , 8] #
[?r , r ; multiple , * , 8] #
@@ -225,7 +226,7 @@ (define_insn "aarch64_store_lane0<mode>"
[(set (match_operand:<VEL> 0 "memory_operand" "=m")
(vec_select:<VEL> (match_operand:VALL_F16 1 "register_operand" "w")
(parallel [(match_operand 2 "const_int_operand" "n")])))]
- "TARGET_SIMD
+ "TARGET_FLOAT
&& ENDIAN_LANE_N (<nunits>, INTVAL (operands[2])) == 0"
"str\\t%<Vetype>1, %0"
[(set_attr "type" "neon_store1_1reg<q>")]
@@ -374,18 +375,18 @@ (define_insn_and_split "aarch64_simd_mov_from_<mode>low"
(vec_select:<VHALF>
(match_operand:VQMOV_NO2E 1 "register_operand")
(match_operand:VQMOV_NO2E 2 "vect_par_cnst_lo_half")))]
- "TARGET_SIMD"
- {@ [ cons: =0 , 1 ; attrs: type ]
- [ w , w ; mov_reg ] #
- [ ?r , w ; neon_to_gp<q> ] umov\t%0, %1.d[0]
+ "TARGET_FLOAT"
+ {@ [ cons: =0 , 1 ; attrs: type , arch ]
+ [ w , w ; mov_reg , simd ] #
+ [ ?r , w ; neon_to_gp<q> , base_simd ] umov\t%0, %1.d[0]
+ [ ?r , w ; f_mrc , * ] fmov\t%0, %d1
}
"&& reload_completed && aarch64_simd_register (operands[0], <VHALF>mode)"
[(set (match_dup 0) (match_dup 1))]
{
operands[1] = aarch64_replace_reg_mode (operands[1], <VHALF>mode);
}
- [
- (set_attr "length" "4")]
+ [(set_attr "length" "4")]
)
(define_insn "aarch64_simd_mov_from_<mode>high"
@@ -396,12 +397,11 @@ (define_insn "aarch64_simd_mov_from_<mode>high"
"TARGET_FLOAT"
{@ [ cons: =0 , 1 ; attrs: type , arch ]
[ w , w ; neon_dup<q> , simd ] dup\t%d0, %1.d[1]
+ [ w , w ; * , sve ] ext\t%Z0.b, %Z0.b, %Z0.b, #8
[ ?r , w ; neon_to_gp<q> , simd ] umov\t%0, %1.d[1]
[ ?r , w ; f_mrc , * ] fmov\t%0, %1.d[1]
}
- [
-
- (set_attr "length" "4")]
+ [(set_attr "length" "4")]
)
(define_insn "orn<mode>3<vczle><vczbe>"
@@ -3774,7 +3774,7 @@ static bool
aarch64_array_mode_supported_p (machine_mode mode,
unsigned HOST_WIDE_INT nelems)
{
- if (TARGET_SIMD
+ if (TARGET_BASE_SIMD
&& (AARCH64_VALID_SIMD_QREG_MODE (mode)
|| AARCH64_VALID_SIMD_DREG_MODE (mode))
&& (nelems >= 2 && nelems <= 4))
@@ -13171,8 +13171,8 @@ aarch64_secondary_reload (bool in_p ATTRIBUTE_UNUSED, rtx x,
return NO_REGS;
}
- /* Without the TARGET_SIMD instructions we cannot move a Q register
- to a Q register directly. We need a scratch. */
+ /* Without the TARGET_SIMD or TARGET_SVE instructions we cannot move a
+ Q register to a Q register directly. We need a scratch. */
if (REG_P (x)
&& (mode == TFmode
|| mode == TImode
@@ -15765,7 +15765,7 @@ aarch64_register_move_cost (machine_mode mode,
secondary reload. A general register is used as a scratch to move
the upper DI value and the lower DI value is moved directly,
hence the cost is the sum of three moves. */
- if (! TARGET_SIMD)
+ if (!TARGET_SIMD && !TARGET_SVE)
return regmove_cost->GP2FP + regmove_cost->FP2GP + regmove_cost->FP2FP;
return regmove_cost->FP2FP;
@@ -21374,7 +21374,7 @@ aarch64_simd_container_mode (scalar_mode mode, poly_int64 width)
return aarch64_full_sve_mode (mode).else_mode (word_mode);
gcc_assert (known_eq (width, 64) || known_eq (width, 128));
- if (TARGET_SIMD)
+ if (TARGET_BASE_SIMD)
{
if (known_eq (width, 128))
return aarch64_vq_mode (mode).else_mode (word_mode);
@@ -25764,7 +25764,11 @@ aarch64_expand_cpymem (rtx *operands)
int copy_bits = 256;
/* Default to 256-bit LDP/STP on large copies, however small copies, no SIMD
- support or slow 256-bit LDP/STP fall back to 128-bit chunks. */
+ support or slow 256-bit LDP/STP fall back to 128-bit chunks.
+
+ ??? Although it would be possible to use LDP/STP Qn in streaming mode
+ (so using TARGET_BASE_SIMD instead of TARGET_SIMD), it isn't clear
+ whether that would improve performance. */
if (size <= 24
|| !TARGET_SIMD
|| (aarch64_tune_params.extra_tuning_flags
@@ -61,8 +61,15 @@
#define WORDS_BIG_ENDIAN (BYTES_BIG_ENDIAN)
/* AdvSIMD is supported in the default configuration, unless disabled by
- -mgeneral-regs-only or by the +nosimd extension. */
-#define TARGET_SIMD (AARCH64_ISA_SIMD)
+ -mgeneral-regs-only or by the +nosimd extension. The set of available
+ instructions is then subdivided into:
+
+ - the "base" set, available both in SME streaming mode and in
+ non-streaming mode
+
+ - the full set, available only in non-streaming mode. */
+#define TARGET_BASE_SIMD (AARCH64_ISA_SIMD)
+#define TARGET_SIMD (AARCH64_ISA_SIMD && AARCH64_ISA_SM_OFF)
#define TARGET_FLOAT (AARCH64_ISA_FP)
#define UNITS_PER_WORD 8
@@ -199,6 +206,7 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE = AARCH64_FL_SM_OFF;
/* Macros to test ISA flags. */
+#define AARCH64_ISA_SM_OFF (aarch64_isa_flags & AARCH64_FL_SM_OFF)
#define AARCH64_ISA_MODE (aarch64_isa_flags & AARCH64_FL_ISA_MODES)
#define AARCH64_ISA_CRC (aarch64_isa_flags & AARCH64_FL_CRC)
#define AARCH64_ISA_CRYPTO (aarch64_isa_flags & AARCH64_FL_CRYPTO)
@@ -366,7 +366,8 @@ (define_constants
;; As a convenience, "fp_q" means "fp" + the ability to move between
;; Q registers and is equivalent to "simd".
-(define_enum "arches" [ any rcpc8_4 fp fp_q simd nosimd sve fp16])
+(define_enum "arches" [any rcpc8_4 fp fp_q base_simd nobase_simd
+ simd nosimd sve fp16])
(define_enum_attr "arch" "arches" (const_string "any"))
@@ -394,6 +395,12 @@ (define_attr "arch_enabled" "no,yes"
(and (eq_attr "arch" "fp")
(match_test "TARGET_FLOAT"))
+ (and (eq_attr "arch" "base_simd")
+ (match_test "TARGET_BASE_SIMD"))
+
+ (and (eq_attr "arch" "nobase_simd")
+ (match_test "!TARGET_BASE_SIMD"))
+
(and (eq_attr "arch" "fp_q, simd")
(match_test "TARGET_SIMD"))
@@ -1224,23 +1231,23 @@ (define_insn "*mov<mode>_aarch64"
"(register_operand (operands[0], <MODE>mode)
|| aarch64_reg_or_zero (operands[1], <MODE>mode))"
{@ [cons: =0, 1; attrs: type, arch]
- [w, Z ; neon_move , simd ] movi\t%0.<Vbtype>, #0
- [r, r ; mov_reg , * ] mov\t%w0, %w1
- [r, M ; mov_imm , * ] mov\t%w0, %1
- [w, D<hq>; neon_move , simd ] << aarch64_output_scalar_simd_mov_immediate (operands[1], <MODE>mode);
+ [w, Z ; neon_move , simd ] movi\t%0.<Vbtype>, #0
+ [r, r ; mov_reg , * ] mov\t%w0, %w1
+ [r, M ; mov_imm , * ] mov\t%w0, %1
+ [w, D<hq>; neon_move , simd ] << aarch64_output_scalar_simd_mov_immediate (operands[1], <MODE>mode);
/* The "mov_imm" type for CNT is just a placeholder. */
- [r, Usv ; mov_imm , sve ] << aarch64_output_sve_cnt_immediate ("cnt", "%x0", operands[1]);
- [r, Usr ; mov_imm , sve ] << aarch64_output_sve_rdvl (operands[1]);
- [r, m ; load_4 , * ] ldr<size>\t%w0, %1
- [w, m ; load_4 , * ] ldr\t%<size>0, %1
- [m, r Z ; store_4 , * ] str<size>\\t%w1, %0
- [m, w ; store_4 , * ] str\t%<size>1, %0
- [r, w ; neon_to_gp<q> , simd ] umov\t%w0, %1.<v>[0]
- [r, w ; neon_to_gp<q> , nosimd] fmov\t%w0, %s1
- [w, r Z ; neon_from_gp<q>, simd ] dup\t%0.<Vallxd>, %w1
- [w, r Z ; neon_from_gp<q>, nosimd] fmov\t%s0, %w1
- [w, w ; neon_dup , simd ] dup\t%<Vetype>0, %1.<v>[0]
- [w, w ; neon_dup , nosimd] fmov\t%s0, %s1
+ [r, Usv ; mov_imm , sve ] << aarch64_output_sve_cnt_immediate ("cnt", "%x0", operands[1]);
+ [r, Usr ; mov_imm , sve ] << aarch64_output_sve_rdvl (operands[1]);
+ [r, m ; load_4 , * ] ldr<size>\t%w0, %1
+ [w, m ; load_4 , * ] ldr\t%<size>0, %1
+ [m, r Z ; store_4 , * ] str<size>\\t%w1, %0
+ [m, w ; store_4 , * ] str\t%<size>1, %0
+ [r, w ; neon_to_gp<q> , base_simd ] umov\t%w0, %1.<v>[0]
+ [r, w ; neon_to_gp<q> , nobase_simd] fmov\t%w0, %s1
+ [w, r Z ; neon_from_gp<q>, simd ] dup\t%0.<Vallxd>, %w1
+ [w, r Z ; neon_from_gp<q>, nosimd ] fmov\t%s0, %w1
+ [w, w ; neon_dup , simd ] dup\t%<Vetype>0, %1.<v>[0]
+ [w, w ; neon_dup , nosimd ] fmov\t%s0, %s1
}
)
@@ -1405,9 +1412,9 @@ (define_expand "movti"
(define_insn "*movti_aarch64"
[(set (match_operand:TI 0
- "nonimmediate_operand" "= r,w,w,w, r,w,r,m,m,w,m")
+ "nonimmediate_operand" "= r,w,w,w, r,w,w,r,m,m,w,m")
(match_operand:TI 1
- "aarch64_movti_operand" " rUti,Z,Z,r, w,w,m,r,Z,m,w"))]
+ "aarch64_movti_operand" " rUti,Z,Z,r, w,w,w,m,r,Z,m,w"))]
"(register_operand (operands[0], TImode)
|| aarch64_reg_or_zero (operands[1], TImode))"
"@
@@ -1417,16 +1424,17 @@ (define_insn "*movti_aarch64"
#
#
mov\\t%0.16b, %1.16b
+ mov\\t%Z0.d, %Z1.d
ldp\\t%0, %H0, %1
stp\\t%1, %H1, %0
stp\\txzr, xzr, %0
ldr\\t%q0, %1
str\\t%q1, %0"
- [(set_attr "type" "multiple,neon_move,f_mcr,f_mcr,f_mrc,neon_logic_q, \
+ [(set_attr "type" "multiple,neon_move,f_mcr,f_mcr,f_mrc,neon_logic_q,*,\
load_16,store_16,store_16,\
load_16,store_16")
- (set_attr "length" "8,4,4,8,8,4,4,4,4,4,4")
- (set_attr "arch" "*,simd,*,*,*,simd,*,*,*,fp,fp")]
+ (set_attr "length" "8,4,4,8,8,4,4,4,4,4,4,4")
+ (set_attr "arch" "*,simd,*,*,*,simd,sve,*,*,*,fp,fp")]
)
;; Split a TImode register-register or register-immediate move into
@@ -1553,13 +1561,14 @@ (define_split
(define_insn "*mov<mode>_aarch64"
[(set (match_operand:TFD 0
- "nonimmediate_operand" "=w,?r ,w ,?r,w,?w,w,m,?r,m ,m")
+ "nonimmediate_operand" "=w,w,?r ,w ,?r,w,?w,w,m,?r,m ,m")
(match_operand:TFD 1
- "general_operand" " w,?rY,?r,w ,Y,Y ,m,w,m ,?r,Y"))]
+ "general_operand" " w,w,?rY,?r,w ,Y,Y ,m,w,m ,?r,Y"))]
"TARGET_FLOAT && (register_operand (operands[0], <MODE>mode)
|| aarch64_reg_or_fp_zero (operands[1], <MODE>mode))"
"@
mov\\t%0.16b, %1.16b
+ mov\\t%Z0.d, %Z1.d
#
#
#
@@ -1570,10 +1579,10 @@ (define_insn "*mov<mode>_aarch64"
ldp\\t%0, %H0, %1
stp\\t%1, %H1, %0
stp\\txzr, xzr, %0"
- [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
+ [(set_attr "type" "logic_reg,*,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
f_loadd,f_stored,load_16,store_16,store_16")
- (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
- (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
+ (set_attr "length" "4,4,8,8,8,4,4,4,4,4,4,4")
+ (set_attr "arch" "simd,sve,*,*,*,simd,*,*,*,*,*,*")]
)
(define_split
@@ -1767,7 +1776,7 @@ (define_insn "load_pair_dw_<TX:mode><TX2:mode>"
(match_operand:TX 1 "aarch64_mem_pair_operand" "Ump"))
(set (match_operand:TX2 2 "register_operand" "=w")
(match_operand:TX2 3 "memory_operand" "m"))]
- "TARGET_SIMD
+ "TARGET_BASE_SIMD
&& rtx_equal_p (XEXP (operands[3], 0),
plus_constant (Pmode,
XEXP (operands[1], 0),
@@ -1815,11 +1824,11 @@ (define_insn "store_pair_dw_<TX:mode><TX2:mode>"
(match_operand:TX 1 "register_operand" "w"))
(set (match_operand:TX2 2 "memory_operand" "=m")
(match_operand:TX2 3 "register_operand" "w"))]
- "TARGET_SIMD &&
- rtx_equal_p (XEXP (operands[2], 0),
- plus_constant (Pmode,
- XEXP (operands[0], 0),
- GET_MODE_SIZE (TFmode)))"
+ "TARGET_BASE_SIMD
+ && rtx_equal_p (XEXP (operands[2], 0),
+ plus_constant (Pmode,
+ XEXP (operands[0], 0),
+ GET_MODE_SIZE (TFmode)))"
"stp\\t%q1, %q3, %z0"
[(set_attr "type" "neon_stp_q")
(set_attr "fp" "yes")]
@@ -1867,7 +1876,7 @@ (define_insn "loadwb_pair<TX:mode>_<P:mode>"
(set (match_operand:TX 3 "register_operand" "=w")
(mem:TX (plus:P (match_dup 1)
(match_operand:P 5 "const_int_operand" "n"))))])]
- "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
+ "TARGET_BASE_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)"
"ldp\\t%q2, %q3, [%1], %4"
[(set_attr "type" "neon_ldp_q")]
)
@@ -1917,7 +1926,7 @@ (define_insn "storewb_pair<TX:mode>_<P:mode>"
(set (mem:TX (plus:P (match_dup 0)
(match_operand:P 5 "const_int_operand" "n")))
(match_operand:TX 3 "register_operand" "w"))])]
- "TARGET_SIMD
+ "TARGET_BASE_SIMD
&& INTVAL (operands[5])
== INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)"
"stp\\t%q2, %q3, [%0, %4]!"
new file mode 100644
@@ -0,0 +1,51 @@
+/* { dg-do assemble } */
+/* { dg-options "-O --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/*
+** fpr_to_fpr:
+** fmov d0, d1
+** ret
+*/
+double
+fpr_to_fpr (double q0, double q1) [[arm::streaming_compatible]]
+{
+ return q1;
+}
+
+/*
+** gpr_to_fpr:
+** fmov d0, x0
+** ret
+*/
+double
+gpr_to_fpr () [[arm::streaming_compatible]]
+{
+ register double x0 asm ("x0");
+ asm volatile ("" : "=r" (x0));
+ return x0;
+}
+
+/*
+** zero_to_fpr:
+** fmov d0, xzr
+** ret
+*/
+double
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ return 0;
+}
+
+/*
+** fpr_to_gpr:
+** fmov x0, d0
+** ret
+*/
+void
+fpr_to_gpr (double q0) [[arm::streaming_compatible]]
+{
+ register double x0 asm ("x0");
+ x0 = q0;
+ asm volatile ("" :: "r" (x0));
+}
new file mode 100644
@@ -0,0 +1,59 @@
+/* { dg-do assemble } */
+/* { dg-options "-O --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <stdint.h>
+
+/*
+** fpr_to_fpr:
+** fmov d0, d1
+** ret
+*/
+void
+fpr_to_fpr (void) [[arm::streaming_compatible]]
+{
+ register uint64_t q0 asm ("q0");
+ register uint64_t q1 asm ("q1");
+ asm volatile ("" : "=w" (q1));
+ q0 = q1;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** gpr_to_fpr:
+** fmov d0, x0
+** ret
+*/
+void
+gpr_to_fpr (uint64_t x0) [[arm::streaming_compatible]]
+{
+ register uint64_t q0 asm ("q0");
+ q0 = x0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** zero_to_fpr:
+** fmov d0, xzr
+** ret
+*/
+void
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ register uint64_t q0 asm ("q0");
+ q0 = 0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** fpr_to_gpr:
+** fmov x0, d0
+** ret
+*/
+uint64_t
+fpr_to_gpr () [[arm::streaming_compatible]]
+{
+ register uint64_t q0 asm ("q0");
+ asm volatile ("" : "=w" (q0));
+ return q0;
+}
new file mode 100644
@@ -0,0 +1,53 @@
+/* { dg-do assemble } */
+/* { dg-options "-O --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#pragma GCC target "+nothing+simd"
+
+/*
+** fpr_to_fpr:
+** fmov s0, s1
+** ret
+*/
+_Float16
+fpr_to_fpr (_Float16 q0, _Float16 q1) [[arm::streaming_compatible]]
+{
+ return q1;
+}
+
+/*
+** gpr_to_fpr:
+** fmov s0, w0
+** ret
+*/
+_Float16
+gpr_to_fpr () [[arm::streaming_compatible]]
+{
+ register _Float16 w0 asm ("w0");
+ asm volatile ("" : "=r" (w0));
+ return w0;
+}
+
+/*
+** zero_to_fpr:
+** fmov s0, wzr
+** ret
+*/
+_Float16
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ return 0;
+}
+
+/*
+** fpr_to_gpr:
+** fmov w0, s0
+** ret
+*/
+void
+fpr_to_gpr (_Float16 q0) [[arm::streaming_compatible]]
+{
+ register _Float16 w0 asm ("w0");
+ w0 = q0;
+ asm volatile ("" :: "r" (w0));
+}
new file mode 100644
@@ -0,0 +1,61 @@
+/* { dg-do assemble } */
+/* { dg-options "-O --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#pragma GCC target "+nothing+simd"
+
+#include <stdint.h>
+
+/*
+** fpr_to_fpr:
+** fmov s0, s1
+** ret
+*/
+void
+fpr_to_fpr (void) [[arm::streaming_compatible]]
+{
+ register uint16_t q0 asm ("q0");
+ register uint16_t q1 asm ("q1");
+ asm volatile ("" : "=w" (q1));
+ q0 = q1;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** gpr_to_fpr:
+** fmov s0, w0
+** ret
+*/
+void
+gpr_to_fpr (uint16_t w0) [[arm::streaming_compatible]]
+{
+ register uint16_t q0 asm ("q0");
+ q0 = w0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** zero_to_fpr:
+** fmov s0, wzr
+** ret
+*/
+void
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ register uint16_t q0 asm ("q0");
+ q0 = 0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** fpr_to_gpr:
+** umov w0, v0.h\[0\]
+** ret
+*/
+uint16_t
+fpr_to_gpr () [[arm::streaming_compatible]]
+{
+ register uint16_t q0 asm ("q0");
+ asm volatile ("" : "=w" (q0));
+ return q0;
+}
new file mode 100644
@@ -0,0 +1,59 @@
+/* { dg-do assemble } */
+/* { dg-options "-O --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <stdint.h>
+
+/*
+** fpr_to_fpr:
+** fmov s0, s1
+** ret
+*/
+void
+fpr_to_fpr (void) [[arm::streaming_compatible]]
+{
+ register uint8_t q0 asm ("q0");
+ register uint8_t q1 asm ("q1");
+ asm volatile ("" : "=w" (q1));
+ q0 = q1;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** gpr_to_fpr:
+** fmov s0, w0
+** ret
+*/
+void
+gpr_to_fpr (uint8_t w0) [[arm::streaming_compatible]]
+{
+ register uint8_t q0 asm ("q0");
+ q0 = w0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** zero_to_fpr:
+** fmov s0, wzr
+** ret
+*/
+void
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ register uint8_t q0 asm ("q0");
+ q0 = 0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** fpr_to_gpr:
+** umov w0, v0.b\[0\]
+** ret
+*/
+uint8_t
+fpr_to_gpr () [[arm::streaming_compatible]]
+{
+ register uint8_t q0 asm ("q0");
+ asm volatile ("" : "=w" (q0));
+ return q0;
+}
new file mode 100644
@@ -0,0 +1,51 @@
+/* { dg-do assemble } */
+/* { dg-options "-O --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+/*
+** fpr_to_fpr:
+** fmov s0, s1
+** ret
+*/
+float
+fpr_to_fpr (float q0, float q1) [[arm::streaming_compatible]]
+{
+ return q1;
+}
+
+/*
+** gpr_to_fpr:
+** fmov s0, w0
+** ret
+*/
+float
+gpr_to_fpr () [[arm::streaming_compatible]]
+{
+ register float w0 asm ("w0");
+ asm volatile ("" : "=r" (w0));
+ return w0;
+}
+
+/*
+** zero_to_fpr:
+** fmov s0, wzr
+** ret
+*/
+float
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ return 0;
+}
+
+/*
+** fpr_to_gpr:
+** fmov w0, s0
+** ret
+*/
+void
+fpr_to_gpr (float q0) [[arm::streaming_compatible]]
+{
+ register float w0 asm ("w0");
+ w0 = q0;
+ asm volatile ("" :: "r" (w0));
+}
new file mode 100644
@@ -0,0 +1,59 @@
+/* { dg-do assemble } */
+/* { dg-options "-O --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#include <stdint.h>
+
+/*
+** fpr_to_fpr:
+** fmov s0, s1
+** ret
+*/
+void
+fpr_to_fpr (void) [[arm::streaming_compatible]]
+{
+ register uint32_t q0 asm ("q0");
+ register uint32_t q1 asm ("q1");
+ asm volatile ("" : "=w" (q1));
+ q0 = q1;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** gpr_to_fpr:
+** fmov s0, w0
+** ret
+*/
+void
+gpr_to_fpr (uint32_t w0) [[arm::streaming_compatible]]
+{
+ register uint32_t q0 asm ("q0");
+ q0 = w0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** zero_to_fpr:
+** fmov s0, wzr
+** ret
+*/
+void
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ register uint32_t q0 asm ("q0");
+ q0 = 0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** fpr_to_gpr:
+** fmov w0, s0
+** ret
+*/
+uint32_t
+fpr_to_gpr () [[arm::streaming_compatible]]
+{
+ register uint32_t q0 asm ("q0");
+ asm volatile ("" : "=w" (q0));
+ return q0;
+}
new file mode 100644
@@ -0,0 +1,81 @@
+/* { dg-do assemble } */
+/* { dg-require-effective-target large_long_double } */
+/* { dg-options "-O -mtune=neoverse-v1 --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#pragma GCC target "+nosve"
+
+/*
+** fpr_to_fpr:
+** sub sp, sp, #16
+** str q1, \[sp\]
+** ldr q0, \[sp\]
+** add sp, sp, #?16
+** ret
+*/
+long double
+fpr_to_fpr (long double q0, long double q1) [[arm::streaming_compatible]]
+{
+ return q1;
+}
+
+/*
+** gpr_to_fpr: { target aarch64_little_endian }
+** fmov d0, x0
+** fmov v0.d\[1\], x1
+** ret
+*/
+/*
+** gpr_to_fpr: { target aarch64_big_endian }
+** fmov d0, x1
+** fmov v0.d\[1\], x0
+** ret
+*/
+long double
+gpr_to_fpr () [[arm::streaming_compatible]]
+{
+ register long double x0 asm ("x0");
+ asm volatile ("" : "=r" (x0));
+ return x0;
+}
+
+/*
+** zero_to_fpr:
+** fmov s0, wzr
+** ret
+*/
+long double
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ return 0;
+}
+
+/*
+** fpr_to_gpr: { target aarch64_little_endian }
+** (
+** fmov x0, d0
+** fmov x1, v0.d\[1\]
+** |
+** fmov x1, v0.d\[1\]
+** fmov x0, d0
+** )
+** ret
+*/
+/*
+** fpr_to_gpr: { target aarch64_big_endian }
+** (
+** fmov x1, d0
+** fmov x0, v0.d\[1\]
+** |
+** fmov x0, v0.d\[1\]
+** fmov x1, d0
+** )
+** ret
+*/
+void
+fpr_to_gpr (long double q0) [[arm::streaming_compatible]]
+{
+ register long double x0 asm ("x0");
+ x0 = q0;
+ asm volatile ("" :: "r" (x0));
+}
new file mode 100644
@@ -0,0 +1,78 @@
+/* { dg-do assemble } */
+/* { dg-require-effective-target large_long_double } */
+/* { dg-options "-O -mtune=neoverse-v1 --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#pragma GCC target "+sve"
+
+/*
+** fpr_to_fpr:
+** mov z0.d, z1.d
+** ret
+*/
+long double
+fpr_to_fpr (long double q0, long double q1) [[arm::streaming_compatible]]
+{
+ return q1;
+}
+
+/*
+** gpr_to_fpr: { target aarch64_little_endian }
+** fmov d0, x0
+** fmov v0.d\[1\], x1
+** ret
+*/
+/*
+** gpr_to_fpr: { target aarch64_big_endian }
+** fmov d0, x1
+** fmov v0.d\[1\], x0
+** ret
+*/
+long double
+gpr_to_fpr () [[arm::streaming_compatible]]
+{
+ register long double x0 asm ("x0");
+ asm volatile ("" : "=r" (x0));
+ return x0;
+}
+
+/*
+** zero_to_fpr:
+** fmov s0, wzr
+** ret
+*/
+long double
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ return 0;
+}
+
+/*
+** fpr_to_gpr: { target aarch64_little_endian }
+** (
+** fmov x0, d0
+** fmov x1, v0.d\[1\]
+** |
+** fmov x1, v0.d\[1\]
+** fmov x0, d0
+** )
+** ret
+*/
+/*
+** fpr_to_gpr: { target aarch64_big_endian }
+** (
+** fmov x1, d0
+** fmov x0, v0.d\[1\]
+** |
+** fmov x0, v0.d\[1\]
+** fmov x1, d0
+** )
+** ret
+*/
+void
+fpr_to_gpr (long double q0) [[arm::streaming_compatible]]
+{
+ register long double x0 asm ("x0");
+ x0 = q0;
+ asm volatile ("" :: "r" (x0));
+}
new file mode 100644
@@ -0,0 +1,86 @@
+/* { dg-do assemble } */
+/* { dg-options "-O -mtune=neoverse-v1 --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#pragma GCC target "+nosve"
+
+/*
+** fpr_to_fpr:
+** sub sp, sp, #16
+** str q1, \[sp\]
+** ldr q0, \[sp\]
+** add sp, sp, #?16
+** ret
+*/
+void
+fpr_to_fpr (void) [[arm::streaming_compatible]]
+{
+ register __int128_t q0 asm ("q0");
+ register __int128_t q1 asm ("q1");
+ asm volatile ("" : "=w" (q1));
+ q0 = q1;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** gpr_to_fpr: { target aarch64_little_endian }
+** fmov d0, x0
+** fmov v0.d\[1\], x1
+** ret
+*/
+/*
+** gpr_to_fpr: { target aarch64_big_endian }
+** fmov d0, x1
+** fmov v0.d\[1\], x0
+** ret
+*/
+void
+gpr_to_fpr (__int128_t x0) [[arm::streaming_compatible]]
+{
+ register __int128_t q0 asm ("q0");
+ q0 = x0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** zero_to_fpr:
+** fmov d0, xzr
+** ret
+*/
+void
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ register __int128_t q0 asm ("q0");
+ q0 = 0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** fpr_to_gpr: { target aarch64_little_endian }
+** (
+** fmov x0, d0
+** fmov x1, v0.d\[1\]
+** |
+** fmov x1, v0.d\[1\]
+** fmov x0, d0
+** )
+** ret
+*/
+/*
+** fpr_to_gpr: { target aarch64_big_endian }
+** (
+** fmov x1, d0
+** fmov x0, v0.d\[1\]
+** |
+** fmov x0, v0.d\[1\]
+** fmov x1, d0
+** )
+** ret
+*/
+__int128_t
+fpr_to_gpr () [[arm::streaming_compatible]]
+{
+ register __int128_t q0 asm ("q0");
+ asm volatile ("" : "=w" (q0));
+ return q0;
+}
new file mode 100644
@@ -0,0 +1,83 @@
+/* { dg-do assemble } */
+/* { dg-options "-O -mtune=neoverse-v1 --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#pragma GCC target "+sve"
+
+/*
+** fpr_to_fpr:
+** mov z0\.d, z1\.d
+** ret
+*/
+void
+fpr_to_fpr (void) [[arm::streaming_compatible]]
+{
+ register __int128_t q0 asm ("q0");
+ register __int128_t q1 asm ("q1");
+ asm volatile ("" : "=w" (q1));
+ q0 = q1;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** gpr_to_fpr: { target aarch64_little_endian }
+** fmov d0, x0
+** fmov v0.d\[1\], x1
+** ret
+*/
+/*
+** gpr_to_fpr: { target aarch64_big_endian }
+** fmov d0, x1
+** fmov v0.d\[1\], x0
+** ret
+*/
+void
+gpr_to_fpr (__int128_t x0) [[arm::streaming_compatible]]
+{
+ register __int128_t q0 asm ("q0");
+ q0 = x0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** zero_to_fpr:
+** fmov d0, xzr
+** ret
+*/
+void
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ register __int128_t q0 asm ("q0");
+ q0 = 0;
+ asm volatile ("" :: "w" (q0));
+}
+
+/*
+** fpr_to_gpr: { target aarch64_little_endian }
+** (
+** fmov x0, d0
+** fmov x1, v0.d\[1\]
+** |
+** fmov x1, v0.d\[1\]
+** fmov x0, d0
+** )
+** ret
+*/
+/*
+** fpr_to_gpr: { target aarch64_big_endian }
+** (
+** fmov x1, d0
+** fmov x0, v0.d\[1\]
+** |
+** fmov x0, v0.d\[1\]
+** fmov x1, d0
+** )
+** ret
+*/
+__int128_t
+fpr_to_gpr () [[arm::streaming_compatible]]
+{
+ register __int128_t q0 asm ("q0");
+ asm volatile ("" : "=w" (q0));
+ return q0;
+}
new file mode 100644
@@ -0,0 +1,82 @@
+/* { dg-do assemble } */
+/* { dg-options "-O -mtune=neoverse-v1 --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#pragma GCC target "+nosve"
+
+typedef unsigned char v16qi __attribute__((vector_size(16)));
+
+/*
+** fpr_to_fpr:
+** sub sp, sp, #16
+** str q1, \[sp\]
+** ldr q0, \[sp\]
+** add sp, sp, #?16
+** ret
+*/
+v16qi
+fpr_to_fpr (v16qi q0, v16qi q1) [[arm::streaming_compatible]]
+{
+ return q1;
+}
+
+/*
+** gpr_to_fpr: { target aarch64_little_endian }
+** fmov d0, x0
+** fmov v0.d\[1\], x1
+** ret
+*/
+/*
+** gpr_to_fpr: { target aarch64_big_endian }
+** fmov d0, x1
+** fmov v0.d\[1\], x0
+** ret
+*/
+v16qi
+gpr_to_fpr () [[arm::streaming_compatible]]
+{
+ register v16qi x0 asm ("x0");
+ asm volatile ("" : "=r" (x0));
+ return x0;
+}
+
+/*
+** zero_to_fpr:
+** fmov d0, xzr
+** ret
+*/
+v16qi
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ return (v16qi) {};
+}
+
+/*
+** fpr_to_gpr: { target aarch64_little_endian }
+** (
+** umov x0, v0.d\[0\]
+** fmov x1, v0.d\[1\]
+** |
+** fmov x1, v0.d\[1\]
+** umov x0, v0.d\[0\]
+** )
+** ret
+*/
+/*
+** fpr_to_gpr: { target aarch64_big_endian }
+** (
+** umov x1, v0.d\[0\]
+** fmov x0, v0.d\[1\]
+** |
+** fmov x0, v0.d\[1\]
+** umov x1, v0.d\[0\]
+** )
+** ret
+*/
+void
+fpr_to_gpr (v16qi q0) [[arm::streaming_compatible]]
+{
+ register v16qi x0 asm ("x0");
+ x0 = q0;
+ asm volatile ("" :: "r" (x0));
+}
new file mode 100644
@@ -0,0 +1,79 @@
+/* { dg-do assemble } */
+/* { dg-options "-O -mtune=neoverse-v1 --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#pragma GCC target "+sve"
+
+typedef unsigned char v16qi __attribute__((vector_size(16)));
+
+/*
+** fpr_to_fpr:
+** mov z0.d, z1.d
+** ret
+*/
+v16qi
+fpr_to_fpr (v16qi q0, v16qi q1) [[arm::streaming_compatible]]
+{
+ return q1;
+}
+
+/*
+** gpr_to_fpr: { target aarch64_little_endian }
+** fmov d0, x0
+** fmov v0.d\[1\], x1
+** ret
+*/
+/*
+** gpr_to_fpr: { target aarch64_big_endian }
+** fmov d0, x1
+** fmov v0.d\[1\], x0
+** ret
+*/
+v16qi
+gpr_to_fpr () [[arm::streaming_compatible]]
+{
+ register v16qi x0 asm ("x0");
+ asm volatile ("" : "=r" (x0));
+ return x0;
+}
+
+/*
+** zero_to_fpr:
+** fmov d0, xzr
+** ret
+*/
+v16qi
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ return (v16qi) {};
+}
+
+/*
+** fpr_to_gpr: { target aarch64_little_endian }
+** (
+** umov x0, v0.d\[0\]
+** fmov x1, v0.d\[1\]
+** |
+** fmov x1, v0.d\[1\]
+** umov x0, v0.d\[0\]
+** )
+** ret
+*/
+/*
+** fpr_to_gpr: { target aarch64_big_endian }
+** (
+** umov x1, v0.d\[0\]
+** fmov x0, v0.d\[1\]
+** |
+** fmov x0, v0.d\[1\]
+** umov x1, v0.d\[0\]
+** )
+** ret
+*/
+void
+fpr_to_gpr (v16qi q0) [[arm::streaming_compatible]]
+{
+ register v16qi x0 asm ("x0");
+ x0 = q0;
+ asm volatile ("" :: "r" (x0));
+}
new file mode 100644
@@ -0,0 +1,55 @@
+/* { dg-do assemble } */
+/* { dg-options "-O -mtune=neoverse-v1 --save-temps" } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+
+#pragma GCC target "+nosve"
+
+typedef unsigned char v8qi __attribute__((vector_size(8)));
+
+/*
+** fpr_to_fpr:
+** fmov d0, d1
+** ret
+*/
+v8qi
+fpr_to_fpr (v8qi q0, v8qi q1) [[arm::streaming_compatible]]
+{
+ return q1;
+}
+
+/*
+** gpr_to_fpr:
+** fmov d0, x0
+** ret
+*/
+v8qi
+gpr_to_fpr () [[arm::streaming_compatible]]
+{
+ register v8qi x0 asm ("x0");
+ asm volatile ("" : "=r" (x0));
+ return x0;
+}
+
+/*
+** zero_to_fpr:
+** fmov d0, xzr
+** ret
+*/
+v8qi
+zero_to_fpr () [[arm::streaming_compatible]]
+{
+ return (v8qi) {};
+}
+
+/*
+** fpr_to_gpr:
+** umov x0, v0\.d\[0\]
+** ret
+*/
+void
+fpr_to_gpr (v8qi q0) [[arm::streaming_compatible]]
+{
+ register v8qi x0 asm ("x0");
+ x0 = q0;
+ asm volatile ("" :: "r" (x0));
+}
new file mode 100644
@@ -0,0 +1,13 @@
+// { dg-options "" }
+
+#include <arm_neon.h>
+
+#pragma GCC target "+nosme"
+
+// { dg-error {inlining failed.*'vhaddq_s32'} "" { target *-*-* } 0 }
+
+int32x4_t
+foo (int32x4_t x, int32x4_t y) [[arm::streaming_compatible]]
+{
+ return vhaddq_s32 (x, y);
+}
new file mode 100644
@@ -0,0 +1,11 @@
+// { dg-options "" }
+
+#include <arm_neon.h>
+
+// { dg-error {inlining failed.*'vhaddq_s32'} "" { target *-*-* } 0 }
+
+int32x4_t
+foo (int32x4_t x, int32x4_t y) [[arm::streaming_compatible]]
+{
+ return vhaddq_s32 (x, y);
+}
new file mode 100644
@@ -0,0 +1,11 @@
+// { dg-options "" }
+
+#include <arm_neon.h>
+
+// { dg-error {inlining failed.*'vhaddq_s32'} "" { target *-*-* } 0 }
+
+int32x4_t
+foo (int32x4_t x, int32x4_t y) [[arm::streaming]]
+{
+ return vhaddq_s32 (x, y);
+}