@@ -138,10 +138,6 @@ aarch64_handle_option (struct gcc_options *opts,
opts->x_flag_omit_leaf_frame_pointer = val;
return true;
- case OPT_moutline_atomics:
- opts->x_aarch64_flag_outline_atomics = val;
- return true;
-
case OPT_mmax_vectorization:
opts->x_flag_aarch64_max_vectorization = val;
return true;
@@ -634,6 +634,13 @@
#endif
+/* Define if your assembler can enable the LSE extension for a single
+ instruction. */
+#ifndef USED_FOR_TARGET
+#undef HAVE_AS_LSE
+#endif
+
+
/* Define if your assembler supports ltoffx and ldxmov relocations. */
#ifndef USED_FOR_TARGET
#undef HAVE_AS_LTOFFX_LDXMOV_RELOCS
@@ -147,4 +147,19 @@ enum aarch64_early_ra_scope {
AARCH64_EARLY_RA_NONE
};
+/* How to implement an atomic operation when LSE is not known to be
+ available at compile time:
+ - AARCH64_OUTLINE_ATOMICS_NONE: do not use the libgcc helpers, expand
+ an LDXR/STXR sequence inline.
+ - AARCH64_OUTLINE_ATOMICS_CALL: call the libgcc helper, which tests for
+ LSE and selects between an LSE instruction and an LDXR/STXR sequence.
+ - AARCH64_OUTLINE_ATOMICS_INLINE: test for LSE inline and use an LSE
+ instruction directly, expanding an LDXR/STXR sequence on the unlikely
+ path where LSE is unavailable. */
+enum aarch64_outline_atomics {
+ AARCH64_OUTLINE_ATOMICS_NONE,
+ AARCH64_OUTLINE_ATOMICS_CALL,
+ AARCH64_OUTLINE_ATOMICS_INLINE
+};
+
#endif
@@ -1288,6 +1288,24 @@ extern const atomic_ool_names aarch64_ool_ldset_names;
extern const atomic_ool_names aarch64_ool_ldclr_names;
extern const atomic_ool_names aarch64_ool_ldeor_names;
+const char *aarch64_output_lse_insn (const char *, rtx *);
+
+/* Emit the code that -moutline-atomics=inline puts around an atomic
+ operation. The caller emits the LSE form after start and the inline
+ LDXR/STXR fallback after mid. */
+
+class aarch64_lse_dispatch
+{
+public:
+ void start ();
+ void mid ();
+ void end ();
+
+private:
+ rtx_code_label *m_fallback;
+ rtx_code_label *m_done;
+};
+
tree aarch64_resolve_overloaded_builtin_general (location_t, tree, void *);
const char *aarch64_sls_barrier (int);
@@ -20257,6 +20257,18 @@ aarch64_override_options_internal (struct gcc_options *opts,
aarch64_autovec_preference,
opts->x_autovec_preference);
+#ifndef HAVE_AS_LSE
+ /* Emitting an LSE instruction under a run-time test needs an assembler
+ that can enable the extension for that one instruction. */
+ if (opts->x_aarch64_flag_outline_atomics == AARCH64_OUTLINE_ATOMICS_INLINE
+ && !(isa_flags & AARCH64_FL_LSE))
+ {
+ error ("%<-moutline-atomics=inline%> requires assembler support for "
+ "%<.arch_extension lse%> and %<.arch_extension nolse%>");
+ opts->x_aarch64_flag_outline_atomics = AARCH64_OUTLINE_ATOMICS_CALL;
+ }
+#endif
+
aarch64_override_options_after_change_1 (opts);
}
@@ -20777,6 +20789,7 @@ enum aarch64_attr_opt_type
aarch64_attr_mask, /* Attribute should set a bit in target_flags. */
aarch64_attr_bool, /* Attribute sets or unsets a boolean variable. */
aarch64_attr_enum, /* Attribute sets an enum variable. */
+ aarch64_attr_enum_or_bool, /* Attribute accepts an enum or boolean. */
aarch64_attr_custom /* Attribute requires a custom handling function. */
};
@@ -20789,8 +20802,8 @@ enum aarch64_attr_opt_type
It is needed only when the ATTR_TYPE is aarch64_attr_custom.
OPT_NUM is the enum specifying the option that the attribute modifies.
This is needed for attributes that mirror the behavior of a command-line
- option, that is it has ATTR_TYPE aarch64_attr_mask, aarch64_attr_bool or
- aarch64_attr_enum. */
+ option, that is it has ATTR_TYPE aarch64_attr_mask, aarch64_attr_bool,
+ aarch64_attr_enum or aarch64_attr_enum_or_bool. */
struct aarch64_attribute_info
{
@@ -20998,8 +21011,8 @@ static const struct aarch64_attribute_info aarch64_attributes[] =
aarch64_handle_attr_branch_protection, OPT_mbranch_protection_ },
{ "sign-return-address", aarch64_attr_enum, false, NULL,
OPT_msign_return_address_ },
- { "outline-atomics", aarch64_attr_bool, true, NULL,
- OPT_moutline_atomics},
+ { "outline-atomics", aarch64_attr_enum_or_bool, true, NULL,
+ OPT_moutline_atomics_ },
{ "max-vectorization", aarch64_attr_bool, false, NULL,
OPT_mmax_vectorization},
{ NULL, aarch64_attr_custom, false, NULL, OPT____ }
@@ -21060,8 +21073,10 @@ aarch64_process_one_target_attr (char *arg_str)
found = true;
bool attr_need_arg_p = p_attr->attr_type == aarch64_attr_custom
|| p_attr->attr_type == aarch64_attr_enum;
+ bool attr_optional_arg_p
+ = p_attr->attr_type == aarch64_attr_enum_or_bool;
- if (attr_need_arg_p ^ (arg != NULL))
+ if (!attr_optional_arg_p && (attr_need_arg_p ^ (arg != NULL)))
{
error ("pragma or attribute %<target(\"%s\")%> does not accept an argument", str_to_check);
return false;
@@ -21111,12 +21126,18 @@ aarch64_process_one_target_attr (char *arg_str)
}
/* Use the option setting machinery to set an option to an enum. */
case aarch64_attr_enum:
+ case aarch64_attr_enum_or_bool:
{
- gcc_assert (arg);
- bool valid;
+ const char *enum_arg = arg;
+ if (!enum_arg)
+ {
+ gcc_assert (p_attr->attr_type == aarch64_attr_enum_or_bool);
+ enum_arg = invert ? "none" : "call";
+ }
int value;
- valid = opt_enum_arg_to_value (p_attr->opt_num, arg,
- &value, CL_TARGET);
+ bool valid = !(invert && arg)
+ && opt_enum_arg_to_value (p_attr->opt_num, enum_arg,
+ &value, CL_TARGET);
if (valid)
{
set_option (&global_options, NULL, p_attr->opt_num, value,
@@ -21126,7 +21147,8 @@ aarch64_process_one_target_attr (char *arg_str)
}
else
{
- error ("pragma or attribute %<target(\"%s=%s\")%> is not valid", str_to_check, arg);
+ error ("invalid %<target%> pragma or attribute argument %qs",
+ arg_str);
}
break;
}
@@ -27236,12 +27258,86 @@ const atomic_ool_names aarch64_ool_ldeor_names = { { DEF4(ldeor) } };
#undef DEF4
#undef DEF5
+/* Return TEMPL for the LSE instruction with OPERANDS if the current function
+ enables LSE. Otherwise emit the instruction between assembler directives
+ that enable and disable LSE, and return an empty template. */
+
+const char *
+aarch64_output_lse_insn (const char *templ, rtx *operands)
+{
+ if (TARGET_LSE)
+ return templ;
+
+ output_asm_insn (".arch_extension lse", operands);
+ output_asm_insn (templ, operands);
+ output_asm_insn (".arch_extension nolse", operands);
+ return "";
+}
+
+/* Return a register that is nonzero when the LSE instructions are available
+ at run time. The value is loaded from a location that does not change
+ after startup initialization, so the test can be hoisted out of loops. */
+
+static rtx
+aarch64_load_lse_available (void)
+{
+ rtx sym = gen_rtx_SYMBOL_REF (Pmode, "__aarch64_have_lse_atomics");
+ SYMBOL_REF_FLAGS (sym) = SYMBOL_FLAG_LOCAL;
+
+ rtx mem = gen_const_mem (QImode, force_reg (Pmode, sym));
+ rtx res = gen_reg_rtx (SImode);
+ emit_insn (gen_zero_extendqisi2 (res, mem));
+ return res;
+}
+
+/* Begin an inline LSE dispatch. Test whether LSE is available and branch
+ to the inline LDXR/STXR fallback when it is not. The caller emits the
+ LSE form of the operation next. */
+
+void
+aarch64_lse_dispatch::start ()
+{
+ gcc_checking_assert (TARGET_OUTLINE_ATOMICS_INLINE && !TARGET_LSE);
+ m_fallback = gen_label_rtx ();
+ m_done = gen_label_rtx ();
+ rtx avail = aarch64_load_lse_available ();
+ emit_unlikely_jump_insn (aarch64_gen_compare_zero_and_branch (EQ, avail,
+ m_fallback));
+}
+
+/* Step over the fallback, which is reached when the run-time flag is clear,
+ and start it. The caller emits the fallback next and then calls end. */
+
+void
+aarch64_lse_dispatch::mid ()
+{
+ emit_jump (m_done);
+ emit_label (m_fallback);
+}
+
+/* Emit the label that joins the fallback with the LSE path. */
+
+void
+aarch64_lse_dispatch::end ()
+{
+ emit_label (m_done);
+}
+
+/* Set BVAL to true when CC_REG indicates equality, and false otherwise. */
+
+static void
+aarch64_emit_compare_and_swap_result (rtx bval, rtx cc_reg)
+{
+ rtx x = gen_rtx_EQ (SImode, cc_reg, const0_rtx);
+ emit_insn (gen_rtx_SET (bval, x));
+}
+
/* Expand a compare and swap pattern. */
void
aarch64_expand_compare_and_swap (rtx operands[])
{
- rtx bval, rval, mem, oldval, newval, is_weak, mod_s, mod_f, x, cc_reg;
+ rtx bval, rval, mem, oldval, newval, is_weak, mod_s, mod_f;
machine_mode mode, r_mode;
bval = operands[0];
@@ -27268,8 +27364,15 @@ aarch64_expand_compare_and_swap (rtx operands[])
rval = gen_reg_rtx (r_mode);
}
- if (TARGET_LSE)
+ if (TARGET_LSE_INSNS)
{
+ aarch64_lse_dispatch dispatch;
+
+ /* Oldval must satisfy the compare afterward, and is also the first
+ input of the inline fallback. */
+ if (!TARGET_LSE && !aarch64_plus_operand (oldval, mode))
+ oldval = force_reg (mode, oldval);
+
/* The CAS insn requires oldval and rval overlap, but we need to
have a copy of oldval saved across the operation to tell if
the operation is successful. */
@@ -27280,9 +27383,29 @@ aarch64_expand_compare_and_swap (rtx operands[])
if (mode == TImode)
newval = force_reg (mode, newval);
+ if (!TARGET_LSE)
+ dispatch.start ();
+
emit_insn (gen_aarch64_compare_and_swap_lse (mode, rval, mem,
newval, mod_s));
- cc_reg = aarch64_gen_compare_reg_maybe_ze (NE, rval, oldval, mode);
+
+ rtx cc_reg
+ = aarch64_gen_compare_reg_maybe_ze (NE, rval, oldval, mode);
+ aarch64_emit_compare_and_swap_result (bval, cc_reg);
+
+ if (!TARGET_LSE)
+ {
+ dispatch.mid ();
+ insn_code code = code_for_aarch64_compare_and_swap (mode);
+ rtx cmp_oldval = oldval;
+ if (!insn_data[code].operand[2].predicate (cmp_oldval, mode))
+ cmp_oldval = force_reg (mode, cmp_oldval);
+ emit_insn (GEN_FCN (code) (rval, mem, cmp_oldval, newval,
+ is_weak, mod_s, mod_f));
+ cc_reg = gen_rtx_REG (CCmode, CC_REGNUM);
+ aarch64_emit_compare_and_swap_result (bval, cc_reg);
+ dispatch.end ();
+ }
}
else if (TARGET_OUTLINE_ATOMICS)
{
@@ -27293,7 +27416,9 @@ aarch64_expand_compare_and_swap (rtx operands[])
rval = emit_library_call_value (func, NULL_RTX, LCT_NORMAL, r_mode,
oldval, mode, newval, mode,
XEXP (mem, 0), Pmode);
- cc_reg = aarch64_gen_compare_reg_maybe_ze (NE, rval, oldval, mode);
+ rtx cc_reg
+ = aarch64_gen_compare_reg_maybe_ze (NE, rval, oldval, mode);
+ aarch64_emit_compare_and_swap_result (bval, cc_reg);
}
else
{
@@ -27304,15 +27429,13 @@ aarch64_expand_compare_and_swap (rtx operands[])
emit_insn (GEN_FCN (code) (rval, mem, oldval, newval,
is_weak, mod_s, mod_f));
- cc_reg = gen_rtx_REG (CCmode, CC_REGNUM);
+ rtx cc_reg = gen_rtx_REG (CCmode, CC_REGNUM);
+ aarch64_emit_compare_and_swap_result (bval, cc_reg);
}
if (r_mode != mode)
rval = gen_lowpart (mode, rval);
emit_move_insn (operands[1], rval);
-
- x = gen_rtx_EQ (SImode, cc_reg, const0_rtx);
- emit_insn (gen_rtx_SET (bval, x));
}
/* Emit a barrier, that is appropriate for memory model MODEL, at the end of a
@@ -115,9 +115,19 @@
#define TARGET_PTRMEMFUNC_VBIT_LOCATION ptrmemfunc_vbit_in_delta
-/* Emit calls to libgcc helpers for atomic operations for runtime detection
- of LSE instructions. */
-#define TARGET_OUTLINE_ATOMICS (aarch64_flag_outline_atomics)
+/* Use run-time LSE detection for atomic operations. */
+#define TARGET_OUTLINE_ATOMICS \
+ (aarch64_flag_outline_atomics != AARCH64_OUTLINE_ATOMICS_NONE)
+
+/* Test for LSE inline and use an LSE instruction directly on the likely
+ path, with an inline LDXR/STXR fallback on the unlikely path. */
+#define TARGET_OUTLINE_ATOMICS_INLINE \
+ (aarch64_flag_outline_atomics == AARCH64_OUTLINE_ATOMICS_INLINE)
+
+/* True if LSE instructions may be emitted, either because the current
+ function is built for them or because they are emitted under a run-time
+ test. */
+#define TARGET_LSE_INSNS (TARGET_LSE || TARGET_OUTLINE_ATOMICS_INLINE)
/* Align global data as an optimization. */
#define DATA_ALIGNMENT(EXP, ALIGN) aarch64_data_alignment (EXP, ALIGN)
@@ -367,9 +367,25 @@ TargetVariable
long aarch64_stack_protector_guard_offset = 0
moutline-atomics
-Target Var(aarch64_flag_outline_atomics) Init(2) Save
+Target Alias(moutline-atomics=, call, none)
Generate local calls to out-of-line atomic operations.
+moutline-atomics=
+Target RejectNegative Joined Enum(aarch64_outline_atomics) Var(aarch64_flag_outline_atomics) Init(AARCH64_OUTLINE_ATOMICS_CALL) Save
+Select how to generate atomic operations that use run-time LSE detection.
+
+Enum
+Name(aarch64_outline_atomics) Type(enum aarch64_outline_atomics) UnknownError(unknown value %qs for %<-moutline-atomics=%>)
+
+EnumValue
+Enum(aarch64_outline_atomics) String(none) Value(AARCH64_OUTLINE_ATOMICS_NONE)
+
+EnumValue
+Enum(aarch64_outline_atomics) String(call) Value(AARCH64_OUTLINE_ATOMICS_CALL)
+
+EnumValue
+Enum(aarch64_outline_atomics) String(inline) Value(AARCH64_OUTLINE_ATOMICS_INLINE)
+
-param=aarch64-vect-compare-costs=
Target Joined UInteger Var(aarch64_vect_compare_costs) Init(1) IntegerRange(0, 1) Param
When vectorizing, consider using multiple different approaches and use
@@ -107,3 +107,5 @@ UrlSuffix(gcc/AArch64-Options.html#index-mstack-protector-guard-offset)
moutline-atomics
UrlSuffix(gcc/AArch64-Options.html#index-moutline-atomics)
+moutline-atomics=
+UrlSuffix(gcc/AArch64-Options.html#index-moutline-atomics)
@@ -125,17 +125,19 @@
(match_operand:SHORT 2 "aarch64_reg_or_zero" "rZ") ;; desired
(match_operand:SI 3 "const_int_operand")] ;; mod_s
UNSPECV_ATOMIC_CMPSW))]
- "TARGET_LSE"
+ "TARGET_LSE_INSNS"
{
enum memmodel model = memmodel_from_int (INTVAL (operands[3]));
+ const char *templ;
if (is_mm_relaxed (model))
- return "cas<atomic_sfx>\t%<w>0, %<w>2, %1";
+ templ = "cas<atomic_sfx>\t%<w>0, %<w>2, %1";
else if (is_mm_acquire (model) || is_mm_consume (model))
- return "casa<atomic_sfx>\t%<w>0, %<w>2, %1";
+ templ = "casa<atomic_sfx>\t%<w>0, %<w>2, %1";
else if (is_mm_release (model))
- return "casl<atomic_sfx>\t%<w>0, %<w>2, %1";
+ templ = "casl<atomic_sfx>\t%<w>0, %<w>2, %1";
else
- return "casal<atomic_sfx>\t%<w>0, %<w>2, %1";
+ templ = "casal<atomic_sfx>\t%<w>0, %<w>2, %1";
+ return aarch64_output_lse_insn (templ, operands);
})
(define_insn "@aarch64_compare_and_swap<mode>_lse"
@@ -147,17 +149,19 @@
(match_operand:GPI 2 "aarch64_reg_or_zero" "rZ") ;; desired
(match_operand:SI 3 "const_int_operand")] ;; mod_s
UNSPECV_ATOMIC_CMPSW))]
- "TARGET_LSE"
+ "TARGET_LSE_INSNS"
{
enum memmodel model = memmodel_from_int (INTVAL (operands[3]));
+ const char *templ;
if (is_mm_relaxed (model))
- return "cas<atomic_sfx>\t%<w>0, %<w>2, %1";
+ templ = "cas<atomic_sfx>\t%<w>0, %<w>2, %1";
else if (is_mm_acquire (model) || is_mm_consume (model))
- return "casa<atomic_sfx>\t%<w>0, %<w>2, %1";
+ templ = "casa<atomic_sfx>\t%<w>0, %<w>2, %1";
else if (is_mm_release (model))
- return "casl<atomic_sfx>\t%<w>0, %<w>2, %1";
+ templ = "casl<atomic_sfx>\t%<w>0, %<w>2, %1";
else
- return "casal<atomic_sfx>\t%<w>0, %<w>2, %1";
+ templ = "casal<atomic_sfx>\t%<w>0, %<w>2, %1";
+ return aarch64_output_lse_insn (templ, operands);
})
(define_insn "@aarch64_compare_and_swap<mode>_lse"
@@ -169,17 +173,19 @@
(match_operand:JUST_TI 2 "register_operand" "r") ;; desired
(match_operand:SI 3 "const_int_operand")] ;; mod_s
UNSPECV_ATOMIC_CMPSW))]
- "TARGET_LSE"
+ "TARGET_LSE_INSNS"
{
enum memmodel model = memmodel_from_int (INTVAL (operands[3]));
+ const char *templ;
if (is_mm_relaxed (model))
- return "casp\t%0, %R0, %2, %R2, %1";
+ templ = "casp\t%0, %R0, %2, %R2, %1";
else if (is_mm_acquire (model) || is_mm_consume (model))
- return "caspa\t%0, %R0, %2, %R2, %1";
+ templ = "caspa\t%0, %R0, %2, %R2, %1";
else if (is_mm_release (model))
- return "caspl\t%0, %R0, %2, %R2, %1";
+ templ = "caspl\t%0, %R0, %2, %R2, %1";
else
- return "caspal\t%0, %R0, %2, %R2, %1";
+ templ = "caspal\t%0, %R0, %2, %R2, %1";
+ return aarch64_output_lse_insn (templ, operands);
})
(define_expand "atomic_exchange<mode>"
@@ -190,10 +196,22 @@
""
{
/* Use an atomic SWP when available. */
- if (TARGET_LSE)
+ if (TARGET_LSE_INSNS)
{
+ aarch64_lse_dispatch dispatch;
+ if (!TARGET_LSE)
+ dispatch.start ();
+
emit_insn (gen_aarch64_atomic_exchange<mode>_lse
(operands[0], operands[1], operands[2], operands[3]));
+
+ if (!TARGET_LSE)
+ {
+ dispatch.mid ();
+ emit_insn (gen_aarch64_atomic_exchange<mode>
+ (operands[0], operands[1], operands[2], operands[3]));
+ dispatch.end ();
+ }
}
else if (TARGET_OUTLINE_ATOMICS)
{
@@ -221,7 +239,7 @@
(unspec_volatile:ALLI
[(match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ") ;; input
(match_operand:SI 3 "const_int_operand" "")] ;; model
- UNSPECV_ATOMIC_EXCHG))
+ UNSPECV_ATOMIC_EXCHG_LLSC))
(clobber (reg:CC CC_REGNUM))
(clobber (match_scratch:SI 4 "=&r"))]
"!TARGET_LSE"
@@ -243,17 +261,19 @@
[(match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
(match_operand:SI 3 "const_int_operand" "")]
UNSPECV_ATOMIC_EXCHG))]
- "TARGET_LSE"
+ "TARGET_LSE_INSNS"
{
enum memmodel model = memmodel_from_int (INTVAL (operands[3]));
+ const char *templ;
if (is_mm_relaxed (model))
- return "swp<atomic_sfx>\t%<w>2, %<w>0, %1";
+ templ = "swp<atomic_sfx>\t%<w>2, %<w>0, %1";
else if (is_mm_acquire (model) || is_mm_consume (model))
- return "swpa<atomic_sfx>\t%<w>2, %<w>0, %1";
+ templ = "swpa<atomic_sfx>\t%<w>2, %<w>0, %1";
else if (is_mm_release (model))
- return "swpl<atomic_sfx>\t%<w>2, %<w>0, %1";
+ templ = "swpl<atomic_sfx>\t%<w>2, %<w>0, %1";
else
- return "swpal<atomic_sfx>\t%<w>2, %<w>0, %1";
+ templ = "swpal<atomic_sfx>\t%<w>2, %<w>0, %1";
+ return aarch64_output_lse_insn (templ, operands);
}
)
@@ -267,8 +287,13 @@
rtx (*gen) (rtx, rtx, rtx);
/* Use an atomic load-operate instruction when possible. */
- if (TARGET_LSE)
+ if (TARGET_LSE || TARGET_OUTLINE_ATOMICS)
{
+ const atomic_ool_names *names;
+ /* MINUS and AND are done as an addition of the negated operand and
+ a bit clear of the inverted one, so an LDXR/STXR fallback, which
+ implements the original operation, needs the original operand. */
+ rtx orig_val = operands[1];
switch (<CODE>)
{
case MINUS:
@@ -277,59 +302,54 @@
/* fallthru */
case PLUS:
gen = gen_aarch64_atomic_add<mode>_lse;
+ names = &aarch64_ool_ldadd_names;
break;
case IOR:
gen = gen_aarch64_atomic_ior<mode>_lse;
+ names = &aarch64_ool_ldset_names;
break;
case XOR:
gen = gen_aarch64_atomic_xor<mode>_lse;
+ names = &aarch64_ool_ldeor_names;
break;
case AND:
operands[1] = expand_simple_unop (<MODE>mode, NOT, operands[1],
NULL, 1);
gen = gen_aarch64_atomic_bic<mode>_lse;
+ names = &aarch64_ool_ldclr_names;
break;
default:
gcc_unreachable ();
}
operands[1] = force_reg (<MODE>mode, operands[1]);
- }
- else if (TARGET_OUTLINE_ATOMICS)
- {
- const atomic_ool_names *names;
- switch (<CODE>)
+
+ if (!TARGET_LSE_INSNS)
{
- case MINUS:
- operands[1] = expand_simple_unop (<MODE>mode, NEG, operands[1],
- NULL, 1);
- /* fallthru */
- case PLUS:
- names = &aarch64_ool_ldadd_names;
- break;
- case IOR:
- names = &aarch64_ool_ldset_names;
- break;
- case XOR:
- names = &aarch64_ool_ldeor_names;
- break;
- case AND:
- operands[1] = expand_simple_unop (<MODE>mode, NOT, operands[1],
- NULL, 1);
- names = &aarch64_ool_ldclr_names;
- break;
- default:
- gcc_unreachable ();
+ rtx func = aarch64_atomic_ool_func (<MODE>mode, operands[2], names);
+ emit_library_call_value (func, NULL_RTX, LCT_NORMAL, <MODE>mode,
+ operands[1], <MODE>mode,
+ XEXP (operands[0], 0), Pmode);
+ DONE;
+ }
+
+ aarch64_lse_dispatch dispatch;
+ if (!TARGET_LSE)
+ dispatch.start ();
+
+ emit_insn (gen (operands[0], operands[1], operands[2]));
+
+ if (!TARGET_LSE)
+ {
+ dispatch.mid ();
+ emit_insn (gen_aarch64_atomic_<atomic_optab><mode>
+ (operands[0], force_reg (<MODE>mode, orig_val),
+ operands[2]));
+ dispatch.end ();
}
- machine_mode mode = <MODE>mode;
- rtx func = aarch64_atomic_ool_func (mode, operands[2], names);
- emit_library_call_value (func, NULL_RTX, LCT_NORMAL, mode,
- operands[1], mode,
- XEXP (operands[0], 0), Pmode);
- DONE;
+ DONE;
}
- else
- gen = gen_aarch64_atomic_<atomic_optab><mode>;
+ gen = gen_aarch64_atomic_<atomic_optab><mode>;
emit_insn (gen (operands[0], operands[1], operands[2]));
DONE;
}
@@ -376,17 +396,19 @@
(match_operand:SI 2 "const_int_operand")]
ATOMIC_LDOP))
(clobber (match_scratch:ALLI 3 "=r"))]
- "TARGET_LSE"
+ "TARGET_LSE_INSNS"
{
enum memmodel model = memmodel_from_int (INTVAL (operands[2]));
+ const char *templ;
if (is_mm_relaxed (model))
- return "ld<atomic_ldop><atomic_sfx>\t%<w>1, %<w>3, %0";
+ templ = "ld<atomic_ldop><atomic_sfx>\t%<w>1, %<w>3, %0";
else if (is_mm_release (model))
- return "ld<atomic_ldop>l<atomic_sfx>\t%<w>1, %<w>3, %0";
+ templ = "ld<atomic_ldop>l<atomic_sfx>\t%<w>1, %<w>3, %0";
else if (is_mm_acquire (model) || is_mm_consume (model))
- return "ld<atomic_ldop>a<atomic_sfx>\t%<w>1, %<w>3, %0";
+ templ = "ld<atomic_ldop>a<atomic_sfx>\t%<w>1, %<w>3, %0";
else
- return "ld<atomic_ldop>al<atomic_sfx>\t%<w>1, %<w>3, %0";
+ templ = "ld<atomic_ldop>al<atomic_sfx>\t%<w>1, %<w>3, %0";
+ return aarch64_output_lse_insn (templ, operands);
}
)
@@ -425,8 +447,11 @@
rtx (*gen) (rtx, rtx, rtx, rtx);
/* Use an atomic load-operate instruction when possible. */
- if (TARGET_LSE)
+ if (TARGET_LSE || TARGET_OUTLINE_ATOMICS)
{
+ const atomic_ool_names *names;
+ /* See the note in atomic_<atomic_optab><mode>. */
+ rtx orig_val = operands[2];
switch (<CODE>)
{
case MINUS:
@@ -435,60 +460,56 @@
/* fallthru */
case PLUS:
gen = gen_aarch64_atomic_fetch_add<mode>_lse;
+ names = &aarch64_ool_ldadd_names;
break;
case IOR:
gen = gen_aarch64_atomic_fetch_ior<mode>_lse;
+ names = &aarch64_ool_ldset_names;
break;
case XOR:
gen = gen_aarch64_atomic_fetch_xor<mode>_lse;
+ names = &aarch64_ool_ldeor_names;
break;
case AND:
operands[2] = expand_simple_unop (<MODE>mode, NOT, operands[2],
NULL, 1);
gen = gen_aarch64_atomic_fetch_bic<mode>_lse;
+ names = &aarch64_ool_ldclr_names;
break;
default:
gcc_unreachable ();
}
operands[2] = force_reg (<MODE>mode, operands[2]);
- }
- else if (TARGET_OUTLINE_ATOMICS)
- {
- const atomic_ool_names *names;
- switch (<CODE>)
+
+ if (!TARGET_LSE_INSNS)
{
- case MINUS:
- operands[2] = expand_simple_unop (<MODE>mode, NEG, operands[2],
- NULL, 1);
- /* fallthru */
- case PLUS:
- names = &aarch64_ool_ldadd_names;
- break;
- case IOR:
- names = &aarch64_ool_ldset_names;
- break;
- case XOR:
- names = &aarch64_ool_ldeor_names;
- break;
- case AND:
- operands[2] = expand_simple_unop (<MODE>mode, NOT, operands[2],
- NULL, 1);
- names = &aarch64_ool_ldclr_names;
- break;
- default:
- gcc_unreachable ();
+ rtx func = aarch64_atomic_ool_func (<MODE>mode, operands[3], names);
+ rtx rval = emit_library_call_value (func, operands[0], LCT_NORMAL,
+ <MODE>mode, operands[2],
+ <MODE>mode,
+ XEXP (operands[1], 0), Pmode);
+ emit_move_insn (operands[0], rval);
+ DONE;
+ }
+
+ aarch64_lse_dispatch dispatch;
+ if (!TARGET_LSE)
+ dispatch.start ();
+
+ emit_insn (gen (operands[0], operands[1], operands[2], operands[3]));
+
+ if (!TARGET_LSE)
+ {
+ dispatch.mid ();
+ emit_insn (gen_aarch64_atomic_fetch_<atomic_optab><mode>
+ (operands[0], operands[1],
+ force_reg (<MODE>mode, orig_val), operands[3]));
+ dispatch.end ();
}
- machine_mode mode = <MODE>mode;
- rtx func = aarch64_atomic_ool_func (mode, operands[3], names);
- rtx rval = emit_library_call_value (func, operands[0], LCT_NORMAL, mode,
- operands[2], mode,
- XEXP (operands[1], 0), Pmode);
- emit_move_insn (operands[0], rval);
DONE;
}
- else
- gen = gen_aarch64_atomic_fetch_<atomic_optab><mode>;
+ gen = gen_aarch64_atomic_fetch_<atomic_optab><mode>;
emit_insn (gen (operands[0], operands[1], operands[2], operands[3]));
DONE;
})
@@ -525,17 +546,19 @@
(match_operand:ALLI 2 "register_operand" "r")
(match_operand:SI 3 "const_int_operand")]
ATOMIC_LDOP))]
- "TARGET_LSE"
+ "TARGET_LSE_INSNS"
{
enum memmodel model = memmodel_from_int (INTVAL (operands[3]));
+ const char *templ;
if (is_mm_relaxed (model))
- return "ld<atomic_ldop><atomic_sfx>\t%<w>2, %<w>0, %1";
+ templ = "ld<atomic_ldop><atomic_sfx>\t%<w>2, %<w>0, %1";
else if (is_mm_acquire (model) || is_mm_consume (model))
- return "ld<atomic_ldop>a<atomic_sfx>\t%<w>2, %<w>0, %1";
+ templ = "ld<atomic_ldop>a<atomic_sfx>\t%<w>2, %<w>0, %1";
else if (is_mm_release (model))
- return "ld<atomic_ldop>l<atomic_sfx>\t%<w>2, %<w>0, %1";
+ templ = "ld<atomic_ldop>l<atomic_sfx>\t%<w>2, %<w>0, %1";
else
- return "ld<atomic_ldop>al<atomic_sfx>\t%<w>2, %<w>0, %1";
+ templ = "ld<atomic_ldop>al<atomic_sfx>\t%<w>2, %<w>0, %1";
+ return aarch64_output_lse_insn (templ, operands);
}
)
@@ -1486,6 +1486,7 @@
UNSPECV_HINTS_FETCH ; Likewise.
UNSPECV_ATOMIC_CMPSW ; Represent an atomic compare swap.
UNSPECV_ATOMIC_EXCHG ; Represent an atomic exchange.
+ UNSPECV_ATOMIC_EXCHG_LLSC ; Represent an LDXR/STXR exchange.
UNSPECV_ATOMIC_CAS ; Represent an atomic CAS.
UNSPECV_ATOMIC_SWP ; Represent an atomic SWP.
UNSPECV_ATOMIC_OP ; Represent an atomic operation.
@@ -28975,6 +28975,45 @@ if test $gcc_cv_as_aarch64_dtprel_relocation = yes; then
$as_echo "#define HAVE_AS_DTPREL_RELOC 1" >>confdefs.h
+fi
+
+ # Check if we can turn the LSE extension on and off around a single
+ # instruction, which -moutline-atomics=inline needs in order to emit an
+ # LSE instruction under a run-time test.
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for LSE" >&5
+$as_echo_n "checking assembler for LSE... " >&6; }
+if ${gcc_cv_as_aarch64_lse+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ gcc_cv_as_aarch64_lse=no
+ if test x"$gcc_cv_as" != x; then
+ $as_echo '
+ .arch armv8-a
+ .arch_extension lse
+ casal x0, x1, [x2]
+ .arch_extension nolse
+ ' > conftest.s
+ if { ac_try='$gcc_cv_as $gcc_cv_as_flags -o conftest.o conftest.s >&5'
+ { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5
+ (eval $ac_try) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; }
+ then
+ gcc_cv_as_aarch64_lse=yes
+ else
+ echo "configure: failed program was" >&5
+ cat conftest.s >&5
+ fi
+ rm -f conftest.o conftest.s
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_as_aarch64_lse" >&5
+$as_echo "$gcc_cv_as_aarch64_lse" >&6; }
+if test $gcc_cv_as_aarch64_lse = yes; then
+
+$as_echo "#define HAVE_AS_LSE 1" >>confdefs.h
+
fi
# Enable Branch Target Identification Mechanism and Return Address
@@ -4683,6 +4683,19 @@ case "$target" in
.xword %dtprel(expr)
],,[AC_DEFINE(HAVE_AS_DTPREL_RELOC, 1,
[Define if your assembler supports DTPREL relocation.])])
+ # Check if we can turn the LSE extension on and off around a single
+ # instruction, which -moutline-atomics=inline needs in order to emit an
+ # LSE instruction under a run-time test.
+ gcc_GAS_CHECK_FEATURE([LSE],
+ gcc_cv_as_aarch64_lse,,
+ [
+ .arch armv8-a
+ .arch_extension lse
+ casal x0, x1, [[x2]]
+ .arch_extension nolse
+ ],,[AC_DEFINE(HAVE_AS_LSE, 1,
+ [Define if your assembler can enable the LSE extension for a single
+ instruction.])])
# Enable Branch Target Identification Mechanism and Return Address
# Signing by default.
AC_ARG_ENABLE(standard-branch-protection,
@@ -5981,11 +5981,14 @@ behavior and permissible arguments are the same as for the command-line option
@option{-mbranch-protection=}. The default value is @code{none}.
@atindex @code{target("outline-atomics")}, AArch64
+@atindex @code{target("outline-atomics=")}, AArch64
@item outline-atomics
@itemx no-outline-atomics
-Enable or disable calls to out-of-line helpers to implement atomic operations.
-This corresponds to the behavior of the command-line options
-@option{-moutline-atomics} and @option{-mno-outline-atomics}.
+@itemx outline-atomics=@var{variant}
+Select how atomic operations use run-time LSE detection. The behavior and
+valid arguments are the same as for @option{-moutline-atomics},
+@option{-mno-outline-atomics}, and
+@option{-moutline-atomics=@var{variant}}.
@atindex @code{target("max-vectorization")}, AArch64
@item max-vectorization
@@ -906,7 +906,8 @@ Objective-C and Objective-C++ Dialects}.
-moverride=@var{string}
-mstack-protector-guard=@var{guard} -mstack-protector-guard-reg=@var{sysreg}
-mstack-protector-guard-offset=@var{offset} -mtrack-speculation
--moutline-atomics -mearly-ra -mearly-ldp-fusion -mlate-ldp-fusion -mnarrow-gp-writes
+-moutline-atomics -moutline-atomics=@var{variant}
+-mearly-ra -mearly-ldp-fusion -mlate-ldp-fusion -mnarrow-gp-writes
-msve-vector-bits=@var{bits}}
@emph{Adapteva Epiphany Options} (@ref{Adapteva Epiphany Options})
@@ -21623,18 +21624,48 @@ sequence to be generated.
@opindex mno-outline-atomics
@item -moutline-atomics
@itemx -mno-outline-atomics
-Enable or disable calls to out-of-line helpers to implement atomic operations.
-These helpers will, at runtime, determine if the LSE instructions from
-ARMv8.1-A can be used; if not, they will use the load/store-exclusive
-instructions that are present in the base ARMv8.0 ISA.
-
-This option is only applicable when compiling for the base ARMv8.0
-instruction set. If using a later revision, e.g. @option{-march=armv8.1-a}
-or @option{-march=armv8-a+lse}, the ARMv8.1-Atomics instructions will be
-used directly. The same applies when using @option{-mcpu=} when the
-selected CPU supports the @samp{lse} feature.
+@itemx -moutline-atomics=@var{variant}
+Select how atomic operations use run-time detection of the LSE instructions
+from ARMv8.1-A. Code for processors without LSE uses load/store-exclusive
+instructions from the base ARMv8.0 ISA.
+
+This option only affects functions compiled without LSE. If LSE is enabled,
+for example by @option{-march=armv8.1-a}, @option{-march=armv8-a+lse}, or an
+@option{-mcpu=} option for a processor that supports LSE, the LSE
+instructions are used directly.
This option is on by default.
+@var{variant} selects how the run-time test is done:
+
+@table @samp
+@item none
+Do not test for LSE. Equivalent to @option{-mno-outline-atomics}.
+
+@item call
+Do the whole operation, test included, in a helper. Each affected atomic
+operation becomes one call, which has the smallest per-site dispatch, but the
+call follows the base procedure call standard. The compiler must therefore
+treat every call-clobbered register as clobbered.
+Equivalent to @option{-moutline-atomics}. This is the default.
+
+@item inline
+Do the test inline and use an LSE instruction directly. Emit the ARMv8.0
+sequence inline on a branch that is predicted not taken. No outline-atomics
+helper call is generated, so registers can stay live across an atomic
+operation and a leaf function remains one. The test can be hoisted out of
+loops. This is larger than @samp{call}.
+@end table
+
+The last outline-atomics option on the command line takes effect.
+@option{-moutline-atomics} is an alias for
+@option{-moutline-atomics=call}. @option{-mno-outline-atomics} is an alias
+for @option{-moutline-atomics=none}.
+
+The @samp{inline} variant needs assembler support for enabling and disabling
+LSE around one instruction. This support is not needed when LSE is enabled
+at compile time for the current function. It also uses the existing
+@code{__aarch64_have_lse_atomics} flag from libgcc.
+
@opindex mmax-vectorization
@opindex mno-max-vectorization
@item -mmax-vectorization
new file mode 100644
@@ -0,0 +1,67 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -O2 -moutline-atomics=inline" } */
+
+/* Each operation tests __aarch64_have_lse_atomics inline, uses an LSE
+ instruction when it is set, and uses an inline LDXR/STXR fallback when it
+ is not. */
+
+int
+cas (int *p, int expected, int desired)
+{
+ return __atomic_compare_exchange_n (p, &expected, desired, 0,
+ __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+}
+
+int
+swp (int *p, int v)
+{
+ return __atomic_exchange_n (p, v, __ATOMIC_SEQ_CST);
+}
+
+int
+ldadd (int *p, int v)
+{
+ return __atomic_fetch_add (p, v, __ATOMIC_SEQ_CST);
+}
+
+int
+ldset (int *p, int v)
+{
+ return __atomic_fetch_or (p, v, __ATOMIC_SEQ_CST);
+}
+
+int
+ldclr (int *p, int v)
+{
+ return __atomic_fetch_and (p, v, __ATOMIC_SEQ_CST);
+}
+
+int
+ldeor (int *p, int v)
+{
+ return __atomic_fetch_xor (p, v, __ATOMIC_SEQ_CST);
+}
+
+/* { dg-final { scan-assembler "__aarch64_have_lse_atomics" } } */
+
+/* { dg-final { scan-assembler-times "\tcasal\tw" 1 } } */
+/* { dg-final { scan-assembler-times "\tswpal\tw" 1 } } */
+/* { dg-final { scan-assembler-times "\tldaddal\tw" 1 } } */
+/* { dg-final { scan-assembler-times "\tldsetal\tw" 1 } } */
+/* { dg-final { scan-assembler-times "\tldclral\tw" 1 } } */
+/* { dg-final { scan-assembler-times "\tldeoral\tw" 1 } } */
+
+/* Each operation falls back to an LDXR/STXR sequence rather than a call,
+ so nothing is left out of line. */
+/* { dg-final { scan-assembler-times "ldaxr\tw" 6 } } */
+/* { dg-final { scan-assembler-times "stlxr\tw" 6 } } */
+/* Ordinary __atomic models do not need the final __sync barrier. */
+/* { dg-final { scan-assembler-not "dmb\tish" } } */
+/* { dg-final { scan-assembler-not "bl\t__aarch64_" } } */
+
+/* The LSE instructions are emitted for a target that does not have LSE
+ enabled, so they have to be bracketed by directives that let the
+ assembler accept them. */
+/* { dg-final { scan-assembler-times "\\.arch_extension lse" 6 } } */
+/* { dg-final { scan-assembler-times "\\.arch_extension nolse" 6 } } */
new file mode 100644
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -mcmodel=small -O2" } */
+/* { dg-additional-options "-moutline-atomics=inline" } */
+/* { dg-additional-options "-fdump-rtl-loop2_invariant-details" } */
+
+/* The test for LSE is an ordinary load from a location that does not
+ change, so it is hoisted out of a loop rather than repeated on every
+ iteration. */
+
+void
+loop (int *p, int n)
+{
+ for (int i = 0; i < n; i++)
+ __atomic_fetch_add (p, i, __ATOMIC_RELAXED);
+}
+
+/* One load of the flag for the one operation, hoisted out of the loop. */
+/* { dg-final { scan-assembler-times ":lo12:__aarch64_have_lse_atomics" 1 } } */
+/* { dg-final { scan-assembler-times "adrp\tx\[0-9\]+, __aarch64_have_lse_atomics" 1 } } */
+/* The loop body may be duplicated by later passes, so only require that
+ both arms are present. */
+/* { dg-final { scan-assembler "ldadd\tw" } } */
+/* { dg-final { scan-assembler "ldxr\tw" } } */
+/* { dg-final { scan-assembler-not "bl\t__aarch64_" } } */
+
+/* Check that loop invariant motion moved the read-only, non-trapping flag
+ load. */
+/* { dg-final { scan-rtl-dump "Decided to move invariant" "loop2_invariant" } } */
+/* { dg-final { scan-rtl-dump "zero_extend:SI \\(mem/u/c:QI" "loop2_invariant" } } */
new file mode 100644
@@ -0,0 +1,26 @@
+/* { dg-do compile } */
+/* { dg-options "-march=armv8-a+lse -O2 -fno-ipa-icf" } */
+/* { dg-additional-options "-moutline-atomics=inline" } */
+
+/* When LSE is available at compile time there is nothing to test for, so
+ -moutline-atomics=inline generates the plain LSE instruction. */
+
+int
+ldadd (int *p, int v)
+{
+ return __atomic_fetch_add (p, v, __ATOMIC_SEQ_CST);
+}
+
+int
+sync_ldadd (int *p, int v)
+{
+ return __sync_fetch_and_add (p, v);
+}
+
+/* { dg-final { scan-assembler-times "ldaddal\tw" 2 } } */
+/* LSE instructions encode the ordering needed by the legacy operation. */
+/* { dg-final { scan-assembler-not "dmb\tish" } } */
+/* { dg-final { scan-assembler-not "lda?xr\tw" } } */
+/* { dg-final { scan-assembler-not "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-assembler-not "__aarch64_ldadd" } } */
+/* { dg-final { scan-assembler-not "arch_extension" } } */
new file mode 100644
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-options "-march=armv8-a+nolse -O2 -moutline-atomics=none" } */
+
+/* -moutline-atomics=none is -mno-outline-atomics. */
+
+int
+ldadd (int *p, int v)
+{
+ return __atomic_fetch_add (p, v, __ATOMIC_SEQ_CST);
+}
+
+/* { dg-final { scan-assembler "ldaxr\tw" } } */
+/* { dg-final { scan-assembler "stlxr\tw" } } */
+/* { dg-final { scan-assembler-not "__aarch64_" } } */
new file mode 100644
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -O2 -moutline-atomics=inline" } */
+
+/* The 16-byte compare and swap uses CASP on the inline path and the
+ LDXP/STXP sequence on the fallback path. A 16-byte __atomic operation
+ is not lock free on this target, so use the __sync form, which reaches
+ the pattern. */
+
+__int128
+cas16 (__int128 *p, __int128 expected, __int128 desired)
+{
+ return __sync_val_compare_and_swap (p, expected, desired);
+}
+
+/* { dg-final { scan-assembler-times "caspal\tx" 1 } } */
+/* { dg-final { scan-assembler-times "ldxp\tx" 1 } } */
+/* { dg-final { scan-assembler-times "stlxp\tw" 2 } } */
+/* The LL/SC arm needs the final legacy __sync barrier. */
+/* { dg-final { scan-assembler-times "dmb\tish" 1 } } */
+/* { dg-final { scan-assembler-not "bl\t__aarch64_" } } */
+/* { dg-final { scan-assembler "__aarch64_have_lse_atomics" } } */
new file mode 100644
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -mcmodel=small -O2" } */
+/* { dg-additional-options "-moutline-atomics=inline" } */
+
+/* Several operations in one function each get their own inline LSE
+ sequence, and share the address computation for the flag. */
+
+void
+several (int *p, int *q)
+{
+ __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+ __atomic_fetch_add (q, 1, __ATOMIC_RELAXED);
+ __atomic_fetch_or (p, 2, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler-times "ldadd\tw" 2 } } */
+/* { dg-final { scan-assembler-times "ldset\tw" 1 } } */
+/* { dg-final { scan-assembler-times "adrp\tx\[0-9\]+, __aarch64_have_lse_atomics" 1 } } */
+/* { dg-final { scan-assembler-times "ldxr\tw" 3 } } */
+/* { dg-final { scan-assembler-times "stxr\tw" 3 } } */
+/* { dg-final { scan-assembler-not "bl\t__aarch64_" } } */
new file mode 100644
@@ -0,0 +1,8 @@
+/* { dg-do compile } */
+/* { dg-options "-O2" } */
+
+int __attribute__((target ("no-outline-atomics=inline")))
+f (int *p)
+{ /* { dg-error "invalid .* argument" } */
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
new file mode 100644
@@ -0,0 +1,95 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -O2 -fno-ipa-icf" } */
+/* { dg-additional-options "-moutline-atomics=none" } */
+
+#define TEST(NAME, ATTR) \
+ int __attribute__((target (ATTR))) \
+ NAME (int *p) \
+ { \
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED); \
+ }
+
+/*
+** inline_enum:
+** ...
+** ldadd w[0-9]+, w[0-9]+, \[x[0-9]+\]
+** ...
+** ldxr w[0-9]+, \[x[0-9]+\]
+** ...
+*/
+TEST (inline_enum, "outline-atomics=inline")
+
+/*
+** call_enum:
+** ...
+** bl __aarch64_ldadd4_relax
+** ...
+*/
+TEST (call_enum, "outline-atomics=call")
+
+/*
+** call_bool:
+** ...
+** bl __aarch64_ldadd4_relax
+** ...
+*/
+TEST (call_bool, "outline-atomics")
+
+/*
+** none_enum:
+** ...
+** ldxr w[0-9]+, \[x[0-9]+\]
+** ...
+*/
+TEST (none_enum, "outline-atomics=none")
+
+/*
+** none_bool:
+** ...
+** ldxr w[0-9]+, \[x[0-9]+\]
+** ...
+*/
+TEST (none_bool, "no-outline-atomics")
+
+#pragma GCC push_options
+#pragma GCC target ("outline-atomics=inline")
+/*
+** inline_pragma:
+** ...
+** ldadd w[0-9]+, w[0-9]+, \[x[0-9]+\]
+** ...
+** ldxr w[0-9]+, \[x[0-9]+\]
+** ...
+*/
+int
+inline_pragma (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+#pragma GCC pop_options
+
+/*
+** last_none:
+** ...
+** ldxr w[0-9]+, \[x[0-9]+\]
+** ...
+*/
+TEST (last_none, "outline-atomics=inline,no-outline-atomics")
+
+/*
+** last_inline:
+** ...
+** ldadd w[0-9]+, w[0-9]+, \[x[0-9]+\]
+** ...
+** ldxr w[0-9]+, \[x[0-9]+\]
+** ...
+*/
+TEST (last_inline, "no-outline-atomics,outline-atomics=inline")
+
+/* Three inline functions contain both arms. Two call functions contain a
+ helper call. Three functions use only the LL/SC sequence. */
+/* { dg-final { scan-assembler-times "\tldadd\tw" 3 } } */
+/* { dg-final { scan-assembler-times "\tldxr\tw" 6 } } */
+/* { dg-final { scan-assembler-times "bl\t__aarch64_ldadd4_relax" 2 } } */
+/* { dg-final { check-function-bodies "**" "" "" } } */
new file mode 100644
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-additional-options "-moutline-atomics=inline -moutline-atomics" } */
+
+/* The bare spelling is an alias for =call. The last option wins. */
+
+int
+f (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "bl\t__aarch64_ldadd4_relax" } } */
+/* { dg-final { scan-assembler-not "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-assembler-not "\tldadd\tw" } } */
new file mode 100644
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-additional-options "-moutline-atomics -moutline-atomics=inline" } */
+
+/* The enum spelling overrides the bare spelling when it is last. */
+
+int
+f (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-assembler "\tldadd\tw" } } */
+/* { dg-final { scan-assembler "\tldxr\tw" } } */
+/* { dg-final { scan-assembler-not "bl\t__aarch64_" } } */
new file mode 100644
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -Os" } */
+/* { dg-additional-options "-mno-outline-atomics -moutline-atomics=inline" } */
+
+/* An explicit inline request overrides both -Os and the earlier option. */
+
+int
+f (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-assembler "\tldadd\tw" } } */
+/* { dg-final { scan-assembler "\tldxr\tw" } } */
+/* { dg-final { scan-assembler-not "bl\t__aarch64_" } } */
new file mode 100644
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-additional-options "-moutline-atomics=inline -mno-outline-atomics" } */
+
+/* The negative spelling is an alias for =none. The last option wins. */
+
+int
+f (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "\tldxr\tw" } } */
+/* { dg-final { scan-assembler-not "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-assembler-not "__aarch64_ldadd" } } */
new file mode 100644
@@ -0,0 +1,181 @@
+/* { dg-do run { target aarch64*-*-linux* } } */
+/* { dg-require-effective-target glibc } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -O2 -moutline-atomics=inline" } */
+
+/* Check both arms on an LSE processor. Check the cold arm on every system. */
+
+#include <sys/auxv.h>
+
+#ifndef HWCAP_ATOMICS
+#define HWCAP_ATOMICS (1 << 8)
+#endif
+
+extern void abort (void);
+
+/* Defined by libgcc, and always linked in because the inline test refers
+ to it. */
+extern volatile _Bool __aarch64_have_lse_atomics
+ __attribute__((visibility ("hidden")));
+
+#define CHECK(cond) do { if (!(cond)) abort (); } while (0)
+
+#define TEST_TYPE(T, NAME, V0, B) \
+static void __attribute__((noinline)) \
+test_##NAME (void) \
+{ \
+ const T v0 = (T) (V0), b = (T) (B); \
+ T m, old, expected; \
+ \
+ m = v0; \
+ old = __atomic_exchange_n (&m, b, __ATOMIC_SEQ_CST); \
+ CHECK (old == v0 && m == b); \
+ \
+ m = v0; \
+ old = __atomic_fetch_add (&m, b, __ATOMIC_SEQ_CST); \
+ CHECK (old == v0 && m == (T) (v0 + b)); \
+ \
+ m = v0; \
+ old = __atomic_fetch_sub (&m, b, __ATOMIC_RELAXED); \
+ CHECK (old == v0 && m == (T) (v0 - b)); \
+ \
+ m = v0; \
+ __atomic_fetch_sub (&m, b, __ATOMIC_RELAXED); \
+ CHECK (m == (T) (v0 - b)); \
+ \
+ m = v0; \
+ old = __atomic_add_fetch (&m, b, __ATOMIC_ACQ_REL); \
+ CHECK (old == (T) (v0 + b) && m == old); \
+ \
+ m = v0; \
+ old = __atomic_sub_fetch (&m, b, __ATOMIC_RELEASE); \
+ CHECK (old == (T) (v0 - b) && m == old); \
+ \
+ m = v0; \
+ old = __atomic_fetch_and (&m, b, __ATOMIC_ACQUIRE); \
+ CHECK (old == v0 && m == (T) (v0 & b)); \
+ \
+ m = v0; \
+ __atomic_fetch_and (&m, b, __ATOMIC_RELAXED); \
+ CHECK (m == (T) (v0 & b)); \
+ \
+ m = v0; \
+ old = __atomic_fetch_or (&m, b, __ATOMIC_RELEASE); \
+ CHECK (old == v0 && m == (T) (v0 | b)); \
+ \
+ m = v0; \
+ old = __atomic_fetch_xor (&m, b, __ATOMIC_RELAXED); \
+ CHECK (old == v0 && m == (T) (v0 ^ b)); \
+ \
+ m = v0; \
+ old = __atomic_and_fetch (&m, b, __ATOMIC_ACQUIRE); \
+ CHECK (old == (T) (v0 & b) && m == old); \
+ \
+ m = v0; \
+ old = __atomic_or_fetch (&m, b, __ATOMIC_RELEASE); \
+ CHECK (old == (T) (v0 | b) && m == old); \
+ \
+ m = v0; \
+ old = __atomic_xor_fetch (&m, b, __ATOMIC_RELAXED); \
+ CHECK (old == (T) (v0 ^ b) && m == old); \
+ \
+ m = v0; expected = v0; \
+ CHECK (__atomic_compare_exchange_n (&m, &expected, b, 0, \
+ __ATOMIC_SEQ_CST, \
+ __ATOMIC_SEQ_CST)); \
+ CHECK (m == b); \
+ \
+ m = v0; expected = b; \
+ CHECK (!__atomic_compare_exchange_n (&m, &expected, (T) 0, 0, \
+ __ATOMIC_SEQ_CST, \
+ __ATOMIC_SEQ_CST)); \
+ CHECK (m == v0 && expected == v0); \
+ \
+ m = v0; expected = v0; \
+ while (!__atomic_compare_exchange_n (&m, &expected, b, 1, \
+ __ATOMIC_ACQ_REL, \
+ __ATOMIC_ACQUIRE)) \
+ CHECK (m == v0 && expected == v0); \
+ CHECK (m == b); \
+ \
+ m = v0; \
+ CHECK (__sync_fetch_and_add (&m, b) == v0); \
+ CHECK (m == (T) (v0 + b)); \
+}
+
+TEST_TYPE (unsigned char, u8, 0xa5, 0x3c)
+TEST_TYPE (unsigned short, u16, 0xa53c, 0x5a71)
+TEST_TYPE (unsigned int, u32, 0xa53c5a71u, 0x0f1e2d3cu)
+TEST_TYPE (unsigned long long, u64, 0xa53c5a710f1e2d3cull,
+ 0x123456789abcdef0ull)
+
+static void __attribute__((noinline))
+test_u128 (void)
+{
+ const unsigned __int128 v0
+ = ((unsigned __int128) 0xa53c5a710f1e2d3cull << 64)
+ | 0x0123456789abcdefull;
+ const unsigned __int128 b
+ = ((unsigned __int128) 0x123456789abcdef0ull << 64)
+ | 0xfedcba9876543210ull;
+ unsigned __int128 m = v0;
+
+ /* A 16-byte __atomic operation is not lock free on this target, so use
+ the __sync form, which is what reaches the 16-byte pattern. */
+ CHECK (__sync_val_compare_and_swap (&m, v0, b) == v0);
+ CHECK (m == b);
+
+ m = v0;
+ CHECK (!__sync_bool_compare_and_swap (&m, b, (unsigned __int128) 0));
+ CHECK (m == v0);
+
+ /* Only one half differing must still fail. */
+ m = v0;
+ CHECK (!__sync_bool_compare_and_swap
+ (&m, (v0 & ~(unsigned __int128) 0xffffffffffffffffull) | 1, b));
+ CHECK (m == v0);
+}
+
+/* Neither arm of the dispatch may disturb values that are live across the
+ atomic operation, including on the non-LSE fallback path. */
+
+static void __attribute__((noinline))
+test_live_across (unsigned int *p)
+{
+ double d = 1.5;
+ unsigned long g = 0x1234;
+
+ __asm__ volatile ("" : "+w" (d), "+r" (g));
+ CHECK (__atomic_fetch_add (p, 7, __ATOMIC_SEQ_CST) == 100);
+ __asm__ volatile ("" : "+w" (d), "+r" (g));
+
+ CHECK (d == 1.5 && g == 0x1234 && *p == 107);
+}
+
+static void
+run_all (void)
+{
+ unsigned int v = 100;
+
+ test_u8 ();
+ test_u16 ();
+ test_u32 ();
+ test_u64 ();
+ test_u128 ();
+ test_live_across (&v);
+}
+
+int
+main (void)
+{
+ if (getauxval (AT_HWCAP) & HWCAP_ATOMICS)
+ CHECK (__aarch64_have_lse_atomics);
+
+ run_all ();
+
+ /* Now force the non-LSE fallback. */
+ __aarch64_have_lse_atomics = 0;
+ run_all ();
+
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,44 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -O2 -moutline-atomics=inline" } */
+
+/* Legacy __sync operations need a final barrier on the LL/SC arm. The LSE
+ instructions encode the needed ordering. Exercise each expander that
+ starts an inline dispatch. */
+
+int
+fetch_add (int *p, int value)
+{
+ return __sync_fetch_and_add (p, value);
+}
+
+int
+compare_exchange (int *p, int old, int value)
+{
+ return __sync_val_compare_and_swap (p, old, value);
+}
+
+int
+exchange (int *p, int value)
+{
+ return __sync_lock_test_and_set (p, value);
+}
+
+void
+set (int *p, int value)
+{
+ __sync_fetch_and_or (p, value);
+}
+
+/* { dg-final { scan-assembler-times "\tldaddal\tw" 1 } } */
+/* { dg-final { scan-assembler-times "\tcasal\tw" 1 } } */
+/* { dg-final { scan-assembler-times "\tswpa\tw" 1 } } */
+/* { dg-final { scan-assembler-times "\tldsetal\tw" 1 } } */
+/* { dg-final { scan-assembler-times "\tldxr\tw" 4 } } */
+/* { dg-final { scan-assembler-times "\tstlxr\tw" 3 } } */
+/* { dg-final { scan-assembler-times "\tstxr\tw" 1 } } */
+/* Each of the four LL/SC sequences has a final barrier. */
+/* { dg-final { scan-assembler-times "\tdmb\tish" 4 } } */
+/* { dg-final { scan-assembler-times {\tstl?xr\tw[^\n]*\n\tcbnz\tw[^\n]*\n(?:\.L[0-9]+:\n)?\tdmb\tish} 4 } } */
+/* { dg-final { scan-assembler-not {\.arch_extension nolse\n\tdmb\tish} } } */
+/* { dg-final { scan-assembler-not "bl\t__aarch64_" } } */
new file mode 100644
@@ -0,0 +1,74 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -O2 -moutline-atomics=inline" } */
+
+/* The LSE arm determines success from its returned old value. The weak
+ LL/SC arm must instead use the STXR status so a failed store is not
+ reported as successful. Check the short modes as well because their LSE
+ comparisons use a different CC mode. */
+
+_Bool
+weak_cas_qi (unsigned char *p, unsigned char *expected, unsigned char desired)
+{
+ return __atomic_compare_exchange_n (p, expected, desired, 1,
+ __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+}
+
+/*
+** weak_cas_qi:
+** ...
+** casb (w[0-9]+), w[0-9]+, \[x[0-9]+\]
+** cmp \1, w[0-9]+, uxtb
+** cset w[0-9]+, eq
+** ...
+** stxrb (w[0-9]+), w[0-9]+, \[x[0-9]+\]
+** cmp \2, 0
+** cset w[0-9]+, eq
+** ...
+*/
+
+_Bool
+weak_cas_hi (unsigned short *p, unsigned short *expected,
+ unsigned short desired)
+{
+ return __atomic_compare_exchange_n (p, expected, desired, 1,
+ __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+}
+
+/*
+** weak_cas_hi:
+** ...
+** cash (w[0-9]+), w[0-9]+, \[x[0-9]+\]
+** cmp \1, w[0-9]+, uxth
+** cset w[0-9]+, eq
+** ...
+** stxrh (w[0-9]+), w[0-9]+, \[x[0-9]+\]
+** cmp \2, 0
+** cset w[0-9]+, eq
+** ...
+*/
+
+_Bool
+weak_cas_si (unsigned int *p, unsigned int *expected, unsigned int desired)
+{
+ return __atomic_compare_exchange_n (p, expected, desired, 1,
+ __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+}
+
+/*
+** weak_cas_si:
+** ...
+** cas (w[0-9]+), w[0-9]+, \[x[0-9]+\]
+** cmp \1, w[0-9]+
+** cset w[0-9]+, eq
+** ...
+** stxr (w[0-9]+), w[0-9]+, \[x[0-9]+\]
+** cmp \2, 0
+** cset w[0-9]+, eq
+** ...
+*/
+
+/* Each arm materializes its own Boolean result. Matching the STXR status
+ register in the following CMP also rejects a retry on store failure. */
+/* { dg-final { check-function-bodies "**" "" "" } } */
+/* { dg-final { scan-assembler-not "bl\t__aarch64_" } } */
@@ -13197,6 +13197,20 @@ proc check_effective_target_aarch64_asm_sve2p1_ok { } {
}
}
+# Return 1 if the AArch64 assembler can enable and then disable LSE around
+# one instruction.
+proc check_effective_target_aarch64_asm_lse_nolse_ok { } {
+ if { [istarget aarch64*-*-*] } {
+ return [check_no_compiler_messages aarch64_lse_nolse_assembler object {
+ __asm__ (".arch armv8-a\n"
+ ".arch_extension lse\n"
+ "casal x0, x1, [x2]\n"
+ ".arch_extension nolse");
+ }]
+ }
+ return 0
+}
+
proc check_effective_target_aarch64_asm_sve2p2_ok { } {
if { [istarget aarch64*-*-*] } {
return [check_no_compiler_messages aarch64_sve2p2_assembler object {