@@ -72,6 +72,10 @@
#define TARGET_ASM_FILE_END aarch64_file_end_indicate_exec_stack
+/* libgcc initializes the LSE availability flag when built for glibc. */
+#undef AARCH64_LIBGCC_LSE_INIT
+#define AARCH64_LIBGCC_LSE_INIT (DEFAULT_LIBC == LIBC_GLIBC)
+
/* Uninitialized common symbols in non-PIE executables, even with
strong definitions in dependent shared libraries, will resolve
to COPY relocated symbol in the executable. See PR65780. */
@@ -155,11 +155,14 @@ enum aarch64_early_ra_scope {
LSE and selects between an LSE instruction and an LDXR/STXR sequence.
- AARCH64_OUTLINE_ATOMICS_INLINE: test for LSE inline and use an LSE
instruction directly, expanding an LDXR/STXR sequence on the unlikely
- path where LSE is unavailable. */
+ path where LSE is unavailable.
+ - AARCH64_OUTLINE_ATOMICS_DEFAULT: select CALL or INLINE from the current
+ function options and the target runtime. */
enum aarch64_outline_atomics {
AARCH64_OUTLINE_ATOMICS_NONE,
AARCH64_OUTLINE_ATOMICS_CALL,
- AARCH64_OUTLINE_ATOMICS_INLINE
+ AARCH64_OUTLINE_ATOMICS_INLINE,
+ AARCH64_OUTLINE_ATOMICS_DEFAULT
};
#endif
@@ -1289,6 +1289,7 @@ extern const atomic_ool_names aarch64_ool_ldclr_names;
extern const atomic_ool_names aarch64_ool_ldeor_names;
const char *aarch64_output_lse_insn (const char *, rtx *);
+bool aarch64_outline_atomics_default_inline_p ();
/* Emit the code that -moutline-atomics=inline puts around an atomic
operation. The caller emits the LSE form after start and the inline
@@ -27258,6 +27258,22 @@ const atomic_ool_names aarch64_ool_ldeor_names = { { DEF4(ldeor) } };
#undef DEF4
#undef DEF5
+/* Return true if the default atomic dispatch should be emitted inline for
+ the current function. */
+
+bool
+aarch64_outline_atomics_default_inline_p ()
+{
+#ifdef HAVE_AS_LSE
+ return (optimize > 0
+ && !optimize_debug
+ && optimize_function_for_speed_p (cfun)
+ && AARCH64_LIBGCC_LSE_INIT);
+#else
+ return false;
+#endif
+}
+
/* Return TEMPL for the LSE instruction with OPERANDS if the current function
enables LSE. Otherwise emit the instruction between assembler directives
that enable and disable LSE, and return an empty template. */
@@ -119,10 +119,17 @@
#define TARGET_OUTLINE_ATOMICS \
(aarch64_flag_outline_atomics != AARCH64_OUTLINE_ATOMICS_NONE)
+/* True if libgcc initializes its LSE availability flag. */
+#ifndef AARCH64_LIBGCC_LSE_INIT
+#define AARCH64_LIBGCC_LSE_INIT 0
+#endif
+
/* Test for LSE inline and use an LSE instruction directly on the likely
path, with an inline LDXR/STXR fallback on the unlikely path. */
#define TARGET_OUTLINE_ATOMICS_INLINE \
- (aarch64_flag_outline_atomics == AARCH64_OUTLINE_ATOMICS_INLINE)
+ (aarch64_flag_outline_atomics == AARCH64_OUTLINE_ATOMICS_INLINE \
+ || (aarch64_flag_outline_atomics == AARCH64_OUTLINE_ATOMICS_DEFAULT \
+ && aarch64_outline_atomics_default_inline_p ()))
/* True if LSE instructions may be emitted, either because the current
function is built for them or because they are emitted under a run-time
@@ -371,7 +371,7 @@ Target Alias(moutline-atomics=, call, none)
Generate local calls to out-of-line atomic operations.
moutline-atomics=
-Target RejectNegative Joined Enum(aarch64_outline_atomics) Var(aarch64_flag_outline_atomics) Init(AARCH64_OUTLINE_ATOMICS_CALL) Save
+Target RejectNegative Joined Enum(aarch64_outline_atomics) Var(aarch64_flag_outline_atomics) Init(AARCH64_OUTLINE_ATOMICS_DEFAULT) Save
Select how to generate atomic operations that use run-time LSE detection.
Enum
@@ -21633,7 +21633,7 @@ This option only affects functions compiled without LSE. If LSE is enabled,
for example by @option{-march=armv8.1-a}, @option{-march=armv8-a+lse}, or an
@option{-mcpu=} option for a processor that supports LSE, the LSE
instructions are used directly.
-This option is on by default.
+Run-time detection is enabled by default.
@var{variant} selects how the run-time test is done:
@@ -21646,7 +21646,7 @@ Do the whole operation, test included, in a helper. Each affected atomic
operation becomes one call, which has the smallest per-site dispatch, but the
call follows the base procedure call standard. The compiler must therefore
treat every call-clobbered register as clobbered.
-Equivalent to @option{-moutline-atomics}. This is the default.
+Equivalent to @option{-moutline-atomics}.
@item inline
Do the test inline and use an LSE instruction directly. Emit the ARMv8.0
@@ -21656,6 +21656,12 @@ operation and a leaf function remains one. The test can be hoisted out of
loops. This is larger than @samp{call}.
@end table
+If no outline-atomics option is specified, GCC selects @samp{inline} when it
+is configured for GNU/Linux with glibc, the current function is optimized for
+speed, and the assembler supports the required LSE directives. GCC selects
+@samp{call} otherwise. Explicit outline-atomics options override this
+selection.
+
The last outline-atomics option on the command line takes effect.
@option{-moutline-atomics} is an alias for
@option{-moutline-atomics=call}. @option{-mno-outline-atomics} is an alias
new file mode 100644
@@ -0,0 +1,18 @@
+/* { dg-do compile { target aarch64*-*-linux* } } */
+/* { dg-require-effective-target glibc } */
+/* { dg-options "-march=armv8-a+nolse -O2" } */
+
+/* Optimized GNU/Linux glibc code uses inline dispatch by default. */
+
+int
+f (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "__aarch64_have_lse_atomics" { target aarch64_asm_lse_nolse_ok } } } */
+/* { dg-final { scan-assembler "\tldadd\tw" { target aarch64_asm_lse_nolse_ok } } } */
+/* { dg-final { scan-assembler "\tldxr\tw" { target aarch64_asm_lse_nolse_ok } } } */
+/* { dg-final { scan-assembler-not "bl\t__aarch64_" { target aarch64_asm_lse_nolse_ok } } } */
+/* { dg-final { scan-assembler "bl\t__aarch64_ldadd4_relax" { target { ! aarch64_asm_lse_nolse_ok } } } } */
+/* { dg-final { scan-assembler-not "__aarch64_have_lse_atomics" { target { ! aarch64_asm_lse_nolse_ok } } } } */
new file mode 100644
@@ -0,0 +1,15 @@
+/* { dg-do compile { target aarch64*-*-linux* } } */
+/* { dg-require-effective-target glibc } */
+/* { dg-options "-march=armv8-a+nolse -Os" } */
+
+/* Size optimization uses a helper call by default. */
+
+int
+f (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "bl\t__aarch64_ldadd4_relax" } } */
+/* { dg-final { scan-assembler-not "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-assembler-not "\tldadd\tw" } } */
new file mode 100644
@@ -0,0 +1,15 @@
+/* { dg-do compile { target aarch64*-*-linux* } } */
+/* { dg-require-effective-target glibc } */
+/* { dg-options "-march=armv8-a+nolse -O0" } */
+
+/* Unoptimized code uses a helper call by default. */
+
+int
+f (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "bl\t__aarch64_ldadd4_relax" } } */
+/* { dg-final { scan-assembler-not "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-assembler-not "\tldadd\tw" } } */
new file mode 100644
@@ -0,0 +1,15 @@
+/* { dg-do compile { target aarch64*-*-linux* } } */
+/* { dg-require-effective-target glibc } */
+/* { dg-options "-march=armv8-a+nolse -Og" } */
+
+/* -Og uses a helper call by default. */
+
+int
+f (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "bl\t__aarch64_ldadd4_relax" } } */
+/* { dg-final { scan-assembler-not "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-assembler-not "\tldadd\tw" } } */
new file mode 100644
@@ -0,0 +1,38 @@
+/* { dg-do compile { target aarch64*-*-linux* } } */
+/* { dg-require-effective-target glibc } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -O2 -fno-ipa-icf" } */
+
+int
+speed (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+int __attribute__ ((optimize ("Os")))
+size (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/*
+** speed:
+** ...
+** ldadd w[0-9]+, w[0-9]+, \[x[0-9]+\]
+** ...
+** ldxr w[0-9]+, \[x[0-9]+\]
+** ...
+*/
+
+/*
+** size:
+** ...
+** bl __aarch64_ldadd4_relax
+** ...
+*/
+
+/* The default follows each function's optimization mode. */
+/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-final { scan-assembler-times "\tldadd\tw" 1 } } */
+/* { dg-final { scan-assembler-times "\tldxr\tw" 1 } } */
+/* { dg-final { scan-assembler-times "bl\t__aarch64_ldadd4_relax" 1 } } */
new file mode 100644
@@ -0,0 +1,38 @@
+/* { dg-do compile { target aarch64*-*-linux* } } */
+/* { dg-require-effective-target glibc } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -Os -fno-ipa-icf" } */
+
+int
+size (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+int __attribute__ ((optimize ("O2")))
+speed (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/*
+** size:
+** ...
+** bl __aarch64_ldadd4_relax
+** ...
+*/
+
+/*
+** speed:
+** ...
+** ldadd w[0-9]+, w[0-9]+, \[x[0-9]+\]
+** ...
+** ldxr w[0-9]+, \[x[0-9]+\]
+** ...
+*/
+
+/* Function optimization can select inline in a size-optimized unit. */
+/* { dg-final { check-function-bodies "**" "" } } */
+/* { dg-final { scan-assembler-times "\tldadd\tw" 1 } } */
+/* { dg-final { scan-assembler-times "\tldxr\tw" 1 } } */
+/* { dg-final { scan-assembler-times "bl\t__aarch64_ldadd4_relax" 1 } } */
new file mode 100644
@@ -0,0 +1,14 @@
+/* { dg-do compile { target aarch64*-*-linux-musl* } } */
+/* { dg-options "-march=armv8-a+nolse -O2" } */
+
+/* A compiler configured for musl has no flag initializer. */
+
+int
+f (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "bl\t__aarch64_ldadd4_relax" } } */
+/* { dg-final { scan-assembler-not "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-assembler-not "\tldadd\tw" } } */
new file mode 100644
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-options "-march=armv8-a+lse -Os" } */
+
+/* Compile-time LSE support takes precedence over the dispatch default. */
+
+int
+f (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "\tldadd\tw" } } */
+/* { dg-final { scan-assembler-not "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-assembler-not "\tldxr\tw" } } */
+/* { dg-final { scan-assembler-not "bl\t__aarch64_" } } */
+/* { dg-final { scan-assembler-not "\.arch_extension" } } */
new file mode 100644
@@ -0,0 +1,16 @@
+/* { dg-do compile { target aarch64*-*-linux* } } */
+/* { dg-require-effective-target glibc } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-options "-march=armv8-a+nolse -O2" } */
+
+/* A cold function uses a helper call by default. */
+
+int __attribute__((cold))
+f (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+/* { dg-final { scan-assembler "bl\t__aarch64_ldadd4_relax" } } */
+/* { dg-final { scan-assembler-not "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-assembler-not "\tldadd\tw" } } */
new file mode 100644
@@ -0,0 +1,30 @@
+/* { dg-do link { target aarch64*-*-linux* } } */
+/* { dg-require-effective-target glibc } */
+/* { dg-require-effective-target aarch64_asm_lse_nolse_ok } */
+/* { dg-require-effective-target lto } */
+/* { dg-options "-march=armv8-a+nolse -O2 -flto -flto-partition=one --save-temps" } */
+
+int __attribute__((noipa))
+speed (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+int __attribute__((noipa, optimize ("Os")))
+size (int *p)
+{
+ return __atomic_fetch_add (p, 1, __ATOMIC_RELAXED);
+}
+
+int
+main (void)
+{
+ int x = 0;
+ return speed (&x) + size (&x);
+}
+
+/* The default policy and the per-function optimization mode survive LTO. */
+/* { dg-final { scan-lto-assembler "__aarch64_have_lse_atomics" } } */
+/* { dg-final { scan-lto-assembler "\tldadd\tw" } } */
+/* { dg-final { scan-lto-assembler "\tldxr\tw" } } */
+/* { dg-final { scan-lto-assembler "bl\t__aarch64_ldadd4_relax" } } */