@@ -1448,6 +1448,12 @@
#endif
+/* Define if your assembler accepts the flex argument to .bundle_lock. */
+#ifndef USED_FOR_TARGET
+#undef HAVE_GAS_BUNDLE_FLEX_LOCK
+#endif
+
+
/* Define 0/1 if your assembler supports CFI directives. */
#undef HAVE_GAS_CFI_DIRECTIVE
@@ -83,6 +83,7 @@ along with GCC; see the file COPYING3. If not see
#include "ifcvt.h"
#include "symbol-summary.h"
#include "sreal.h"
+#include "fibonacci_heap.h"
#include "ipa-cp.h"
#include "ipa-prop.h"
#include "ipa-fnsummary.h"
@@ -5646,12 +5647,314 @@ ix86_align_loops ()
free_dominance_info (CDI_DOMINATORS);
}
+/* ix86_bundle_align_loops pass.
+
+ 1. Compute layout offsets and identify feasible loop-backward edges.
+ 2. Add synthetic unions for candidates sharing a K-byte window.
+ 3. Weight candidates by the back-edge counts of loops contained in
+ them.
+ 4. Select a maximum-weight disjoint set with weighted interval
+ scheduling.
+ 5. Emit .bundle_lock flex for the selected regions.
+
+ K is param_x86_tight_loop_max_size, or ix86_cost->prefetch_block when
+ unset. */
+
+/* A candidate loop region: the byte interval [lo, hi] on the layout axis
+ (hi inclusive), weight w, interval-scheduling predecessor p, the lock and
+ unlock blocks, and a feasibility flag. A synthetic union has w == -1 until
+ the weight sweep replaces it with its contained weight. */
+
+struct bundle_cand
+{
+ class loop *loop;
+ basic_block lock, unlock;
+ HOST_WIDE_INT lo, hi;
+ gcov_type w;
+ int p;
+ bool feasible;
+};
+
+/* Sort by hi ascending. Break ties by lo descending (a contained candidate
+ before its container), then by real loops before a synthetic union of the
+ same span. */
+
+static int
+bundle_cand_cmp_hi (const void *a, const void *b)
+{
+ const bundle_cand *x = (const bundle_cand *)a, *y = (const bundle_cand *)b;
+ if (x->hi != y->hi)
+ return x->hi < y->hi ? -1 : 1;
+ if (x->lo != y->lo)
+ return x->lo > y->lo ? -1 : 1;
+ if ((x->w == -1) != (y->w == -1))
+ return x->w == -1 ? 1 : -1;
+ return 0;
+}
+
+/* qsort comparator for HOST_WIDE_INT, ascending (lo-value compression). */
+
+static int
+bundle_hwi_cmp (const void *a, const void *b)
+{
+ HOST_WIDE_INT x = *(const HOST_WIDE_INT *)a, y = *(const HOST_WIDE_INT *)b;
+ return x < y ? -1 : x > y ? 1 : 0;
+}
+
+/* Less-than on HOST_WIDE_INT, for vec::lower_bound over the compressed
+ lo-value list. */
+
+static bool
+bundle_hwi_less (const HOST_WIDE_INT &a, const HOST_WIDE_INT &b)
+{
+ return a < b;
+}
+
+/* Less-than on interval end, for vec::lower_bound in the WIS predecessor
+ search over the hi-sorted feasible set. */
+
+static bool
+bundle_cand_hi_less (const bundle_cand &a, const bundle_cand &b)
+{
+ return a.hi < b.hi;
+}
+
+static void
+bundle_compute_layout (HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
+ int *asm_lo, int *asm_hi)
+{
+ HOST_WIDE_INT off = 0;
+ int nasm = 0;
+ basic_block bb;
+ FOR_EACH_BB_FN (bb, cfun)
+ {
+ rtx_insn *insn;
+ bb_lo[bb->index] = off;
+ asm_lo[bb->index] = nasm;
+ FOR_BB_INSNS (bb, insn)
+ if (NONDEBUG_INSN_P (insn))
+ {
+ /* Inline asm has no reliable min size; count its presence so any
+ candidate spanning this block can be excluded outright. */
+ if (asm_noperands (PATTERN (insn)) >= 0)
+ nasm++;
+ else
+ off += ix86_min_insn_size (insn);
+ }
+ bb_hi[bb->index] = off;
+ asm_hi[bb->index] = nasm;
+ }
+}
+
+static void
+bundle_collect_candidates (vec<bundle_cand> &feas, HOST_WIDE_INT k,
+ HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
+ int *asm_lo, int *asm_hi)
+{
+ /* Step 1: identify feasible candidates from layout-backward loop edges. */
+ basic_block bb;
+ FOR_EACH_BB_FN (bb, cfun)
+ {
+ edge e;
+ edge_iterator ei;
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ if (e->count ().initialized_p ()
+ && e->dest->index >= NUM_FIXED_BLOCKS
+ && (e->dest == bb
+ || bb_lo[e->dest->index] < bb_lo[bb->index]))
+ {
+ bundle_cand c = {};
+ c.loop = bb->loop_father;
+ c.lock = e->dest;
+ c.unlock = bb;
+ c.lo = bb_lo[e->dest->index];
+ c.hi = bb_hi[bb->index];
+ c.w = e->count ().to_gcov_type ();
+ /* Both ends must lie in a real loop, not the function-wide fake
+ loop whose latch is EXIT. This rejects a layout-backward exit
+ edge whose destination is outside any loop. */
+ c.feasible = (c.loop
+ && c.loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
+ && e->dest->loop_father
+ && (e->dest->loop_father->latch
+ != EXIT_BLOCK_PTR_FOR_FN (cfun))
+ && asm_hi[bb->index] == asm_lo[e->dest->index]
+ && c.hi - c.lo <= k
+ && optimize_bb_for_speed_p (c.lock));
+ if (c.feasible)
+ feas.safe_push (c);
+ }
+ }
+}
+
+static void
+bundle_add_unions (vec<bundle_cand> &feas, HOST_WIDE_INT k)
+{
+ /* Step 2: add synthetic unions for candidates sharing a K-byte window. */
+ feas.qsort (bundle_cand_cmp_hi);
+ unsigned base = feas.length ();
+ /* At most one union is added per candidate, so reserve now; this keeps the
+ &feas[i] pointers held by the heap valid across the push below. */
+ feas.reserve (2 * base);
+ fibonacci_heap<HOST_WIDE_INT, bundle_cand> heap (HOST_WIDE_INT_MIN);
+ for (unsigned i = 0; i < base; i++)
+ {
+ heap.insert (feas[i].lo, &feas[i]);
+ HOST_WIDE_INT thr = feas[i].hi - k;
+ while (heap.min_key () < thr)
+ heap.extract_min ();
+ if (heap.nodes () >= 2)
+ {
+ bundle_cand *s = heap.min ();
+ bundle_cand c = {};
+ c.loop = s->loop;
+ c.lock = s->lock;
+ c.unlock = feas[i].unlock;
+ c.lo = s->lo;
+ c.hi = feas[i].hi;
+ c.w = -1; /* Union; the weight sweep fills this in. */
+ c.feasible = true;
+ feas.safe_push (c);
+ }
+ }
+}
+
+static void
+bundle_compute_weights (vec<bundle_cand> &feas)
+{
+ /* Step 3: compute containment weights with a Fenwick tree. */
+ feas.qsort (bundle_cand_cmp_hi);
+ unsigned n_all = feas.length ();
+ /* Compress lo values into a sorted, unique list; POS below maps a lo to its
+ reversed rank via lower_bound. */
+ auto_vec<HOST_WIDE_INT> los;
+ los.reserve (n_all);
+ for (unsigned p = 0; p < n_all; p++)
+ los.quick_push (feas[p].lo);
+ los.qsort (bundle_hwi_cmp);
+ int m = 0;
+ for (unsigned p = 0; p < n_all; p++)
+ if (m == 0 || los[m - 1] != los[p])
+ los[m++] = los[p];
+ los.truncate (m);
+ /* BIT[1..m] keyed by reversed lo-rank, so a prefix sum [1..pos] covers
+ exactly lo_j >= lo_i. */
+ gcov_type *bit = XCNEWVEC (gcov_type, m + 1);
+ for (unsigned p = 0; p < n_all; p++)
+ {
+ /* Reversed rank of feas[p].lo: LOWER_BOUND is the first LOS >= lo. */
+ int pos = m - (int) los.lower_bound (feas[p].lo, bundle_hwi_less);
+ /* A union is not a real loop, so do not insert it into the tree. Its
+ weight is the contained sum, less one to break ties toward it. */
+ bool u = feas[p].w == -1;
+ if (!u)
+ for (int i = pos; i <= m; i += i & -i)
+ bit[i] += feas[p].w;
+ gcov_type sum = 0;
+ for (int i = pos; i > 0; i -= i & -i)
+ sum += bit[i];
+ feas[p].w = u ? sum - 1 : sum;
+ }
+ XDELETEVEC (bit);
+}
+
+static void
+bundle_select_and_emit (vec<bundle_cand> &feas)
+{
+ /* Step 4: select the maximum-weight disjoint set. */
+ unsigned n = feas.length ();
+ /* dp[i+1] is the best weight over feas[0..i]. Whether i is taken is
+ recomputed while backtracking rather than stored. */
+ gcov_type *dp = XCNEWVEC (gcov_type, n + 1);
+
+ for (unsigned i = 0; i < n; i++)
+ {
+ /* Predecessor: the last candidate that ends before feas[i] starts.
+ hi is inclusive, so find the first candidate with hi >= lo. */
+ bundle_cand key = {};
+ key.hi = feas[i].lo;
+ int p = (int) feas.lower_bound (key, bundle_cand_hi_less) - 1;
+ feas[i].p = p;
+ gcov_type take = feas[i].w + dp[p + 1], skip = dp[i];
+ dp[i + 1] = take >= skip ? take : skip;
+ if (dump_file)
+ fprintf (dump_file,
+ "bundle-align: cand %u loop %d [bb%d..bb%d] "
+ "lo=%ld hi=%ld w=%ld p=%d take=%ld skip=%ld => %s\n",
+ i, feas[i].loop->num, feas[i].lock->index,
+ feas[i].unlock->index, (long) feas[i].lo,
+ (long) feas[i].hi, (long) feas[i].w, p,
+ (long) take, (long) skip, take >= skip ? "take" : "skip");
+ }
+
+ /* Step 5: emit the selected regions. */
+ for (int i = (int) n - 1; i >= 0;)
+ if (feas[i].w + dp[feas[i].p + 1] >= dp[i])
+ {
+ basic_block h = feas[i].lock, l = feas[i].unlock;
+ emit_insn_before (gen_bundle_lock (), BB_HEAD (h));
+ emit_insn_after (gen_bundle_unlock (), BB_END (l));
+ if (dump_file)
+ fprintf (dump_file,
+ "bundle-align: wrap loop %d [bb%d..bb%d] "
+ "w=%ld span=%ld\n", feas[i].loop->num, h->index,
+ l->index, (long) feas[i].w,
+ (long) (feas[i].hi - feas[i].lo));
+ i = feas[i].p;
+ }
+ else
+ i--;
+
+ XDELETEVEC (dp);
+}
+
+/* Wrap the selected disjoint hot loop regions in .bundle_lock/.bundle_unlock.
+ Each region is an interval on the layout byte axis, which handles rotated
+ (latch-before-header) loops uniformly. */
+
+static void
+ix86_do_bundle_align_loops ()
+{
+ if (ix86_cost->prefetch_block == 0)
+ return;
+
+ /* Region size cap: tuning param if set, else the CPU bundle size. */
+ HOST_WIDE_INT k = (param_x86_tight_loop_max_size > 0
+ ? (HOST_WIDE_INT)param_x86_tight_loop_max_size
+ : (HOST_WIDE_INT)ix86_cost->prefetch_block);
+
+ loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
+
+ /* Per-block layout byte offsets [bb_lo, bb_hi). */
+ int nbb = last_basic_block_for_fn (cfun);
+ HOST_WIDE_INT *bb_lo = XCNEWVEC (HOST_WIDE_INT, nbb);
+ HOST_WIDE_INT *bb_hi = XCNEWVEC (HOST_WIDE_INT, nbb);
+ /* Running count of inline-asm insns in layout order; [asm_lo, asm_hi) lets a
+ candidate test whether any block it spans contains an asm. */
+ int *asm_lo = XCNEWVEC (int, nbb);
+ int *asm_hi = XCNEWVEC (int, nbb);
+ bundle_compute_layout (bb_lo, bb_hi, asm_lo, asm_hi);
+
+ auto_vec<bundle_cand> feas;
+ bundle_collect_candidates (feas, k, bb_lo, bb_hi, asm_lo, asm_hi);
+ bundle_add_unions (feas, k);
+ bundle_compute_weights (feas);
+ if (!feas.is_empty ())
+ bundle_select_and_emit (feas);
+
+ XDELETEVEC (bb_lo);
+ XDELETEVEC (bb_hi);
+ XDELETEVEC (asm_lo);
+ XDELETEVEC (asm_hi);
+ loop_optimizer_finalize ();
+}
+
namespace {
-const pass_data pass_data_align_tight_loops =
+const pass_data pass_data_bundle_align_loops =
{
RTL_PASS, /* type */
- "align_tight_loops", /* name */
+ "bundle_align_loops", /* name */
OPTGROUP_NONE, /* optinfo_flags */
TV_MACH_DEP, /* tv_id */
0, /* properties_required */
@@ -5661,40 +5964,44 @@ const pass_data pass_data_align_tight_loops =
0, /* todo_flags_finish */
};
-class pass_align_tight_loops : public rtl_opt_pass
+class pass_bundle_align_loops : public rtl_opt_pass
{
public:
- pass_align_tight_loops (gcc::context *ctxt)
- : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
+ pass_bundle_align_loops (gcc::context *ctxt)
+ : rtl_opt_pass (pass_data_bundle_align_loops, ctxt)
{}
/* opt_pass methods: */
bool gate (function *) final override
{
- return TARGET_ALIGN_TIGHT_LOOPS
- && optimize
- && optimize_function_for_speed_p (cfun);
+ return optimize
+ && optimize_function_for_speed_p (cfun)
+ && (ix86_bundle_align_loops > 0 || TARGET_ALIGN_TIGHT_LOOPS);
}
unsigned int execute (function *) final override
{
timevar_push (TV_MACH_DEP);
+ if (ix86_bundle_align_loops > 0)
+ ix86_do_bundle_align_loops ();
#ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
- ix86_align_loops ();
+ else
+ ix86_align_loops ();
#endif
timevar_pop (TV_MACH_DEP);
return 0;
}
-}; // class pass_align_tight_loops
+}; // class pass_bundle_align_loops
} // anon namespace
rtl_opt_pass *
-make_pass_align_tight_loops (gcc::context *ctxt)
+make_pass_bundle_align_loops (gcc::context *ctxt)
{
- return new pass_align_tight_loops (ctxt);
+ return new pass_bundle_align_loops (ctxt);
}
+
/* This compares the priority of target features in function DECL1
and DECL2. It returns positive value if DECL1 is higher priority,
negative value if DECL2 is higher priority and 0 if they are the
@@ -1872,8 +1872,16 @@ set_ix86_tune_features (struct gcc_options *opts,
/* Default align_* from the processor table. */
static void
-ix86_default_align (struct gcc_options *opts)
+ix86_default_align (struct gcc_options *opts,
+ struct gcc_options *opts_set)
{
+ /* Bundle alignment replaces generic loop, jump, and label padding. */
+ if (opts->x_ix86_bundle_align_loops > 0)
+ {
+ SET_OPTION_IF_UNSET (opts, opts_set, str_align_loops, "1");
+ SET_OPTION_IF_UNSET (opts, opts_set, str_align_jumps, "1");
+ SET_OPTION_IF_UNSET (opts, opts_set, str_align_labels, "1");
+ }
/* -falign-foo without argument: supply one. */
if (opts->x_flag_align_loops && !opts->x_str_align_loops)
opts->x_str_align_loops = processor_cost_table[ix86_tune]->align_loop;
@@ -1885,6 +1893,16 @@ ix86_default_align (struct gcc_options *opts)
opts->x_str_align_functions = processor_cost_table[ix86_tune]->align_func;
}
+/* Default -mbundle-align-loops from the active tune model. */
+
+static void
+ix86_default_bundle_align (struct gcc_options *opts)
+{
+ if (opts->x_ix86_bundle_align_loops == -1)
+ opts->x_ix86_bundle_align_loops
+ = ix86_tune_features[X86_TUNE_BUNDLE_ALIGN_LOOPS] ? 1 : 0;
+}
+
#ifndef USE_IX86_FRAME_POINTER
#define USE_IX86_FRAME_POINTER 0
#endif
@@ -1986,7 +2004,8 @@ ix86_override_options_after_change_1 (struct gcc_options *opts,
void
ix86_override_options_after_change (void)
{
- ix86_default_align (&global_options);
+ ix86_default_bundle_align (&global_options);
+ ix86_default_align (&global_options, &global_options_set);
ix86_recompute_optlev_based_flags (&global_options, &global_options_set);
@@ -2619,7 +2638,8 @@ ix86_option_override_internal (bool main_args_p,
opts->x_ix86_regparm = REGPARM_MAX;
/* Default align_* from the processor table. */
- ix86_default_align (&global_options);
+ ix86_default_bundle_align (&global_options);
+ ix86_default_align (&global_options, &global_options_set);
/* Provide default for -mbranch-cost= value. */
SET_OPTION_IF_UNSET (opts, opts_set, ix86_branch_cost,
@@ -31,9 +31,9 @@ along with GCC; see the file COPYING3. If not see
INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */);
INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_insert_endbr_and_patchable_area);
- /* pass_align_tight_loops must be after pass_insert_endbr_and_patchable_area.
- PR116174. */
- INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_align_tight_loops);
+ /* Tight/bundle loop aligner; after pass_insert_endbr_and_patchable_area
+ (PR116174) and before pass_shorten_branches. */
+ INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_bundle_align_loops);
INSERT_PASS_AFTER (pass_late_combine, 1, pass_x86_cse);
INSERT_PASS_AFTER (pass_late_combine, 1, pass_remove_partial_avx_dependency);
@@ -434,7 +434,7 @@ extern rtl_opt_pass *make_pass_remove_partial_avx_dependency
(gcc::context *);
extern rtl_opt_pass *make_pass_x86_cse (gcc::context *);
extern rtl_opt_pass *make_pass_apx_nf_convert (gcc::context *);
-extern rtl_opt_pass *make_pass_align_tight_loops (gcc::context *);
+extern rtl_opt_pass *make_pass_bundle_align_loops (gcc::context *);
extern bool ix86_has_no_direct_extern_access;
extern bool ix86_rpad_gate ();
@@ -233,6 +233,8 @@
UNSPECV_STACK_PROBE
UNSPECV_PROBE_STACK_RANGE
UNSPECV_ALIGN
+ UNSPECV_BUNDLE_LOCK
+ UNSPECV_BUNDLE_UNLOCK
UNSPECV_PROLOGUE_USE
UNSPECV_SPLIT_STACK_RETURN
UNSPECV_CLD
@@ -21756,6 +21758,39 @@
}
[(set_attr "length" "16")])
+;; ix86_bundle_align_loops pass: align the locked region's start to the cache
+;; line (prefetch_block). Final instruction sizes may grow during assembly
+;; because of indirect calls, prefixes, and relaxation; once the body exceeds
+;; one bundle, padding cannot prevent a straddle, so flex mode skips padding and
+;; keeps the case non-fatal. Requires flex .bundle_lock support.
+(define_insn "bundle_lock"
+ [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_LOCK)]
+ ""
+{
+#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
+ fprintf (asm_out_file, "\t.bundle_align_mode %d\n\t.bundle_lock flex\n",
+ ceil_log2 (ix86_cost->prefetch_block));
+#endif
+ return "";
+}
+ [(set_attr "length" "0")
+ (set_attr "length_immediate" "0")
+ (set_attr "modrm" "0")])
+
+;; Close the bundle and turn bundle alignment mode back off.
+(define_insn "bundle_unlock"
+ [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_UNLOCK)]
+ ""
+{
+#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
+ fputs ("\t.bundle_unlock\n\t.bundle_align_mode 0\n", asm_out_file);
+#endif
+ return "";
+}
+ [(set_attr "length" "0")
+ (set_attr "length_immediate" "0")
+ (set_attr "modrm" "0")])
+
(define_expand "prologue"
[(const_int 0)]
""
@@ -240,6 +240,10 @@ malign-loops=
Target Undocumented RejectNegative Joined UInteger
Loop code aligned to this power of 2.
+mbundle-align-loops
+Target Var(ix86_bundle_align_loops) Init(-1) Save
+Wrap small hot loops with .bundle_lock so the assembler keeps them inside one cache line.
+
malign-stringops
Target RejectNegative InverseMask(NO_ALIGN_STRINGOPS, ALIGN_STRINGOPS) Save
Align destination of the string operations.
@@ -1221,6 +1225,14 @@ Do not use GOT to access external symbols.
Target Joined UInteger Var(x86_stlf_window_ninsns) Init(64) Param
Instructions number above which STFL stall penalty can be compensated.
+-param=x86-tight-loop-max-size=
+Target Joined UInteger Var(param_x86_tight_loop_max_size) Init(0) Param
+Maximum body size in bytes for the i386 tight-loop aligner; 0 picks the active tune prefetch_block.
+
+-param=x86-bundle-align-branch-limit=
+Target Joined UInteger Var(param_x86_bundle_align_branch_limit) Init(2) Param
+Maximum internal conditional branches allowed in a bundle-locked loop region (the back-edge branch is not counted).
+
mavxifma
Target Mask(ISA2_AVXIFMA) Var(ix86_isa_flags2) Save
Support MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, AVX, AVX2, and
@@ -28,6 +28,9 @@ UrlSuffix(gcc/x86-Options.html#index-malign-double)
; skipping UrlSuffix for 'malign-loops=' due to finding no URLs
+mbundle-align-loops
+UrlSuffix(gcc/x86-Options.html#index-mbundle-align-loops)
+
malign-stringops
UrlSuffix(gcc/x86-Options.html#index-malign-stringops)
@@ -259,6 +259,13 @@ DEF_TUNE (X86_TUNE_ALIGN_TIGHT_LOOPS, "align_tight_loops",
~(m_ZHAOXIN | m_CASCADELAKE | m_SKYLAKE_AVX512
| m_CORE_ATOM))
+/* X86_TUNE_BUNDLE_ALIGN_LOOPS: when true, use the ix86_bundle_align_loops
+ pass with the assembler's .bundle_lock/.bundle_unlock directives instead
+ of static .p2align padding. Targets cores where loops straddling a 64-byte
+ instruction-cache line are particularly costly (AMD Zen4/Zen5/Zen6). */
+DEF_TUNE (X86_TUNE_BUNDLE_ALIGN_LOOPS, "bundle_align_loops",
+ m_ZNVER4 | m_ZNVER5 | m_ZNVER6)
+
/*****************************************************************************/
/* Integer instruction selection tuning */
/*****************************************************************************/
@@ -26306,6 +26306,44 @@ $as_echo "#define HAVE_GAS_MAX_SKIP_P2ALIGN 1" >>confdefs.h
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .bundle_lock flex argument" >&5
+$as_echo_n "checking assembler for .bundle_lock flex argument... " >&6; }
+if ${gcc_cv_as_bundle_flex_lock+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ gcc_cv_as_bundle_flex_lock=no
+ if test x"$gcc_cv_as" != x; then
+ $as_echo '.bundle_align_mode 6
+.bundle_lock flex
+.rept 100
+nop
+.endr
+.bundle_unlock' > conftest.s
+ if { ac_try='$gcc_cv_as $gcc_cv_as_flags -o conftest.o conftest.s >&5'
+ { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5
+ (eval $ac_try) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; }
+ then
+ gcc_cv_as_bundle_flex_lock=yes
+ else
+ echo "configure: failed program was" >&5
+ cat conftest.s >&5
+ fi
+ rm -f conftest.o conftest.s
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_as_bundle_flex_lock" >&5
+$as_echo "$gcc_cv_as_bundle_flex_lock" >&6; }
+if test $gcc_cv_as_bundle_flex_lock = yes; then
+
+$as_echo "#define HAVE_GAS_BUNDLE_FLEX_LOCK 1" >>confdefs.h
+
+fi
+
+
+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .literal16" >&5
$as_echo_n "checking assembler for .literal16... " >&6; }
if ${gcc_cv_as_literal16+:} false; then :
@@ -3142,6 +3142,17 @@ gcc_GAS_CHECK_FEATURE([.p2align with maximum skip], gcc_cv_as_max_skip_p2align,,
[Define if your assembler supports specifying the maximum number
of bytes to skip when using the GAS .p2align command.])])
+gcc_GAS_CHECK_FEATURE([.bundle_lock flex argument], gcc_cv_as_bundle_flex_lock,,
+ [.bundle_align_mode 6
+.bundle_lock flex
+.rept 100
+nop
+.endr
+.bundle_unlock],,
+[AC_DEFINE(HAVE_GAS_BUNDLE_FLEX_LOCK, 1,
+ [Define if your assembler accepts the flex argument to .bundle_lock
+ that permits bundles larger than .bundle_align_mode alignment.])])
+
gcc_GAS_CHECK_FEATURE([.literal16], gcc_cv_as_literal16,,
[.literal16],,
[AC_DEFINE(HAVE_GAS_LITERAL16, 1,
@@ -9598,6 +9598,12 @@ operations and calling the library routines for large operations.
@itemx no-align-stringops
Do/do not align destination of inlined string operations.
+@atindex @code{target("bundle-align-loops")}, x86
+@item bundle-align-loops
+@itemx no-bundle-align-loops
+Enable/disable wrapping of small hot loops in assembler
+@code{.bundle_lock}/@code{.bundle_unlock} directives.
+
@atindex @code{target("recip")}, x86
@item recip
@itemx no-recip
@@ -1554,7 +1554,8 @@ See RS/6000 and PowerPC Options.
-mamx-complex -mavxvnniint16 -msm3 -msha512 -msm4 -mapxf
-musermsr -mavx10.1 -mavx10.2 -mamx-avx512 -mmovrs -mamx-movrs
-mavx512bmm -mavx10v2aux
--mcldemote -mms-bitfields -mno-align-stringops -minline-all-stringops
+-mcldemote -mms-bitfields -mbundle-align-loops -mno-align-stringops
+-minline-all-stringops
-minline-stringops-dynamically -mstringop-strategy=@var{alg}
-mkl -mwidekl
-mmemcpy-strategy=@var{strategy} -mmemset-strategy=@var{strategy}
@@ -37155,6 +37156,18 @@ Here, @code{t5} takes up 2 bytes.
@end enumerate
+@opindex mbundle-align-loops
+@opindex mno-bundle-align-loops
+@item -mbundle-align-loops
+@itemx -mno-bundle-align-loops
+Wrap small hot loops in assembler @code{.bundle_lock}/@code{.bundle_unlock}
+directives so the assembler keeps each loop within a single instruction-cache
+line, aligning the loop header without inserting executed padding. Enabled by
+default for AMD Zen4 and later. It defaults @option{-falign-loops},
+@option{-falign-jumps}, and @option{-falign-labels} to 1 unless explicitly set.
+The maximum loop-body size considered is set by the
+@option{x86-tight-loop-max-size} parameter.
+
@opindex mno-align-stringops
@opindex malign-stringops
@item -mno-align-stringops
@@ -3114,6 +3114,10 @@ Target supports @option{-fstack-protector}.
@item gas
Target uses GNU @command{as}.
+@item gcc_bundle_align_loops
+Target compiler and assembler support @option{-mbundle-align-loops} and
+@code{.bundle_lock flex}.
+
@item gc_sections
Target supports @option{--gc-sections}.
new file mode 100644
@@ -0,0 +1,30 @@
+/* Validator case: slide08 indirect-call loop. */
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
+
+#include <stdio.h>
+#include <stdlib.h>
+typedef int (*callee_t) (int, int *);
+extern int f (int, int *);
+
+int
+main (void)
+{
+ const int n = 37;
+ const long calls = 100000000;
+ int *array = aligned_alloc (64, 1024 * sizeof *array);
+ if (!array)
+ return 2;
+ for (int i = 0; i < 1024; ++i)
+ array[i] = i & 7;
+ volatile callee_t callee = f;
+ volatile int sink = 0;
+ for (long call = 0; call < calls; ++call)
+ sink += callee (n, array);
+ printf ("%d\n", sink);
+ free (array);
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,34 @@
+/* Validator case: slide12 irregular loop. */
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
+
+#include <stdio.h>
+#include <stdlib.h>
+static int a[1024];
+__attribute__((noinline,noipa))
+int f (int n, int *p)
+{
+ int i = 0, s = 0;
+ if (n & 1)
+ goto mid;
+ while (i < n)
+ {
+ s += p[i & 1023];
+ mid:
+ i++;
+ }
+ return s;
+}
+int main (int argc, char **argv)
+{
+ int n = argc > 1 ? atoi (argv[1]) : 1024;
+ long reps = argc > 2 ? atol (argv[2]) : 20000000;
+ for (int i = 0; i < 1024; i++) a[i] = i;
+ long sum = 0;
+ for (long i = 0; i < reps; i++) { int nn = n; asm volatile ("" : "+r" (nn)); sum += f (nn, a); }
+ printf ("%ld\n", sum);
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,27 @@
+/* Validator case: slide14 direct edges to an outer header are wrapped. */
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
+
+int a[1024];
+__attribute__((hot,noinline,noipa))
+int
+two_backedges (int n, int m)
+{
+ int i = 0, s = 0;
+outer_head:
+ {
+ int j = 0;
+ inner_head:
+ s += a[(i + j) & 1023];
+ if (__builtin_expect (s < 0, 0))
+ goto outer_head;
+ if (++j < m)
+ goto inner_head;
+ }
+ if (++i < n)
+ goto outer_head;
+ return s;
+}
new file mode 100644
@@ -0,0 +1,23 @@
+/* Validator case: slide17 loop with multiple conditional branches is wrapped. */
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver5 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
+/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
+
+volatile int sink;
+int a[1024];
+int
+classify (int n)
+{
+ int s = 0;
+ asm volatile (".rept 0\n\t.byte 0x90\n\t.endr" ::: "memory");
+ for (int i = 0; i < n; i++)
+ {
+ if (a[i] & 1) sink = i;
+ if (a[i] & 2) sink = i + 1;
+ if (a[i] & 4) s += 4;
+ s += a[i];
+ }
+ return s;
+}
new file mode 100644
@@ -0,0 +1,19 @@
+/* Validator case: slide19 rotated bottom-tested loop. */
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops -Dmemchr=mc" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
+
+/* Rotated bottom-tested scanning loop with latch above header. */
+void *memchr (const void *s, int c, unsigned long n)
+{
+ const unsigned char *p = s;
+ while (n--)
+ {
+ if (*p == (unsigned char) c)
+ return (void *) p;
+ p++;
+ }
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,25 @@
+/* Validator case: slide20 overlapping back edges are clubbed into one wrap. */
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 --param=x86-tight-loop-max-size=64 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
+/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
+
+struct O { long rc; };
+extern void dealloc (struct O *);
+void
+f (struct O **v, long n)
+{
+ asm volatile (".rept 10\n\t.byte 0x90\n\t.endr" ::: "memory");
+ while (--n >= 0)
+ {
+ struct O *x = v[n];
+ if (x)
+ {
+ if (x->rc < 0)
+ ;
+ else if (--x->rc == 0)
+ dealloc (x);
+ }
+ }
+}
new file mode 100644
@@ -0,0 +1,55 @@
+/* Validator case: slide23 nested loop boundary check. */
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O1 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N (1 << 20)
+
+static int A[N];
+
+int
+main (int argc, char **argv)
+{
+ if (argc != 4)
+ return 2;
+ int n = atoi (argv[1]);
+ int m = atoi (argv[2]);
+ int p = atoi (argv[3]);
+ if (n <= 0 || m <= 0 || p <= 0 || (long long) n * m > N)
+ return 2;
+
+ volatile int sink = 0;
+ for (int i = 0; i < n * m; ++i)
+ A[i] = i & 7;
+
+ unsigned start_lo, start_hi, end_lo, end_hi;
+ __asm__ volatile ("lfence\n\trdtsc\n\tlfence"
+ : "=a" (start_lo), "=d" (start_hi));
+ for (int repeat = 0; repeat < p; ++repeat)
+ {
+ int sum = 0;
+ __asm__ volatile ("nop\n\tnop\n\tnop\n\tnop\n\tnop\n\tnop\n\tnop\n\tnop");
+ for (int i = 0; i < n; ++i)
+ for (int j = 0; j < m; ++j)
+ sum += A[i * m + j];
+ sink += sum;
+ }
+ __asm__ volatile ("lfence\n\trdtsc\n\tlfence"
+ : "=a" (end_lo), "=d" (end_hi));
+ uint64_t start = ((uint64_t) start_hi << 32) | start_lo;
+ uint64_t end = ((uint64_t) end_hi << 32) | end_lo;
+ uint64_t elapsed = end - start;
+
+ if (sink == -1)
+ puts ("impossible");
+ printf ("n=%d m=%d p=%d cycles=%llu cycles_per_pass=%llu\n",
+ n, m, p, (unsigned long long) elapsed,
+ (unsigned long long) (elapsed / p));
+ return 0;
+}
@@ -12491,6 +12491,22 @@ proc check_effective_target_gas { } {
return $use_gas_saved
}
+# Return 1 if the compiler was configured with assembler support for
+# the flex .bundle_lock argument used by -mbundle-align-loops.
+
+proc check_effective_target_gcc_bundle_align_loops { } {
+ return [check_no_messages_and_pattern gcc_bundle_align_loops \
+ "\\.bundle_lock flex" assembly {
+ int a[1024];
+ int foo (int n) {
+ int s = 0;
+ for (int i = 0; i < n; i++)
+ s += a[i];
+ return s;
+ }
+ } "-O2 -march=znver4 -mbundle-align-loops -falign-loops=1"]
+}
+
# Return 1 if GNU ld is used.
proc check_effective_target_gld { } {