[RFC,v2] i386: add bundle alignment pass
Commit Message
Add ix86_bundle_align_loops, an RTL pass that marks selected loop regions
with .bundle_lock flex. The assembler aligns each marked region after
instruction relaxation, when the final instruction sizes are known, and
flex mode accepts a region that ends up larger than one bundle.
Candidates are formed from layout-backward loop edges and selected as a
disjoint set by weighted interval scheduling.
Enable the pass for Zen4 and later, add the target options and assembler
feature check, and add tests for candidate formation, union construction,
weight calculation, selection, and representative loop cases.
gcc/
* config/i386/i386-features.cc (struct bundle_cand): New.
(bundle_cand_cmp_hi, bundle_hwi_cmp, bundle_hwi_less)
(bundle_cand_hi_less, bundle_compute_layout)
(bundle_collect_candidates, bundle_add_unions)
(bundle_compute_weights, bundle_select_and_emit)
(ix86_do_bundle_align_loops): New functions.
(class pass_bundle_align_loops): New.
(make_pass_bundle_align_loops): New function.
* config/i386/i386-protos.h (make_pass_bundle_align_loops): Declare.
* config/i386/i386-passes.def (pass_bundle_align_loops): Insert before
pass_shorten_branches.
* config/i386/i386-options.cc (ix86_default_bundle_align): New function.
(ix86_default_align): Call it.
* config/i386/i386.md (bundle_lock, bundle_unlock): New insns.
* config/i386/i386.opt (mbundle-align-loops): New option.
(x86-tight-loop-max-size, x86-bundle-align-branch-limit): New params.
* config/i386/i386.opt.urls: Regenerate.
* config/i386/x86-tune.def (X86_TUNE_BUNDLE_ALIGN_LOOPS): New tune flag.
* configure.ac: Check for .bundle_lock flex support.
* configure: Regenerate.
* config.in: Regenerate.
* doc/invoke.texi (-mbundle-align-loops, x86-tight-loop-max-size):
Document.
* doc/extend.texi (x86 Function Attributes): Document
bundle-align-loops.
* doc/sourcebuild.texi (gcc_bundle_align_loops): Document.
gcc/testsuite/
* gcc.target/i386/bundle-align-1.c: New test.
* gcc.target/i386/bundle-align-2.c: New test.
* gcc.target/i386/bundle-align-3.c: New test.
* gcc.target/i386/bundle-align-4.c: New test.
* gcc.target/i386/bundle-align-5.c: New test.
* gcc.target/i386/bundle-align-6.c: New test.
* gcc.target/i386/bundle-align-7.c: New test.
* lib/target-supports.exp
(check_effective_target_gcc_bundle_align_loops): New.
Co-authored-by: Branden Moore <Branden.Moore@amd.com>
Co-authored-by: Venkataramanan Kumar <vekumar@gcc.gnu.org>
Signed-off-by: Sarvesh Chandra <Sarvesh.Chandra@amd.com>
---
[Presented at GNU Tools Cauldron 2026.]
Cosmetic changes in v2:
- Add co-authors Branden Moore and Venkataramanan Kumar.
- Drop the slide references from the test comments. They pointed at
the Cauldron presentation, but GCC does not allow attaching the PPT
to the patch, so the references are irrelevant here.
- Correct the i386.md comment: the simple flex mode accepts an
oversized locked region and still aligns its start; it does not skip
padding. An experimental post-relaxation patch does skip padding,
but that is not part of this proposal.
- bundle-align-5.c: rename the scan kernel instead of redefining
memchr with -D.
- bundle-align-6.c: drop the unneeded x86-tight-loop-max-size param.
- Reduce the tests to the scanned kernel: drop the main drivers and
headers.
gcc/config.in | 6 +
gcc/config/i386/i386-features.cc | 331 +++++++++++++++++-
gcc/config/i386/i386-options.cc | 26 +-
gcc/config/i386/i386-passes.def | 6 +-
gcc/config/i386/i386-protos.h | 2 +-
gcc/config/i386/i386.md | 35 ++
gcc/config/i386/i386.opt | 12 +
gcc/config/i386/i386.opt.urls | 3 +
gcc/config/i386/x86-tune.def | 7 +
gcc/configure | 38 ++
gcc/configure.ac | 11 +
gcc/doc/extend.texi | 6 +
gcc/doc/invoke.texi | 15 +-
gcc/doc/sourcebuild.texi | 4 +
.../gcc.target/i386/bundle-align-1.c | 16 +
.../gcc.target/i386/bundle-align-2.c | 21 ++
.../gcc.target/i386/bundle-align-3.c | 26 ++
.../gcc.target/i386/bundle-align-4.c | 22 ++
.../gcc.target/i386/bundle-align-5.c | 17 +
.../gcc.target/i386/bundle-align-6.c | 24 ++
.../gcc.target/i386/bundle-align-7.c | 17 +
gcc/testsuite/lib/target-supports.exp | 16 +
22 files changed, 641 insertions(+), 20 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-1.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-2.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-3.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-4.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-5.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-6.c
create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-7.c
Comments
On Sat, Oct 3, 2026 at 4:03 PM Sarvesh Chandra <Sarvesh.Chandra@amd.com> wrote:
>
> Add ix86_bundle_align_loops, an RTL pass that marks selected loop regions
> with .bundle_lock flex. The assembler aligns each marked region after
> instruction relaxation, when the final instruction sizes are known, and
> flex mode accepts a region that ends up larger than one bundle.
>
> Candidates are formed from layout-backward loop edges and selected as a
> disjoint set by weighted interval scheduling.
>
> Enable the pass for Zen4 and later, add the target options and assembler
> feature check, and add tests for candidate formation, union construction,
> weight calculation, selection, and representative loop cases.
>
> gcc/
> * config/i386/i386-features.cc (struct bundle_cand): New.
> (bundle_cand_cmp_hi, bundle_hwi_cmp, bundle_hwi_less)
> (bundle_cand_hi_less, bundle_compute_layout)
> (bundle_collect_candidates, bundle_add_unions)
> (bundle_compute_weights, bundle_select_and_emit)
> (ix86_do_bundle_align_loops): New functions.
> (class pass_bundle_align_loops): New.
> (make_pass_bundle_align_loops): New function.
> * config/i386/i386-protos.h (make_pass_bundle_align_loops): Declare.
> * config/i386/i386-passes.def (pass_bundle_align_loops): Insert before
> pass_shorten_branches.
> * config/i386/i386-options.cc (ix86_default_bundle_align): New function.
> (ix86_default_align): Call it.
> * config/i386/i386.md (bundle_lock, bundle_unlock): New insns.
> * config/i386/i386.opt (mbundle-align-loops): New option.
> (x86-tight-loop-max-size, x86-bundle-align-branch-limit): New params.
> * config/i386/i386.opt.urls: Regenerate.
> * config/i386/x86-tune.def (X86_TUNE_BUNDLE_ALIGN_LOOPS): New tune flag.
> * configure.ac: Check for .bundle_lock flex support.
> * configure: Regenerate.
> * config.in: Regenerate.
> * doc/invoke.texi (-mbundle-align-loops, x86-tight-loop-max-size):
> Document.
> * doc/extend.texi (x86 Function Attributes): Document
> bundle-align-loops.
> * doc/sourcebuild.texi (gcc_bundle_align_loops): Document.
>
> gcc/testsuite/
> * gcc.target/i386/bundle-align-1.c: New test.
> * gcc.target/i386/bundle-align-2.c: New test.
> * gcc.target/i386/bundle-align-3.c: New test.
> * gcc.target/i386/bundle-align-4.c: New test.
> * gcc.target/i386/bundle-align-5.c: New test.
> * gcc.target/i386/bundle-align-6.c: New test.
> * gcc.target/i386/bundle-align-7.c: New test.
> * lib/target-supports.exp
> (check_effective_target_gcc_bundle_align_loops): New.
>
> Co-authored-by: Branden Moore <Branden.Moore@amd.com>
> Co-authored-by: Venkataramanan Kumar <vekumar@gcc.gnu.org>
> Signed-off-by: Sarvesh Chandra <Sarvesh.Chandra@amd.com>
> ---
> [Presented at GNU Tools Cauldron 2026.]
>
> Cosmetic changes in v2:
> - Add co-authors Branden Moore and Venkataramanan Kumar.
> - Drop the slide references from the test comments. They pointed at
> the Cauldron presentation, but GCC does not allow attaching the PPT
> to the patch, so the references are irrelevant here.
> - Correct the i386.md comment: the simple flex mode accepts an
> oversized locked region and still aligns its start; it does not skip
> padding. An experimental post-relaxation patch does skip padding,
> but that is not part of this proposal.
> - bundle-align-5.c: rename the scan kernel instead of redefining
> memchr with -D.
> - bundle-align-6.c: drop the unneeded x86-tight-loop-max-size param.
> - Reduce the tests to the scanned kernel: drop the main drivers and
> headers.
>
> gcc/config.in | 6 +
> gcc/config/i386/i386-features.cc | 331 +++++++++++++++++-
> gcc/config/i386/i386-options.cc | 26 +-
> gcc/config/i386/i386-passes.def | 6 +-
> gcc/config/i386/i386-protos.h | 2 +-
> gcc/config/i386/i386.md | 35 ++
> gcc/config/i386/i386.opt | 12 +
> gcc/config/i386/i386.opt.urls | 3 +
> gcc/config/i386/x86-tune.def | 7 +
> gcc/configure | 38 ++
> gcc/configure.ac | 11 +
> gcc/doc/extend.texi | 6 +
> gcc/doc/invoke.texi | 15 +-
> gcc/doc/sourcebuild.texi | 4 +
> .../gcc.target/i386/bundle-align-1.c | 16 +
> .../gcc.target/i386/bundle-align-2.c | 21 ++
> .../gcc.target/i386/bundle-align-3.c | 26 ++
> .../gcc.target/i386/bundle-align-4.c | 22 ++
> .../gcc.target/i386/bundle-align-5.c | 17 +
> .../gcc.target/i386/bundle-align-6.c | 24 ++
> .../gcc.target/i386/bundle-align-7.c | 17 +
> gcc/testsuite/lib/target-supports.exp | 16 +
> 22 files changed, 641 insertions(+), 20 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-1.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-2.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-3.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-4.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-5.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-6.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-7.c
>
> diff --git a/gcc/config.in b/gcc/config.in
> index b90caa0dd57..51dac287796 100644
> --- a/gcc/config.in
> +++ b/gcc/config.in
> @@ -1448,6 +1448,12 @@
> #endif
>
>
> +/* Define if your assembler accepts the flex argument to .bundle_lock. */
> +#ifndef USED_FOR_TARGET
> +#undef HAVE_GAS_BUNDLE_FLEX_LOCK
> +#endif
> +
> +
> /* Define 0/1 if your assembler supports CFI directives. */
> #undef HAVE_GAS_CFI_DIRECTIVE
>
> diff --git a/gcc/config/i386/i386-features.cc b/gcc/config/i386/i386-features.cc
> index eb5f82e42cb..6743c0ebcfd 100644
> --- a/gcc/config/i386/i386-features.cc
> +++ b/gcc/config/i386/i386-features.cc
> @@ -83,6 +83,7 @@ along with GCC; see the file COPYING3. If not see
> #include "ifcvt.h"
> #include "symbol-summary.h"
> #include "sreal.h"
> +#include "fibonacci_heap.h"
> #include "ipa-cp.h"
> #include "ipa-prop.h"
> #include "ipa-fnsummary.h"
> @@ -5646,12 +5647,314 @@ ix86_align_loops ()
> free_dominance_info (CDI_DOMINATORS);
> }
>
> +/* ix86_bundle_align_loops pass.
> +
> + 1. Compute layout offsets and identify feasible loop-backward edges.
> + 2. Add synthetic unions for candidates sharing a K-byte window.
> + 3. Weight candidates by the back-edge counts of loops contained in
> + them.
> + 4. Select a maximum-weight disjoint set with weighted interval
> + scheduling.
> + 5. Emit .bundle_lock flex for the selected regions.
> +
> + K is param_x86_tight_loop_max_size, or ix86_cost->prefetch_block when
> + unset. */
> +
> +/* A candidate loop region: the byte interval [lo, hi] on the layout axis
> + (hi inclusive), weight w, interval-scheduling predecessor p, the lock and
> + unlock blocks, and a feasibility flag. A synthetic union has w == -1 until
> + the weight sweep replaces it with its contained weight. */
> +
> +struct bundle_cand
> +{
> + class loop *loop;
> + basic_block lock, unlock;
> + HOST_WIDE_INT lo, hi;
> + gcov_type w;
> + int p;
> + bool feasible;
> +};
> +
> +/* Sort by hi ascending. Break ties by lo descending (a contained candidate
> + before its container), then by real loops before a synthetic union of the
> + same span. */
> +
> +static int
> +bundle_cand_cmp_hi (const void *a, const void *b)
> +{
> + const bundle_cand *x = (const bundle_cand *)a, *y = (const bundle_cand *)b;
> + if (x->hi != y->hi)
> + return x->hi < y->hi ? -1 : 1;
> + if (x->lo != y->lo)
> + return x->lo > y->lo ? -1 : 1;
> + if ((x->w == -1) != (y->w == -1))
> + return x->w == -1 ? 1 : -1;
> + return 0;
> +}
> +
> +/* qsort comparator for HOST_WIDE_INT, ascending (lo-value compression). */
> +
> +static int
> +bundle_hwi_cmp (const void *a, const void *b)
> +{
> + HOST_WIDE_INT x = *(const HOST_WIDE_INT *)a, y = *(const HOST_WIDE_INT *)b;
> + return x < y ? -1 : x > y ? 1 : 0;
> +}
> +
> +/* Less-than on HOST_WIDE_INT, for vec::lower_bound over the compressed
> + lo-value list. */
> +
> +static bool
> +bundle_hwi_less (const HOST_WIDE_INT &a, const HOST_WIDE_INT &b)
> +{
> + return a < b;
> +}
> +
> +/* Less-than on interval end, for vec::lower_bound in the WIS predecessor
> + search over the hi-sorted feasible set. */
> +
> +static bool
> +bundle_cand_hi_less (const bundle_cand &a, const bundle_cand &b)
> +{
> + return a.hi < b.hi;
> +}
> +
> +static void
> +bundle_compute_layout (HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
> + int *asm_lo, int *asm_hi)
> +{
> + HOST_WIDE_INT off = 0;
> + int nasm = 0;
> + basic_block bb;
> + FOR_EACH_BB_FN (bb, cfun)
can you avoid doing this for non-candidate loop blocks?
> + {
> + rtx_insn *insn;
> + bb_lo[bb->index] = off;
> + asm_lo[bb->index] = nasm;
> + FOR_BB_INSNS (bb, insn)
> + if (NONDEBUG_INSN_P (insn))
> + {
> + /* Inline asm has no reliable min size; count its presence so any
> + candidate spanning this block can be excluded outright. */
> + if (asm_noperands (PATTERN (insn)) >= 0)
> + nasm++;
> + else
> + off += ix86_min_insn_size (insn);
> + }
> + bb_hi[bb->index] = off;
> + asm_hi[bb->index] = nasm;
> + }
> +}
> +
> +static void
> +bundle_collect_candidates (vec<bundle_cand> &feas, HOST_WIDE_INT k,
> + HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
> + int *asm_lo, int *asm_hi)
> +{
> + /* Step 1: identify feasible candidates from layout-backward loop edges. */
> + basic_block bb;
> + FOR_EACH_BB_FN (bb, cfun)
I wonder if you can instead walk the loop tree?
> + {
> + edge e;
> + edge_iterator ei;
> + FOR_EACH_EDGE (e, ei, bb->succs)
> + if (e->count ().initialized_p ()
> + && e->dest->index >= NUM_FIXED_BLOCKS
> + && (e->dest == bb
> + || bb_lo[e->dest->index] < bb_lo[bb->index]))
> + {
> + bundle_cand c = {};
> + c.loop = bb->loop_father;
> + c.lock = e->dest;
> + c.unlock = bb;
> + c.lo = bb_lo[e->dest->index];
> + c.hi = bb_hi[bb->index];
> + c.w = e->count ().to_gcov_type ();
> + /* Both ends must lie in a real loop, not the function-wide fake
> + loop whose latch is EXIT. This rejects a layout-backward exit
> + edge whose destination is outside any loop. */
> + c.feasible = (c.loop
> + && c.loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
> + && e->dest->loop_father
> + && (e->dest->loop_father->latch
> + != EXIT_BLOCK_PTR_FOR_FN (cfun))
> + && asm_hi[bb->index] == asm_lo[e->dest->index]
> + && c.hi - c.lo <= k
> + && optimize_bb_for_speed_p (c.lock));
> + if (c.feasible)
> + feas.safe_push (c);
> + }
> + }
> +}
> +
> +static void
> +bundle_add_unions (vec<bundle_cand> &feas, HOST_WIDE_INT k)
> +{
> + /* Step 2: add synthetic unions for candidates sharing a K-byte window. */
> + feas.qsort (bundle_cand_cmp_hi);
> + unsigned base = feas.length ();
> + /* At most one union is added per candidate, so reserve now; this keeps the
> + &feas[i] pointers held by the heap valid across the push below. */
> + feas.reserve (2 * base);
> + fibonacci_heap<HOST_WIDE_INT, bundle_cand> heap (HOST_WIDE_INT_MIN);
> + for (unsigned i = 0; i < base; i++)
> + {
> + heap.insert (feas[i].lo, &feas[i]);
> + HOST_WIDE_INT thr = feas[i].hi - k;
> + while (heap.min_key () < thr)
> + heap.extract_min ();
> + if (heap.nodes () >= 2)
> + {
> + bundle_cand *s = heap.min ();
> + bundle_cand c = {};
> + c.loop = s->loop;
> + c.lock = s->lock;
> + c.unlock = feas[i].unlock;
> + c.lo = s->lo;
> + c.hi = feas[i].hi;
> + c.w = -1; /* Union; the weight sweep fills this in. */
> + c.feasible = true;
> + feas.safe_push (c);
> + }
> + }
> +}
> +
> +static void
> +bundle_compute_weights (vec<bundle_cand> &feas)
> +{
> + /* Step 3: compute containment weights with a Fenwick tree. */
> + feas.qsort (bundle_cand_cmp_hi);
> + unsigned n_all = feas.length ();
> + /* Compress lo values into a sorted, unique list; POS below maps a lo to its
> + reversed rank via lower_bound. */
> + auto_vec<HOST_WIDE_INT> los;
> + los.reserve (n_all);
> + for (unsigned p = 0; p < n_all; p++)
> + los.quick_push (feas[p].lo);
> + los.qsort (bundle_hwi_cmp);
> + int m = 0;
> + for (unsigned p = 0; p < n_all; p++)
> + if (m == 0 || los[m - 1] != los[p])
> + los[m++] = los[p];
> + los.truncate (m);
> + /* BIT[1..m] keyed by reversed lo-rank, so a prefix sum [1..pos] covers
> + exactly lo_j >= lo_i. */
> + gcov_type *bit = XCNEWVEC (gcov_type, m + 1);
> + for (unsigned p = 0; p < n_all; p++)
> + {
> + /* Reversed rank of feas[p].lo: LOWER_BOUND is the first LOS >= lo. */
> + int pos = m - (int) los.lower_bound (feas[p].lo, bundle_hwi_less);
> + /* A union is not a real loop, so do not insert it into the tree. Its
> + weight is the contained sum, less one to break ties toward it. */
> + bool u = feas[p].w == -1;
> + if (!u)
> + for (int i = pos; i <= m; i += i & -i)
> + bit[i] += feas[p].w;
> + gcov_type sum = 0;
> + for (int i = pos; i > 0; i -= i & -i)
> + sum += bit[i];
> + feas[p].w = u ? sum - 1 : sum;
> + }
> + XDELETEVEC (bit);
> +}
> +
> +static void
> +bundle_select_and_emit (vec<bundle_cand> &feas)
> +{
> + /* Step 4: select the maximum-weight disjoint set. */
> + unsigned n = feas.length ();
> + /* dp[i+1] is the best weight over feas[0..i]. Whether i is taken is
> + recomputed while backtracking rather than stored. */
> + gcov_type *dp = XCNEWVEC (gcov_type, n + 1);
> +
> + for (unsigned i = 0; i < n; i++)
> + {
> + /* Predecessor: the last candidate that ends before feas[i] starts.
> + hi is inclusive, so find the first candidate with hi >= lo. */
> + bundle_cand key = {};
> + key.hi = feas[i].lo;
> + int p = (int) feas.lower_bound (key, bundle_cand_hi_less) - 1;
> + feas[i].p = p;
> + gcov_type take = feas[i].w + dp[p + 1], skip = dp[i];
> + dp[i + 1] = take >= skip ? take : skip;
> + if (dump_file)
> + fprintf (dump_file,
> + "bundle-align: cand %u loop %d [bb%d..bb%d] "
> + "lo=%ld hi=%ld w=%ld p=%d take=%ld skip=%ld => %s\n",
> + i, feas[i].loop->num, feas[i].lock->index,
> + feas[i].unlock->index, (long) feas[i].lo,
> + (long) feas[i].hi, (long) feas[i].w, p,
> + (long) take, (long) skip, take >= skip ? "take" : "skip");
> + }
> +
> + /* Step 5: emit the selected regions. */
> + for (int i = (int) n - 1; i >= 0;)
> + if (feas[i].w + dp[feas[i].p + 1] >= dp[i])
> + {
> + basic_block h = feas[i].lock, l = feas[i].unlock;
> + emit_insn_before (gen_bundle_lock (), BB_HEAD (h));
> + emit_insn_after (gen_bundle_unlock (), BB_END (l));
> + if (dump_file)
> + fprintf (dump_file,
> + "bundle-align: wrap loop %d [bb%d..bb%d] "
> + "w=%ld span=%ld\n", feas[i].loop->num, h->index,
> + l->index, (long) feas[i].w,
> + (long) (feas[i].hi - feas[i].lo));
> + i = feas[i].p;
> + }
> + else
> + i--;
> +
> + XDELETEVEC (dp);
> +}
> +
> +/* Wrap the selected disjoint hot loop regions in .bundle_lock/.bundle_unlock.
> + Each region is an interval on the layout byte axis, which handles rotated
> + (latch-before-header) loops uniformly. */
> +
> +static void
> +ix86_do_bundle_align_loops ()
> +{
> + if (ix86_cost->prefetch_block == 0)
|| number_of_loops (cfun) <= 1
> + return;
> +
> + /* Region size cap: tuning param if set, else the CPU bundle size. */
> + HOST_WIDE_INT k = (param_x86_tight_loop_max_size > 0
> + ? (HOST_WIDE_INT)param_x86_tight_loop_max_size
> + : (HOST_WIDE_INT)ix86_cost->prefetch_block);
> +
> + loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
> +
> + /* Per-block layout byte offsets [bb_lo, bb_hi). */
> + int nbb = last_basic_block_for_fn (cfun);
> + HOST_WIDE_INT *bb_lo = XCNEWVEC (HOST_WIDE_INT, nbb);
> + HOST_WIDE_INT *bb_hi = XCNEWVEC (HOST_WIDE_INT, nbb);
> + /* Running count of inline-asm insns in layout order; [asm_lo, asm_hi) lets a
> + candidate test whether any block it spans contains an asm. */
> + int *asm_lo = XCNEWVEC (int, nbb);
> + int *asm_hi = XCNEWVEC (int, nbb);
> + bundle_compute_layout (bb_lo, bb_hi, asm_lo, asm_hi);
> +
> + auto_vec<bundle_cand> feas;
> + bundle_collect_candidates (feas, k, bb_lo, bb_hi, asm_lo, asm_hi);
> + bundle_add_unions (feas, k);
> + bundle_compute_weights (feas);
> + if (!feas.is_empty ())
> + bundle_select_and_emit (feas);
> +
> + XDELETEVEC (bb_lo);
> + XDELETEVEC (bb_hi);
> + XDELETEVEC (asm_lo);
> + XDELETEVEC (asm_hi);
> + loop_optimizer_finalize ();
> +}
> +
> namespace {
>
> -const pass_data pass_data_align_tight_loops =
> +const pass_data pass_data_bundle_align_loops =
> {
> RTL_PASS, /* type */
> - "align_tight_loops", /* name */
> + "bundle_align_loops", /* name */
> OPTGROUP_NONE, /* optinfo_flags */
> TV_MACH_DEP, /* tv_id */
> 0, /* properties_required */
> @@ -5661,40 +5964,44 @@ const pass_data pass_data_align_tight_loops =
> 0, /* todo_flags_finish */
> };
>
> -class pass_align_tight_loops : public rtl_opt_pass
> +class pass_bundle_align_loops : public rtl_opt_pass
> {
> public:
> - pass_align_tight_loops (gcc::context *ctxt)
> - : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
> + pass_bundle_align_loops (gcc::context *ctxt)
> + : rtl_opt_pass (pass_data_bundle_align_loops, ctxt)
> {}
>
> /* opt_pass methods: */
> bool gate (function *) final override
> {
> - return TARGET_ALIGN_TIGHT_LOOPS
> - && optimize
> - && optimize_function_for_speed_p (cfun);
> + return optimize
> + && optimize_function_for_speed_p (cfun)
> + && (ix86_bundle_align_loops > 0 || TARGET_ALIGN_TIGHT_LOOPS);
> }
>
> unsigned int execute (function *) final override
> {
> timevar_push (TV_MACH_DEP);
> + if (ix86_bundle_align_loops > 0)
> + ix86_do_bundle_align_loops ();
> #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
> - ix86_align_loops ();
> + else
> + ix86_align_loops ();
> #endif
> timevar_pop (TV_MACH_DEP);
> return 0;
> }
> -}; // class pass_align_tight_loops
> +}; // class pass_bundle_align_loops
>
> } // anon namespace
>
> rtl_opt_pass *
> -make_pass_align_tight_loops (gcc::context *ctxt)
> +make_pass_bundle_align_loops (gcc::context *ctxt)
> {
> - return new pass_align_tight_loops (ctxt);
> + return new pass_bundle_align_loops (ctxt);
> }
>
> +
> /* This compares the priority of target features in function DECL1
> and DECL2. It returns positive value if DECL1 is higher priority,
> negative value if DECL2 is higher priority and 0 if they are the
> diff --git a/gcc/config/i386/i386-options.cc b/gcc/config/i386/i386-options.cc
> index e7f06154103..db0a9426b29 100644
> --- a/gcc/config/i386/i386-options.cc
> +++ b/gcc/config/i386/i386-options.cc
> @@ -1872,8 +1872,16 @@ set_ix86_tune_features (struct gcc_options *opts,
> /* Default align_* from the processor table. */
>
> static void
> -ix86_default_align (struct gcc_options *opts)
> +ix86_default_align (struct gcc_options *opts,
> + struct gcc_options *opts_set)
> {
> + /* Bundle alignment replaces generic loop, jump, and label padding. */
> + if (opts->x_ix86_bundle_align_loops > 0)
> + {
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_loops, "1");
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_jumps, "1");
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_labels, "1");
> + }
> /* -falign-foo without argument: supply one. */
> if (opts->x_flag_align_loops && !opts->x_str_align_loops)
> opts->x_str_align_loops = processor_cost_table[ix86_tune]->align_loop;
> @@ -1885,6 +1893,16 @@ ix86_default_align (struct gcc_options *opts)
> opts->x_str_align_functions = processor_cost_table[ix86_tune]->align_func;
> }
>
> +/* Default -mbundle-align-loops from the active tune model. */
> +
> +static void
> +ix86_default_bundle_align (struct gcc_options *opts)
> +{
> + if (opts->x_ix86_bundle_align_loops == -1)
> + opts->x_ix86_bundle_align_loops
> + = ix86_tune_features[X86_TUNE_BUNDLE_ALIGN_LOOPS] ? 1 : 0;
> +}
> +
> #ifndef USE_IX86_FRAME_POINTER
> #define USE_IX86_FRAME_POINTER 0
> #endif
> @@ -1986,7 +2004,8 @@ ix86_override_options_after_change_1 (struct gcc_options *opts,
> void
> ix86_override_options_after_change (void)
> {
> - ix86_default_align (&global_options);
> + ix86_default_bundle_align (&global_options);
> + ix86_default_align (&global_options, &global_options_set);
>
> ix86_recompute_optlev_based_flags (&global_options, &global_options_set);
>
> @@ -2619,7 +2638,8 @@ ix86_option_override_internal (bool main_args_p,
> opts->x_ix86_regparm = REGPARM_MAX;
>
> /* Default align_* from the processor table. */
> - ix86_default_align (&global_options);
> + ix86_default_bundle_align (&global_options);
> + ix86_default_align (&global_options, &global_options_set);
>
> /* Provide default for -mbranch-cost= value. */
> SET_OPTION_IF_UNSET (opts, opts_set, ix86_branch_cost,
> diff --git a/gcc/config/i386/i386-passes.def b/gcc/config/i386/i386-passes.def
> index 4f461116cac..82d4d7411df 100644
> --- a/gcc/config/i386/i386-passes.def
> +++ b/gcc/config/i386/i386-passes.def
> @@ -31,9 +31,9 @@ along with GCC; see the file COPYING3. If not see
> INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */);
>
> INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_insert_endbr_and_patchable_area);
> - /* pass_align_tight_loops must be after pass_insert_endbr_and_patchable_area.
> - PR116174. */
> - INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_align_tight_loops);
> + /* Tight/bundle loop aligner; after pass_insert_endbr_and_patchable_area
> + (PR116174) and before pass_shorten_branches. */
> + INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_bundle_align_loops);
>
> INSERT_PASS_AFTER (pass_late_combine, 1, pass_x86_cse);
> INSERT_PASS_AFTER (pass_late_combine, 1, pass_remove_partial_avx_dependency);
> diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
> index 2687de646b0..84d2a7ad74c 100644
> --- a/gcc/config/i386/i386-protos.h
> +++ b/gcc/config/i386/i386-protos.h
> @@ -434,7 +434,7 @@ extern rtl_opt_pass *make_pass_remove_partial_avx_dependency
> (gcc::context *);
> extern rtl_opt_pass *make_pass_x86_cse (gcc::context *);
> extern rtl_opt_pass *make_pass_apx_nf_convert (gcc::context *);
> -extern rtl_opt_pass *make_pass_align_tight_loops (gcc::context *);
> +extern rtl_opt_pass *make_pass_bundle_align_loops (gcc::context *);
>
> extern bool ix86_has_no_direct_extern_access;
> extern bool ix86_rpad_gate ();
> diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
> index 5768a93d5b0..f85010a41aa 100644
> --- a/gcc/config/i386/i386.md
> +++ b/gcc/config/i386/i386.md
> @@ -233,6 +233,8 @@
> UNSPECV_STACK_PROBE
> UNSPECV_PROBE_STACK_RANGE
> UNSPECV_ALIGN
> + UNSPECV_BUNDLE_LOCK
> + UNSPECV_BUNDLE_UNLOCK
> UNSPECV_PROLOGUE_USE
> UNSPECV_SPLIT_STACK_RETURN
> UNSPECV_CLD
> @@ -21756,6 +21758,39 @@
> }
> [(set_attr "length" "16")])
>
> +;; ix86_bundle_align_loops pass: align the locked region's start to the cache
> +;; line (prefetch_block). Final instruction sizes may grow during assembly
> +;; because of indirect calls, prefixes, and relaxation, so a selected region
> +;; can exceed one bundle; flex .bundle_lock accepts such a region instead of
> +;; raising an error. Requires flex .bundle_lock support.
> +(define_insn "bundle_lock"
> + [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_LOCK)]
> + ""
> +{
> +#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
> + fprintf (asm_out_file, "\t.bundle_align_mode %d\n\t.bundle_lock flex\n",
> + ceil_log2 (ix86_cost->prefetch_block));
> +#endif
> + return "";
> +}
> + [(set_attr "length" "0")
> + (set_attr "length_immediate" "0")
> + (set_attr "modrm" "0")])
> +
> +;; Close the bundle and turn bundle alignment mode back off.
> +(define_insn "bundle_unlock"
> + [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_UNLOCK)]
> + ""
> +{
> +#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
> + fputs ("\t.bundle_unlock\n\t.bundle_align_mode 0\n", asm_out_file);
> +#endif
> + return "";
> +}
> + [(set_attr "length" "0")
> + (set_attr "length_immediate" "0")
> + (set_attr "modrm" "0")])
> +
> (define_expand "prologue"
> [(const_int 0)]
> ""
> diff --git a/gcc/config/i386/i386.opt b/gcc/config/i386/i386.opt
> index 57ba3fbfa8f..148d26b6053 100644
> --- a/gcc/config/i386/i386.opt
> +++ b/gcc/config/i386/i386.opt
> @@ -240,6 +240,10 @@ malign-loops=
> Target Undocumented RejectNegative Joined UInteger
> Loop code aligned to this power of 2.
>
> +mbundle-align-loops
> +Target Var(ix86_bundle_align_loops) Init(-1) Save
> +Wrap small hot loops with .bundle_lock so the assembler keeps them inside one cache line.
> +
> malign-stringops
> Target RejectNegative InverseMask(NO_ALIGN_STRINGOPS, ALIGN_STRINGOPS) Save
> Align destination of the string operations.
> @@ -1221,6 +1225,14 @@ Do not use GOT to access external symbols.
> Target Joined UInteger Var(x86_stlf_window_ninsns) Init(64) Param
> Instructions number above which STFL stall penalty can be compensated.
>
> +-param=x86-tight-loop-max-size=
> +Target Joined UInteger Var(param_x86_tight_loop_max_size) Init(0) Param
> +Maximum body size in bytes for the i386 tight-loop aligner; 0 picks the active tune prefetch_block.
> +
> +-param=x86-bundle-align-branch-limit=
> +Target Joined UInteger Var(param_x86_bundle_align_branch_limit) Init(2) Param
> +Maximum internal conditional branches allowed in a bundle-locked loop region (the back-edge branch is not counted).
> +
> mavxifma
> Target Mask(ISA2_AVXIFMA) Var(ix86_isa_flags2) Save
> Support MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, AVX, AVX2, and
> diff --git a/gcc/config/i386/i386.opt.urls b/gcc/config/i386/i386.opt.urls
> index d52e6de8014..20a638ea1f7 100644
> --- a/gcc/config/i386/i386.opt.urls
> +++ b/gcc/config/i386/i386.opt.urls
> @@ -28,6 +28,9 @@ UrlSuffix(gcc/x86-Options.html#index-malign-double)
>
> ; skipping UrlSuffix for 'malign-loops=' due to finding no URLs
>
> +mbundle-align-loops
> +UrlSuffix(gcc/x86-Options.html#index-mbundle-align-loops)
> +
> malign-stringops
> UrlSuffix(gcc/x86-Options.html#index-malign-stringops)
>
> diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
> index d4c27351ad7..4716f26b2e1 100644
> --- a/gcc/config/i386/x86-tune.def
> +++ b/gcc/config/i386/x86-tune.def
> @@ -259,6 +259,13 @@ DEF_TUNE (X86_TUNE_ALIGN_TIGHT_LOOPS, "align_tight_loops",
> ~(m_ZHAOXIN | m_CASCADELAKE | m_SKYLAKE_AVX512
> | m_CORE_ATOM))
>
> +/* X86_TUNE_BUNDLE_ALIGN_LOOPS: when true, use the ix86_bundle_align_loops
> + pass with the assembler's .bundle_lock/.bundle_unlock directives instead
> + of static .p2align padding. Targets cores where loops straddling a 64-byte
> + instruction-cache line are particularly costly (AMD Zen4/Zen5/Zen6). */
> +DEF_TUNE (X86_TUNE_BUNDLE_ALIGN_LOOPS, "bundle_align_loops",
> + m_ZNVER4 | m_ZNVER5 | m_ZNVER6)
> +
> /*****************************************************************************/
> /* Integer instruction selection tuning */
> /*****************************************************************************/
> diff --git a/gcc/configure b/gcc/configure
> index 43ca45144b8..dd5a0c0a539 100755
> --- a/gcc/configure
> +++ b/gcc/configure
> @@ -26306,6 +26306,44 @@ $as_echo "#define HAVE_GAS_MAX_SKIP_P2ALIGN 1" >>confdefs.h
> fi
>
>
> +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .bundle_lock flex argument" >&5
> +$as_echo_n "checking assembler for .bundle_lock flex argument... " >&6; }
> +if ${gcc_cv_as_bundle_flex_lock+:} false; then :
> + $as_echo_n "(cached) " >&6
> +else
> + gcc_cv_as_bundle_flex_lock=no
> + if test x"$gcc_cv_as" != x; then
> + $as_echo '.bundle_align_mode 6
> +.bundle_lock flex
> +.rept 100
> +nop
> +.endr
> +.bundle_unlock' > conftest.s
> + if { ac_try='$gcc_cv_as $gcc_cv_as_flags -o conftest.o conftest.s >&5'
> + { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5
> + (eval $ac_try) 2>&5
> + ac_status=$?
> + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
> + test $ac_status = 0; }; }
> + then
> + gcc_cv_as_bundle_flex_lock=yes
> + else
> + echo "configure: failed program was" >&5
> + cat conftest.s >&5
> + fi
> + rm -f conftest.o conftest.s
> + fi
> +fi
> +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_as_bundle_flex_lock" >&5
> +$as_echo "$gcc_cv_as_bundle_flex_lock" >&6; }
> +if test $gcc_cv_as_bundle_flex_lock = yes; then
> +
> +$as_echo "#define HAVE_GAS_BUNDLE_FLEX_LOCK 1" >>confdefs.h
> +
> +fi
> +
> +
> +
> { $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .literal16" >&5
> $as_echo_n "checking assembler for .literal16... " >&6; }
> if ${gcc_cv_as_literal16+:} false; then :
> diff --git a/gcc/configure.ac b/gcc/configure.ac
> index 075b6b68ffa..346bf254de3 100644
> --- a/gcc/configure.ac
> +++ b/gcc/configure.ac
> @@ -3142,6 +3142,17 @@ gcc_GAS_CHECK_FEATURE([.p2align with maximum skip], gcc_cv_as_max_skip_p2align,,
> [Define if your assembler supports specifying the maximum number
> of bytes to skip when using the GAS .p2align command.])])
>
> +gcc_GAS_CHECK_FEATURE([.bundle_lock flex argument], gcc_cv_as_bundle_flex_lock,,
> + [.bundle_align_mode 6
> +.bundle_lock flex
> +.rept 100
> +nop
> +.endr
> +.bundle_unlock],,
> +[AC_DEFINE(HAVE_GAS_BUNDLE_FLEX_LOCK, 1,
> + [Define if your assembler accepts the flex argument to .bundle_lock
> + that permits bundles larger than .bundle_align_mode alignment.])])
> +
> gcc_GAS_CHECK_FEATURE([.literal16], gcc_cv_as_literal16,,
> [.literal16],,
> [AC_DEFINE(HAVE_GAS_LITERAL16, 1,
> diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
> index 397b05ab87d..9081317927d 100644
> --- a/gcc/doc/extend.texi
> +++ b/gcc/doc/extend.texi
> @@ -9598,6 +9598,12 @@ operations and calling the library routines for large operations.
> @itemx no-align-stringops
> Do/do not align destination of inlined string operations.
>
> +@atindex @code{target("bundle-align-loops")}, x86
> +@item bundle-align-loops
> +@itemx no-bundle-align-loops
> +Enable/disable wrapping of small hot loops in assembler
> +@code{.bundle_lock}/@code{.bundle_unlock} directives.
> +
> @atindex @code{target("recip")}, x86
> @item recip
> @itemx no-recip
> diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
> index e1870c9c227..619f92185fd 100644
> --- a/gcc/doc/invoke.texi
> +++ b/gcc/doc/invoke.texi
> @@ -1554,7 +1554,8 @@ See RS/6000 and PowerPC Options.
> -mamx-complex -mavxvnniint16 -msm3 -msha512 -msm4 -mapxf
> -musermsr -mavx10.1 -mavx10.2 -mamx-avx512 -mmovrs -mamx-movrs
> -mavx512bmm -mavx10v2aux
> --mcldemote -mms-bitfields -mno-align-stringops -minline-all-stringops
> +-mcldemote -mms-bitfields -mbundle-align-loops -mno-align-stringops
> +-minline-all-stringops
> -minline-stringops-dynamically -mstringop-strategy=@var{alg}
> -mkl -mwidekl
> -mmemcpy-strategy=@var{strategy} -mmemset-strategy=@var{strategy}
> @@ -37155,6 +37156,18 @@ Here, @code{t5} takes up 2 bytes.
> @end enumerate
>
>
> +@opindex mbundle-align-loops
> +@opindex mno-bundle-align-loops
> +@item -mbundle-align-loops
> +@itemx -mno-bundle-align-loops
> +Wrap small hot loops in assembler @code{.bundle_lock}/@code{.bundle_unlock}
> +directives so the assembler keeps each loop within a single instruction-cache
> +line, aligning the loop header without inserting executed padding. Enabled by
> +default for AMD Zen4 and later. It defaults @option{-falign-loops},
> +@option{-falign-jumps}, and @option{-falign-labels} to 1 unless explicitly set.
> +The maximum loop-body size considered is set by the
> +@option{x86-tight-loop-max-size} parameter.
> +
> @opindex mno-align-stringops
> @opindex malign-stringops
> @item -mno-align-stringops
> diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
> index aad51550eba..35324653699 100644
> --- a/gcc/doc/sourcebuild.texi
> +++ b/gcc/doc/sourcebuild.texi
> @@ -3114,6 +3114,10 @@ Target supports @option{-fstack-protector}.
> @item gas
> Target uses GNU @command{as}.
>
> +@item gcc_bundle_align_loops
> +Target compiler and assembler support @option{-mbundle-align-loops} and
> +@code{.bundle_lock flex}.
> +
> @item gc_sections
> Target supports @option{--gc-sections}.
>
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-1.c b/gcc/testsuite/gcc.target/i386/bundle-align-1.c
> new file mode 100644
> index 00000000000..c7120731f5f
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-1.c
> @@ -0,0 +1,16 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +typedef int (*callee_t) (int, int *);
> +
> +int
> +indirect_sum (callee_t callee, int n, int *array, long calls)
> +{
> + int sink = 0;
> + for (long call = 0; call < calls; ++call)
> + sink += callee (n, array);
> + return sink;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-2.c b/gcc/testsuite/gcc.target/i386/bundle-align-2.c
> new file mode 100644
> index 00000000000..957e853dfb2
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-2.c
> @@ -0,0 +1,21 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +__attribute__((noinline,noipa))
> +int
> +f (int n, int *p)
> +{
> + int i = 0, s = 0;
> + if (n & 1)
> + goto mid;
> + while (i < n)
> + {
> + s += p[i & 1023];
> + mid:
> + i++;
> + }
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-3.c b/gcc/testsuite/gcc.target/i386/bundle-align-3.c
> new file mode 100644
> index 00000000000..7cdfa8db985
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-3.c
> @@ -0,0 +1,26 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +int a[1024];
> +__attribute__((hot,noinline,noipa))
> +int
> +two_backedges (int n, int m)
> +{
> + int i = 0, s = 0;
> +outer_head:
> + {
> + int j = 0;
> + inner_head:
> + s += a[(i + j) & 1023];
> + if (__builtin_expect (s < 0, 0))
> + goto outer_head;
> + if (++j < m)
> + goto inner_head;
> + }
> + if (++i < n)
> + goto outer_head;
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-4.c b/gcc/testsuite/gcc.target/i386/bundle-align-4.c
> new file mode 100644
> index 00000000000..80bca4ec200
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-4.c
> @@ -0,0 +1,22 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver5 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
> +/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
> +
> +volatile int sink;
> +int a[1024];
> +int
> +classify (int n)
> +{
> + int s = 0;
> + asm volatile (".rept 0\n\t.byte 0x90\n\t.endr" ::: "memory");
> + for (int i = 0; i < n; i++)
> + {
> + if (a[i] & 1) sink = i;
> + if (a[i] & 2) sink = i + 1;
> + if (a[i] & 4) s += 4;
> + s += a[i];
> + }
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-5.c b/gcc/testsuite/gcc.target/i386/bundle-align-5.c
> new file mode 100644
> index 00000000000..7aa07effa6e
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-5.c
> @@ -0,0 +1,17 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +void *mc (const void *s, int c, unsigned long n)
> +{
> + const unsigned char *p = s;
> + while (n--)
> + {
> + if (*p == (unsigned char) c)
> + return (void *) p;
> + p++;
> + }
> + return 0;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-6.c b/gcc/testsuite/gcc.target/i386/bundle-align-6.c
> new file mode 100644
> index 00000000000..1f561e07caa
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-6.c
> @@ -0,0 +1,24 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
> +/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
> +
> +struct O { long rc; };
> +extern void dealloc (struct O *);
> +void
> +f (struct O **v, long n)
> +{
> + asm volatile (".rept 10\n\t.byte 0x90\n\t.endr" ::: "memory");
> + while (--n >= 0)
> + {
> + struct O *x = v[n];
> + if (x)
> + {
> + if (x->rc < 0)
> + ;
> + else if (--x->rc == 0)
> + dealloc (x);
> + }
> + }
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-7.c b/gcc/testsuite/gcc.target/i386/bundle-align-7.c
> new file mode 100644
> index 00000000000..b713d25d0b7
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-7.c
> @@ -0,0 +1,17 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O1 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +static int A[1 << 20];
> +
> +int
> +nested_sum (int n, int m)
> +{
> + int sum = 0;
> + for (int i = 0; i < n; ++i)
> + for (int j = 0; j < m; ++j)
> + sum += A[i * m + j];
> + return sum;
> +}
> diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
> index d467e9fa3b7..3286c40b07d 100644
> --- a/gcc/testsuite/lib/target-supports.exp
> +++ b/gcc/testsuite/lib/target-supports.exp
> @@ -12491,6 +12491,22 @@ proc check_effective_target_gas { } {
> return $use_gas_saved
> }
>
> +# Return 1 if the compiler was configured with assembler support for
> +# the flex .bundle_lock argument used by -mbundle-align-loops.
> +
> +proc check_effective_target_gcc_bundle_align_loops { } {
> + return [check_no_messages_and_pattern gcc_bundle_align_loops \
> + "\\.bundle_lock flex" assembly {
> + int a[1024];
> + int foo (int n) {
> + int s = 0;
> + for (int i = 0; i < n; i++)
> + s += a[i];
> + return s;
> + }
> + } "-O2 -march=znver4 -mbundle-align-loops -falign-loops=1"]
> +}
> +
> # Return 1 if GNU ld is used.
>
> proc check_effective_target_gld { } {
> --
> 2.34.1
>
On Sat Oct 3, 2026 at 7:02 AM PDT, Sarvesh Chandra wrote:
> Add ix86_bundle_align_loops, an RTL pass that marks selected loop regions
> with .bundle_lock flex. The assembler aligns each marked region after
> instruction relaxation, when the final instruction sizes are known, and
> flex mode accepts a region that ends up larger than one bundle.
>
> Candidates are formed from layout-backward loop edges and selected as a
> disjoint set by weighted interval scheduling.
>
> Enable the pass for Zen4 and later, add the target options and assembler
> feature check, and add tests for candidate formation, union construction,
> weight calculation, selection, and representative loop cases.
>
> gcc/
> * config/i386/i386-features.cc (struct bundle_cand): New.
> (bundle_cand_cmp_hi, bundle_hwi_cmp, bundle_hwi_less)
> (bundle_cand_hi_less, bundle_compute_layout)
> (bundle_collect_candidates, bundle_add_unions)
> (bundle_compute_weights, bundle_select_and_emit)
> (ix86_do_bundle_align_loops): New functions.
> (class pass_bundle_align_loops): New.
> (make_pass_bundle_align_loops): New function.
> * config/i386/i386-protos.h (make_pass_bundle_align_loops): Declare.
> * config/i386/i386-passes.def (pass_bundle_align_loops): Insert before
> pass_shorten_branches.
> * config/i386/i386-options.cc (ix86_default_bundle_align): New function.
> (ix86_default_align): Call it.
> * config/i386/i386.md (bundle_lock, bundle_unlock): New insns.
> * config/i386/i386.opt (mbundle-align-loops): New option.
> (x86-tight-loop-max-size, x86-bundle-align-branch-limit): New params.
> * config/i386/i386.opt.urls: Regenerate.
> * config/i386/x86-tune.def (X86_TUNE_BUNDLE_ALIGN_LOOPS): New tune flag.
> * configure.ac: Check for .bundle_lock flex support.
> * configure: Regenerate.
> * config.in: Regenerate.
> * doc/invoke.texi (-mbundle-align-loops, x86-tight-loop-max-size):
> Document.
> * doc/extend.texi (x86 Function Attributes): Document
> bundle-align-loops.
> * doc/sourcebuild.texi (gcc_bundle_align_loops): Document.
>
> gcc/testsuite/
> * gcc.target/i386/bundle-align-1.c: New test.
> * gcc.target/i386/bundle-align-2.c: New test.
> * gcc.target/i386/bundle-align-3.c: New test.
> * gcc.target/i386/bundle-align-4.c: New test.
> * gcc.target/i386/bundle-align-5.c: New test.
> * gcc.target/i386/bundle-align-6.c: New test.
> * gcc.target/i386/bundle-align-7.c: New test.
> * lib/target-supports.exp
> (check_effective_target_gcc_bundle_align_loops): New.
>
> Co-authored-by: Branden Moore <Branden.Moore@amd.com>
> Co-authored-by: Venkataramanan Kumar <vekumar@gcc.gnu.org>
> Signed-off-by: Sarvesh Chandra <Sarvesh.Chandra@amd.com>
> ---
> [Presented at GNU Tools Cauldron 2026.]
>
> Cosmetic changes in v2:
> - Add co-authors Branden Moore and Venkataramanan Kumar.
> - Drop the slide references from the test comments. They pointed at
> the Cauldron presentation, but GCC does not allow attaching the PPT
> to the patch, so the references are irrelevant here.
> - Correct the i386.md comment: the simple flex mode accepts an
> oversized locked region and still aligns its start; it does not skip
> padding. An experimental post-relaxation patch does skip padding,
> but that is not part of this proposal.
> - bundle-align-5.c: rename the scan kernel instead of redefining
> memchr with -D.
> - bundle-align-6.c: drop the unneeded x86-tight-loop-max-size param.
> - Reduce the tests to the scanned kernel: drop the main drivers and
> headers.
>
> gcc/config.in | 6 +
> gcc/config/i386/i386-features.cc | 331 +++++++++++++++++-
> gcc/config/i386/i386-options.cc | 26 +-
> gcc/config/i386/i386-passes.def | 6 +-
> gcc/config/i386/i386-protos.h | 2 +-
> gcc/config/i386/i386.md | 35 ++
> gcc/config/i386/i386.opt | 12 +
> gcc/config/i386/i386.opt.urls | 3 +
> gcc/config/i386/x86-tune.def | 7 +
> gcc/configure | 38 ++
> gcc/configure.ac | 11 +
> gcc/doc/extend.texi | 6 +
> gcc/doc/invoke.texi | 15 +-
> gcc/doc/sourcebuild.texi | 4 +
> .../gcc.target/i386/bundle-align-1.c | 16 +
> .../gcc.target/i386/bundle-align-2.c | 21 ++
> .../gcc.target/i386/bundle-align-3.c | 26 ++
> .../gcc.target/i386/bundle-align-4.c | 22 ++
> .../gcc.target/i386/bundle-align-5.c | 17 +
> .../gcc.target/i386/bundle-align-6.c | 24 ++
> .../gcc.target/i386/bundle-align-7.c | 17 +
> gcc/testsuite/lib/target-supports.exp | 16 +
> 22 files changed, 641 insertions(+), 20 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-1.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-2.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-3.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-4.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-5.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-6.c
> create mode 100644 gcc/testsuite/gcc.target/i386/bundle-align-7.c
>
> diff --git a/gcc/config.in b/gcc/config.in
> index b90caa0dd57..51dac287796 100644
> --- a/gcc/config.in
> +++ b/gcc/config.in
> @@ -1448,6 +1448,12 @@
> #endif
>
>
> +/* Define if your assembler accepts the flex argument to .bundle_lock. */
> +#ifndef USED_FOR_TARGET
> +#undef HAVE_GAS_BUNDLE_FLEX_LOCK
> +#endif
> +
> +
> /* Define 0/1 if your assembler supports CFI directives. */
> #undef HAVE_GAS_CFI_DIRECTIVE
>
> diff --git a/gcc/config/i386/i386-features.cc b/gcc/config/i386/i386-features.cc
> index eb5f82e42cb..6743c0ebcfd 100644
> --- a/gcc/config/i386/i386-features.cc
> +++ b/gcc/config/i386/i386-features.cc
> @@ -83,6 +83,7 @@ along with GCC; see the file COPYING3. If not see
> #include "ifcvt.h"
> #include "symbol-summary.h"
> #include "sreal.h"
> +#include "fibonacci_heap.h"
> #include "ipa-cp.h"
> #include "ipa-prop.h"
> #include "ipa-fnsummary.h"
> @@ -5646,12 +5647,314 @@ ix86_align_loops ()
> free_dominance_info (CDI_DOMINATORS);
> }
>
> +/* ix86_bundle_align_loops pass.
> +
> + 1. Compute layout offsets and identify feasible loop-backward edges.
> + 2. Add synthetic unions for candidates sharing a K-byte window.
> + 3. Weight candidates by the back-edge counts of loops contained in
> + them.
> + 4. Select a maximum-weight disjoint set with weighted interval
> + scheduling.
> + 5. Emit .bundle_lock flex for the selected regions.
> +
> + K is param_x86_tight_loop_max_size, or ix86_cost->prefetch_block when
> + unset. */
> +
> +/* A candidate loop region: the byte interval [lo, hi] on the layout axis
> + (hi inclusive), weight w, interval-scheduling predecessor p, the lock and
> + unlock blocks, and a feasibility flag. A synthetic union has w == -1 until
> + the weight sweep replaces it with its contained weight. */
> +
> +struct bundle_cand
> +{
> + class loop *loop;
> + basic_block lock, unlock;
> + HOST_WIDE_INT lo, hi;
> + gcov_type w;
> + int p;
> + bool feasible;
> +};
> +
> +/* Sort by hi ascending. Break ties by lo descending (a contained candidate
> + before its container), then by real loops before a synthetic union of the
> + same span. */
> +
> +static int
> +bundle_cand_cmp_hi (const void *a, const void *b)
> +{
> + const bundle_cand *x = (const bundle_cand *)a, *y = (const bundle_cand *)b;
> + if (x->hi != y->hi)
> + return x->hi < y->hi ? -1 : 1;
> + if (x->lo != y->lo)
> + return x->lo > y->lo ? -1 : 1;
> + if ((x->w == -1) != (y->w == -1))
> + return x->w == -1 ? 1 : -1;
> + return 0;
> +}
> +
> +/* qsort comparator for HOST_WIDE_INT, ascending (lo-value compression). */
> +
> +static int
> +bundle_hwi_cmp (const void *a, const void *b)
> +{
> + HOST_WIDE_INT x = *(const HOST_WIDE_INT *)a, y = *(const HOST_WIDE_INT *)b;
> + return x < y ? -1 : x > y ? 1 : 0;
> +}
> +
> +/* Less-than on HOST_WIDE_INT, for vec::lower_bound over the compressed
> + lo-value list. */
> +
> +static bool
> +bundle_hwi_less (const HOST_WIDE_INT &a, const HOST_WIDE_INT &b)
> +{
> + return a < b;
> +}
> +
> +/* Less-than on interval end, for vec::lower_bound in the WIS predecessor
> + search over the hi-sorted feasible set. */
> +
> +static bool
> +bundle_cand_hi_less (const bundle_cand &a, const bundle_cand &b)
> +{
> + return a.hi < b.hi;
> +}
> +
> +static void
> +bundle_compute_layout (HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
> + int *asm_lo, int *asm_hi)
> +{
> + HOST_WIDE_INT off = 0;
> + int nasm = 0;
> + basic_block bb;
> + FOR_EACH_BB_FN (bb, cfun)
> + {
> + rtx_insn *insn;
> + bb_lo[bb->index] = off;
> + asm_lo[bb->index] = nasm;
> + FOR_BB_INSNS (bb, insn)
> + if (NONDEBUG_INSN_P (insn))
> + {
> + /* Inline asm has no reliable min size; count its presence so any
> + candidate spanning this block can be excluded outright. */
> + if (asm_noperands (PATTERN (insn)) >= 0)
> + nasm++;
> + else
> + off += ix86_min_insn_size (insn);
> + }
> + bb_hi[bb->index] = off;
> + asm_hi[bb->index] = nasm;
> + }
> +}
> +
> +static void
> +bundle_collect_candidates (vec<bundle_cand> &feas, HOST_WIDE_INT k,
> + HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
> + int *asm_lo, int *asm_hi)
> +{
> + /* Step 1: identify feasible candidates from layout-backward loop edges. */
> + basic_block bb;
> + FOR_EACH_BB_FN (bb, cfun)
> + {
> + edge e;
> + edge_iterator ei;
> + FOR_EACH_EDGE (e, ei, bb->succs)
> + if (e->count ().initialized_p ()
> + && e->dest->index >= NUM_FIXED_BLOCKS
> + && (e->dest == bb
> + || bb_lo[e->dest->index] < bb_lo[bb->index]))
> + {
> + bundle_cand c = {};
> + c.loop = bb->loop_father;
> + c.lock = e->dest;
> + c.unlock = bb;
> + c.lo = bb_lo[e->dest->index];
> + c.hi = bb_hi[bb->index];
> + c.w = e->count ().to_gcov_type ();
> + /* Both ends must lie in a real loop, not the function-wide fake
> + loop whose latch is EXIT. This rejects a layout-backward exit
> + edge whose destination is outside any loop. */
> + c.feasible = (c.loop
> + && c.loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
> + && e->dest->loop_father
> + && (e->dest->loop_father->latch
> + != EXIT_BLOCK_PTR_FOR_FN (cfun))
> + && asm_hi[bb->index] == asm_lo[e->dest->index]
> + && c.hi - c.lo <= k
> + && optimize_bb_for_speed_p (c.lock));
> + if (c.feasible)
> + feas.safe_push (c);
> + }
> + }
> +}
> +
> +static void
> +bundle_add_unions (vec<bundle_cand> &feas, HOST_WIDE_INT k)
> +{
> + /* Step 2: add synthetic unions for candidates sharing a K-byte window. */
> + feas.qsort (bundle_cand_cmp_hi);
> + unsigned base = feas.length ();
> + /* At most one union is added per candidate, so reserve now; this keeps the
> + &feas[i] pointers held by the heap valid across the push below. */
> + feas.reserve (2 * base);
> + fibonacci_heap<HOST_WIDE_INT, bundle_cand> heap (HOST_WIDE_INT_MIN);
> + for (unsigned i = 0; i < base; i++)
> + {
> + heap.insert (feas[i].lo, &feas[i]);
> + HOST_WIDE_INT thr = feas[i].hi - k;
> + while (heap.min_key () < thr)
> + heap.extract_min ();
> + if (heap.nodes () >= 2)
> + {
> + bundle_cand *s = heap.min ();
> + bundle_cand c = {};
> + c.loop = s->loop;
> + c.lock = s->lock;
> + c.unlock = feas[i].unlock;
> + c.lo = s->lo;
> + c.hi = feas[i].hi;
> + c.w = -1; /* Union; the weight sweep fills this in. */
> + c.feasible = true;
> + feas.safe_push (c);
> + }
> + }
> +}
> +
> +static void
> +bundle_compute_weights (vec<bundle_cand> &feas)
> +{
> + /* Step 3: compute containment weights with a Fenwick tree. */
> + feas.qsort (bundle_cand_cmp_hi);
> + unsigned n_all = feas.length ();
> + /* Compress lo values into a sorted, unique list; POS below maps a lo to its
> + reversed rank via lower_bound. */
> + auto_vec<HOST_WIDE_INT> los;
> + los.reserve (n_all);
> + for (unsigned p = 0; p < n_all; p++)
> + los.quick_push (feas[p].lo);
> + los.qsort (bundle_hwi_cmp);
> + int m = 0;
> + for (unsigned p = 0; p < n_all; p++)
> + if (m == 0 || los[m - 1] != los[p])
> + los[m++] = los[p];
> + los.truncate (m);
> + /* BIT[1..m] keyed by reversed lo-rank, so a prefix sum [1..pos] covers
> + exactly lo_j >= lo_i. */
> + gcov_type *bit = XCNEWVEC (gcov_type, m + 1);
> + for (unsigned p = 0; p < n_all; p++)
> + {
> + /* Reversed rank of feas[p].lo: LOWER_BOUND is the first LOS >= lo. */
> + int pos = m - (int) los.lower_bound (feas[p].lo, bundle_hwi_less);
> + /* A union is not a real loop, so do not insert it into the tree. Its
> + weight is the contained sum, less one to break ties toward it. */
> + bool u = feas[p].w == -1;
> + if (!u)
> + for (int i = pos; i <= m; i += i & -i)
> + bit[i] += feas[p].w;
> + gcov_type sum = 0;
> + for (int i = pos; i > 0; i -= i & -i)
> + sum += bit[i];
> + feas[p].w = u ? sum - 1 : sum;
> + }
> + XDELETEVEC (bit);
> +}
> +
> +static void
> +bundle_select_and_emit (vec<bundle_cand> &feas)
> +{
> + /* Step 4: select the maximum-weight disjoint set. */
> + unsigned n = feas.length ();
> + /* dp[i+1] is the best weight over feas[0..i]. Whether i is taken is
> + recomputed while backtracking rather than stored. */
> + gcov_type *dp = XCNEWVEC (gcov_type, n + 1);
> +
> + for (unsigned i = 0; i < n; i++)
> + {
> + /* Predecessor: the last candidate that ends before feas[i] starts.
> + hi is inclusive, so find the first candidate with hi >= lo. */
> + bundle_cand key = {};
> + key.hi = feas[i].lo;
> + int p = (int) feas.lower_bound (key, bundle_cand_hi_less) - 1;
> + feas[i].p = p;
> + gcov_type take = feas[i].w + dp[p + 1], skip = dp[i];
> + dp[i + 1] = take >= skip ? take : skip;
> + if (dump_file)
> + fprintf (dump_file,
> + "bundle-align: cand %u loop %d [bb%d..bb%d] "
> + "lo=%ld hi=%ld w=%ld p=%d take=%ld skip=%ld => %s\n",
> + i, feas[i].loop->num, feas[i].lock->index,
> + feas[i].unlock->index, (long) feas[i].lo,
> + (long) feas[i].hi, (long) feas[i].w, p,
> + (long) take, (long) skip, take >= skip ? "take" : "skip");
> + }
> +
> + /* Step 5: emit the selected regions. */
> + for (int i = (int) n - 1; i >= 0;)
> + if (feas[i].w + dp[feas[i].p + 1] >= dp[i])
> + {
> + basic_block h = feas[i].lock, l = feas[i].unlock;
> + emit_insn_before (gen_bundle_lock (), BB_HEAD (h));
> + emit_insn_after (gen_bundle_unlock (), BB_END (l));
> + if (dump_file)
> + fprintf (dump_file,
> + "bundle-align: wrap loop %d [bb%d..bb%d] "
> + "w=%ld span=%ld\n", feas[i].loop->num, h->index,
> + l->index, (long) feas[i].w,
> + (long) (feas[i].hi - feas[i].lo));
> + i = feas[i].p;
> + }
> + else
> + i--;
> +
> + XDELETEVEC (dp);
> +}
> +
> +/* Wrap the selected disjoint hot loop regions in .bundle_lock/.bundle_unlock.
> + Each region is an interval on the layout byte axis, which handles rotated
> + (latch-before-header) loops uniformly. */
> +
> +static void
> +ix86_do_bundle_align_loops ()
> +{
> + if (ix86_cost->prefetch_block == 0)
> + return;
> +
> + /* Region size cap: tuning param if set, else the CPU bundle size. */
> + HOST_WIDE_INT k = (param_x86_tight_loop_max_size > 0
> + ? (HOST_WIDE_INT)param_x86_tight_loop_max_size
> + : (HOST_WIDE_INT)ix86_cost->prefetch_block);
> +
> + loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
> +
> + /* Per-block layout byte offsets [bb_lo, bb_hi). */
> + int nbb = last_basic_block_for_fn (cfun);
> + HOST_WIDE_INT *bb_lo = XCNEWVEC (HOST_WIDE_INT, nbb);
> + HOST_WIDE_INT *bb_hi = XCNEWVEC (HOST_WIDE_INT, nbb);
> + /* Running count of inline-asm insns in layout order; [asm_lo, asm_hi) lets a
> + candidate test whether any block it spans contains an asm. */
> + int *asm_lo = XCNEWVEC (int, nbb);
> + int *asm_hi = XCNEWVEC (int, nbb);
> + bundle_compute_layout (bb_lo, bb_hi, asm_lo, asm_hi);
> +
> + auto_vec<bundle_cand> feas;
> + bundle_collect_candidates (feas, k, bb_lo, bb_hi, asm_lo, asm_hi);
> + bundle_add_unions (feas, k);
> + bundle_compute_weights (feas);
> + if (!feas.is_empty ())
> + bundle_select_and_emit (feas);
> +
> + XDELETEVEC (bb_lo);
> + XDELETEVEC (bb_hi);
> + XDELETEVEC (asm_lo);
> + XDELETEVEC (asm_hi);
> + loop_optimizer_finalize ();
> +}
> +
> namespace {
>
> -const pass_data pass_data_align_tight_loops =
> +const pass_data pass_data_bundle_align_loops =
> {
> RTL_PASS, /* type */
> - "align_tight_loops", /* name */
> + "bundle_align_loops", /* name */
> OPTGROUP_NONE, /* optinfo_flags */
> TV_MACH_DEP, /* tv_id */
> 0, /* properties_required */
> @@ -5661,40 +5964,44 @@ const pass_data pass_data_align_tight_loops =
> 0, /* todo_flags_finish */
> };
>
> -class pass_align_tight_loops : public rtl_opt_pass
> +class pass_bundle_align_loops : public rtl_opt_pass
> {
> public:
> - pass_align_tight_loops (gcc::context *ctxt)
> - : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
> + pass_bundle_align_loops (gcc::context *ctxt)
> + : rtl_opt_pass (pass_data_bundle_align_loops, ctxt)
> {}
>
> /* opt_pass methods: */
> bool gate (function *) final override
> {
> - return TARGET_ALIGN_TIGHT_LOOPS
> - && optimize
> - && optimize_function_for_speed_p (cfun);
> + return optimize
> + && optimize_function_for_speed_p (cfun)
> + && (ix86_bundle_align_loops > 0 || TARGET_ALIGN_TIGHT_LOOPS);
> }
>
> unsigned int execute (function *) final override
> {
> timevar_push (TV_MACH_DEP);
> + if (ix86_bundle_align_loops > 0)
> + ix86_do_bundle_align_loops ();
> #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
> - ix86_align_loops ();
> + else
> + ix86_align_loops ();
> #endif
> timevar_pop (TV_MACH_DEP);
> return 0;
> }
> -}; // class pass_align_tight_loops
> +}; // class pass_bundle_align_loops
>
> } // anon namespace
>
> rtl_opt_pass *
> -make_pass_align_tight_loops (gcc::context *ctxt)
> +make_pass_bundle_align_loops (gcc::context *ctxt)
> {
> - return new pass_align_tight_loops (ctxt);
> + return new pass_bundle_align_loops (ctxt);
> }
>
> +
> /* This compares the priority of target features in function DECL1
> and DECL2. It returns positive value if DECL1 is higher priority,
> negative value if DECL2 is higher priority and 0 if they are the
> diff --git a/gcc/config/i386/i386-options.cc b/gcc/config/i386/i386-options.cc
> index e7f06154103..db0a9426b29 100644
> --- a/gcc/config/i386/i386-options.cc
> +++ b/gcc/config/i386/i386-options.cc
> @@ -1872,8 +1872,16 @@ set_ix86_tune_features (struct gcc_options *opts,
> /* Default align_* from the processor table. */
>
> static void
> -ix86_default_align (struct gcc_options *opts)
> +ix86_default_align (struct gcc_options *opts,
> + struct gcc_options *opts_set)
> {
> + /* Bundle alignment replaces generic loop, jump, and label padding. */
> + if (opts->x_ix86_bundle_align_loops > 0)
> + {
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_loops, "1");
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_jumps, "1");
> + SET_OPTION_IF_UNSET (opts, opts_set, str_align_labels, "1");
If we just set loop/jump/label align to 1, and keep align tight loops, can we get similar
code size reduction without performance drop?
It sounds to me bundle alignments is an enhancement to current tightloop alignment with
dynamic decision in assembler instead of hard-coded .palign. so what's the real gain of
the new pass against the original align_tight_loop?
The pass manually setting all other alignments to 1, if the assembler doesn't support
the new directive .bundle_{lock, unlock}, do we have a way to fallback? or just lose
all alignment with some performance loss?
> + }
> /* -falign-foo without argument: supply one. */
> if (opts->x_flag_align_loops && !opts->x_str_align_loops)
> opts->x_str_align_loops = processor_cost_table[ix86_tune]->align_loop;
> @@ -1885,6 +1893,16 @@ ix86_default_align (struct gcc_options *opts)
> opts->x_str_align_functions = processor_cost_table[ix86_tune]->align_func;
> }
>
> +/* Default -mbundle-align-loops from the active tune model. */
> +
> +static void
> +ix86_default_bundle_align (struct gcc_options *opts)
> +{
> + if (opts->x_ix86_bundle_align_loops == -1)
> + opts->x_ix86_bundle_align_loops
> + = ix86_tune_features[X86_TUNE_BUNDLE_ALIGN_LOOPS] ? 1 : 0;
> +}
> +
> #ifndef USE_IX86_FRAME_POINTER
> #define USE_IX86_FRAME_POINTER 0
> #endif
> @@ -1986,7 +2004,8 @@ ix86_override_options_after_change_1 (struct gcc_options *opts,
> void
> ix86_override_options_after_change (void)
> {
> - ix86_default_align (&global_options);
> + ix86_default_bundle_align (&global_options);
> + ix86_default_align (&global_options, &global_options_set);
>
> ix86_recompute_optlev_based_flags (&global_options, &global_options_set);
>
> @@ -2619,7 +2638,8 @@ ix86_option_override_internal (bool main_args_p,
> opts->x_ix86_regparm = REGPARM_MAX;
>
> /* Default align_* from the processor table. */
> - ix86_default_align (&global_options);
> + ix86_default_bundle_align (&global_options);
> + ix86_default_align (&global_options, &global_options_set);
>
> /* Provide default for -mbranch-cost= value. */
> SET_OPTION_IF_UNSET (opts, opts_set, ix86_branch_cost,
> diff --git a/gcc/config/i386/i386-passes.def b/gcc/config/i386/i386-passes.def
> index 4f461116cac..82d4d7411df 100644
> --- a/gcc/config/i386/i386-passes.def
> +++ b/gcc/config/i386/i386-passes.def
> @@ -31,9 +31,9 @@ along with GCC; see the file COPYING3. If not see
> INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */);
>
> INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_insert_endbr_and_patchable_area);
> - /* pass_align_tight_loops must be after pass_insert_endbr_and_patchable_area.
> - PR116174. */
> - INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_align_tight_loops);
> + /* Tight/bundle loop aligner; after pass_insert_endbr_and_patchable_area
> + (PR116174) and before pass_shorten_branches. */
> + INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_bundle_align_loops);
>
> INSERT_PASS_AFTER (pass_late_combine, 1, pass_x86_cse);
> INSERT_PASS_AFTER (pass_late_combine, 1, pass_remove_partial_avx_dependency);
> diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
> index 2687de646b0..84d2a7ad74c 100644
> --- a/gcc/config/i386/i386-protos.h
> +++ b/gcc/config/i386/i386-protos.h
> @@ -434,7 +434,7 @@ extern rtl_opt_pass *make_pass_remove_partial_avx_dependency
> (gcc::context *);
> extern rtl_opt_pass *make_pass_x86_cse (gcc::context *);
> extern rtl_opt_pass *make_pass_apx_nf_convert (gcc::context *);
> -extern rtl_opt_pass *make_pass_align_tight_loops (gcc::context *);
> +extern rtl_opt_pass *make_pass_bundle_align_loops (gcc::context *);
>
> extern bool ix86_has_no_direct_extern_access;
> extern bool ix86_rpad_gate ();
> diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
> index 5768a93d5b0..f85010a41aa 100644
> --- a/gcc/config/i386/i386.md
> +++ b/gcc/config/i386/i386.md
> @@ -233,6 +233,8 @@
> UNSPECV_STACK_PROBE
> UNSPECV_PROBE_STACK_RANGE
> UNSPECV_ALIGN
> + UNSPECV_BUNDLE_LOCK
> + UNSPECV_BUNDLE_UNLOCK
> UNSPECV_PROLOGUE_USE
> UNSPECV_SPLIT_STACK_RETURN
> UNSPECV_CLD
> @@ -21756,6 +21758,39 @@
> }
> [(set_attr "length" "16")])
>
> +;; ix86_bundle_align_loops pass: align the locked region's start to the cache
> +;; line (prefetch_block). Final instruction sizes may grow during assembly
> +;; because of indirect calls, prefixes, and relaxation, so a selected region
> +;; can exceed one bundle; flex .bundle_lock accepts such a region instead of
> +;; raising an error. Requires flex .bundle_lock support.
> +(define_insn "bundle_lock"
> + [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_LOCK)]
> + ""
> +{
> +#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
> + fprintf (asm_out_file, "\t.bundle_align_mode %d\n\t.bundle_lock flex\n",
> + ceil_log2 (ix86_cost->prefetch_block));
> +#endif
> + return "";
> +}
> + [(set_attr "length" "0")
> + (set_attr "length_immediate" "0")
> + (set_attr "modrm" "0")])
> +
> +;; Close the bundle and turn bundle alignment mode back off.
> +(define_insn "bundle_unlock"
> + [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_UNLOCK)]
> + ""
> +{
> +#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
> + fputs ("\t.bundle_unlock\n\t.bundle_align_mode 0\n", asm_out_file);
> +#endif
> + return "";
> +}
> + [(set_attr "length" "0")
> + (set_attr "length_immediate" "0")
> + (set_attr "modrm" "0")])
> +
> (define_expand "prologue"
> [(const_int 0)]
> ""
> diff --git a/gcc/config/i386/i386.opt b/gcc/config/i386/i386.opt
> index 57ba3fbfa8f..148d26b6053 100644
> --- a/gcc/config/i386/i386.opt
> +++ b/gcc/config/i386/i386.opt
> @@ -240,6 +240,10 @@ malign-loops=
> Target Undocumented RejectNegative Joined UInteger
> Loop code aligned to this power of 2.
>
> +mbundle-align-loops
> +Target Var(ix86_bundle_align_loops) Init(-1) Save
> +Wrap small hot loops with .bundle_lock so the assembler keeps them inside one cache line.
> +
> malign-stringops
> Target RejectNegative InverseMask(NO_ALIGN_STRINGOPS, ALIGN_STRINGOPS) Save
> Align destination of the string operations.
> @@ -1221,6 +1225,14 @@ Do not use GOT to access external symbols.
> Target Joined UInteger Var(x86_stlf_window_ninsns) Init(64) Param
> Instructions number above which STFL stall penalty can be compensated.
>
> +-param=x86-tight-loop-max-size=
> +Target Joined UInteger Var(param_x86_tight_loop_max_size) Init(0) Param
> +Maximum body size in bytes for the i386 tight-loop aligner; 0 picks the active tune prefetch_block.
> +
> +-param=x86-bundle-align-branch-limit=
> +Target Joined UInteger Var(param_x86_bundle_align_branch_limit) Init(2) Param
> +Maximum internal conditional branches allowed in a bundle-locked loop region (the back-edge branch is not counted).
> +
> mavxifma
> Target Mask(ISA2_AVXIFMA) Var(ix86_isa_flags2) Save
> Support MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, AVX, AVX2, and
> diff --git a/gcc/config/i386/i386.opt.urls b/gcc/config/i386/i386.opt.urls
> index d52e6de8014..20a638ea1f7 100644
> --- a/gcc/config/i386/i386.opt.urls
> +++ b/gcc/config/i386/i386.opt.urls
> @@ -28,6 +28,9 @@ UrlSuffix(gcc/x86-Options.html#index-malign-double)
>
> ; skipping UrlSuffix for 'malign-loops=' due to finding no URLs
>
> +mbundle-align-loops
> +UrlSuffix(gcc/x86-Options.html#index-mbundle-align-loops)
> +
> malign-stringops
> UrlSuffix(gcc/x86-Options.html#index-malign-stringops)
>
> diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
> index d4c27351ad7..4716f26b2e1 100644
> --- a/gcc/config/i386/x86-tune.def
> +++ b/gcc/config/i386/x86-tune.def
> @@ -259,6 +259,13 @@ DEF_TUNE (X86_TUNE_ALIGN_TIGHT_LOOPS, "align_tight_loops",
> ~(m_ZHAOXIN | m_CASCADELAKE | m_SKYLAKE_AVX512
> | m_CORE_ATOM))
>
> +/* X86_TUNE_BUNDLE_ALIGN_LOOPS: when true, use the ix86_bundle_align_loops
> + pass with the assembler's .bundle_lock/.bundle_unlock directives instead
> + of static .p2align padding. Targets cores where loops straddling a 64-byte
> + instruction-cache line are particularly costly (AMD Zen4/Zen5/Zen6). */
> +DEF_TUNE (X86_TUNE_BUNDLE_ALIGN_LOOPS, "bundle_align_loops",
> + m_ZNVER4 | m_ZNVER5 | m_ZNVER6)
> +
> /*****************************************************************************/
> /* Integer instruction selection tuning */
> /*****************************************************************************/
> diff --git a/gcc/configure b/gcc/configure
> index 43ca45144b8..dd5a0c0a539 100755
> --- a/gcc/configure
> +++ b/gcc/configure
> @@ -26306,6 +26306,44 @@ $as_echo "#define HAVE_GAS_MAX_SKIP_P2ALIGN 1" >>confdefs.h
> fi
>
>
> +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .bundle_lock flex argument" >&5
> +$as_echo_n "checking assembler for .bundle_lock flex argument... " >&6; }
> +if ${gcc_cv_as_bundle_flex_lock+:} false; then :
> + $as_echo_n "(cached) " >&6
> +else
> + gcc_cv_as_bundle_flex_lock=no
> + if test x"$gcc_cv_as" != x; then
> + $as_echo '.bundle_align_mode 6
> +.bundle_lock flex
> +.rept 100
> +nop
> +.endr
> +.bundle_unlock' > conftest.s
> + if { ac_try='$gcc_cv_as $gcc_cv_as_flags -o conftest.o conftest.s >&5'
> + { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5
> + (eval $ac_try) 2>&5
> + ac_status=$?
> + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
> + test $ac_status = 0; }; }
> + then
> + gcc_cv_as_bundle_flex_lock=yes
> + else
> + echo "configure: failed program was" >&5
> + cat conftest.s >&5
> + fi
> + rm -f conftest.o conftest.s
> + fi
> +fi
> +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_as_bundle_flex_lock" >&5
> +$as_echo "$gcc_cv_as_bundle_flex_lock" >&6; }
> +if test $gcc_cv_as_bundle_flex_lock = yes; then
> +
> +$as_echo "#define HAVE_GAS_BUNDLE_FLEX_LOCK 1" >>confdefs.h
> +
> +fi
> +
> +
> +
> { $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .literal16" >&5
> $as_echo_n "checking assembler for .literal16... " >&6; }
> if ${gcc_cv_as_literal16+:} false; then :
> diff --git a/gcc/configure.ac b/gcc/configure.ac
> index 075b6b68ffa..346bf254de3 100644
> --- a/gcc/configure.ac
> +++ b/gcc/configure.ac
> @@ -3142,6 +3142,17 @@ gcc_GAS_CHECK_FEATURE([.p2align with maximum skip], gcc_cv_as_max_skip_p2align,,
> [Define if your assembler supports specifying the maximum number
> of bytes to skip when using the GAS .p2align command.])])
>
> +gcc_GAS_CHECK_FEATURE([.bundle_lock flex argument], gcc_cv_as_bundle_flex_lock,,
> + [.bundle_align_mode 6
> +.bundle_lock flex
> +.rept 100
> +nop
> +.endr
> +.bundle_unlock],,
> +[AC_DEFINE(HAVE_GAS_BUNDLE_FLEX_LOCK, 1,
> + [Define if your assembler accepts the flex argument to .bundle_lock
> + that permits bundles larger than .bundle_align_mode alignment.])])
> +
> gcc_GAS_CHECK_FEATURE([.literal16], gcc_cv_as_literal16,,
> [.literal16],,
> [AC_DEFINE(HAVE_GAS_LITERAL16, 1,
> diff --git a/gcc/doc/extend.texi b/gcc/doc/extend.texi
> index 397b05ab87d..9081317927d 100644
> --- a/gcc/doc/extend.texi
> +++ b/gcc/doc/extend.texi
> @@ -9598,6 +9598,12 @@ operations and calling the library routines for large operations.
> @itemx no-align-stringops
> Do/do not align destination of inlined string operations.
>
> +@atindex @code{target("bundle-align-loops")}, x86
> +@item bundle-align-loops
> +@itemx no-bundle-align-loops
> +Enable/disable wrapping of small hot loops in assembler
> +@code{.bundle_lock}/@code{.bundle_unlock} directives.
> +
> @atindex @code{target("recip")}, x86
> @item recip
> @itemx no-recip
> diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
> index e1870c9c227..619f92185fd 100644
> --- a/gcc/doc/invoke.texi
> +++ b/gcc/doc/invoke.texi
> @@ -1554,7 +1554,8 @@ See RS/6000 and PowerPC Options.
> -mamx-complex -mavxvnniint16 -msm3 -msha512 -msm4 -mapxf
> -musermsr -mavx10.1 -mavx10.2 -mamx-avx512 -mmovrs -mamx-movrs
> -mavx512bmm -mavx10v2aux
> --mcldemote -mms-bitfields -mno-align-stringops -minline-all-stringops
> +-mcldemote -mms-bitfields -mbundle-align-loops -mno-align-stringops
> +-minline-all-stringops
> -minline-stringops-dynamically -mstringop-strategy=@var{alg}
> -mkl -mwidekl
> -mmemcpy-strategy=@var{strategy} -mmemset-strategy=@var{strategy}
> @@ -37155,6 +37156,18 @@ Here, @code{t5} takes up 2 bytes.
> @end enumerate
>
>
> +@opindex mbundle-align-loops
> +@opindex mno-bundle-align-loops
> +@item -mbundle-align-loops
> +@itemx -mno-bundle-align-loops
> +Wrap small hot loops in assembler @code{.bundle_lock}/@code{.bundle_unlock}
> +directives so the assembler keeps each loop within a single instruction-cache
> +line, aligning the loop header without inserting executed padding. Enabled by
> +default for AMD Zen4 and later. It defaults @option{-falign-loops},
> +@option{-falign-jumps}, and @option{-falign-labels} to 1 unless explicitly set.
> +The maximum loop-body size considered is set by the
> +@option{x86-tight-loop-max-size} parameter.
> +
> @opindex mno-align-stringops
> @opindex malign-stringops
> @item -mno-align-stringops
> diff --git a/gcc/doc/sourcebuild.texi b/gcc/doc/sourcebuild.texi
> index aad51550eba..35324653699 100644
> --- a/gcc/doc/sourcebuild.texi
> +++ b/gcc/doc/sourcebuild.texi
> @@ -3114,6 +3114,10 @@ Target supports @option{-fstack-protector}.
> @item gas
> Target uses GNU @command{as}.
>
> +@item gcc_bundle_align_loops
> +Target compiler and assembler support @option{-mbundle-align-loops} and
> +@code{.bundle_lock flex}.
> +
> @item gc_sections
> Target supports @option{--gc-sections}.
>
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-1.c b/gcc/testsuite/gcc.target/i386/bundle-align-1.c
> new file mode 100644
> index 00000000000..c7120731f5f
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-1.c
> @@ -0,0 +1,16 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +typedef int (*callee_t) (int, int *);
> +
> +int
> +indirect_sum (callee_t callee, int n, int *array, long calls)
> +{
> + int sink = 0;
> + for (long call = 0; call < calls; ++call)
> + sink += callee (n, array);
> + return sink;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-2.c b/gcc/testsuite/gcc.target/i386/bundle-align-2.c
> new file mode 100644
> index 00000000000..957e853dfb2
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-2.c
> @@ -0,0 +1,21 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +__attribute__((noinline,noipa))
> +int
> +f (int n, int *p)
> +{
> + int i = 0, s = 0;
> + if (n & 1)
> + goto mid;
> + while (i < n)
> + {
> + s += p[i & 1023];
> + mid:
> + i++;
> + }
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-3.c b/gcc/testsuite/gcc.target/i386/bundle-align-3.c
> new file mode 100644
> index 00000000000..7cdfa8db985
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-3.c
> @@ -0,0 +1,26 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +int a[1024];
> +__attribute__((hot,noinline,noipa))
> +int
> +two_backedges (int n, int m)
> +{
> + int i = 0, s = 0;
> +outer_head:
> + {
> + int j = 0;
> + inner_head:
> + s += a[(i + j) & 1023];
> + if (__builtin_expect (s < 0, 0))
> + goto outer_head;
> + if (++j < m)
> + goto inner_head;
> + }
> + if (++i < n)
> + goto outer_head;
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-4.c b/gcc/testsuite/gcc.target/i386/bundle-align-4.c
> new file mode 100644
> index 00000000000..80bca4ec200
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-4.c
> @@ -0,0 +1,22 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver5 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
> +/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
> +
> +volatile int sink;
> +int a[1024];
> +int
> +classify (int n)
> +{
> + int s = 0;
> + asm volatile (".rept 0\n\t.byte 0x90\n\t.endr" ::: "memory");
> + for (int i = 0; i < n; i++)
> + {
> + if (a[i] & 1) sink = i;
> + if (a[i] & 2) sink = i + 1;
> + if (a[i] & 4) s += 4;
> + s += a[i];
> + }
> + return s;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-5.c b/gcc/testsuite/gcc.target/i386/bundle-align-5.c
> new file mode 100644
> index 00000000000..7aa07effa6e
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-5.c
> @@ -0,0 +1,17 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +void *mc (const void *s, int c, unsigned long n)
> +{
> + const unsigned char *p = s;
> + while (n--)
> + {
> + if (*p == (unsigned char) c)
> + return (void *) p;
> + p++;
> + }
> + return 0;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-6.c b/gcc/testsuite/gcc.target/i386/bundle-align-6.c
> new file mode 100644
> index 00000000000..1f561e07caa
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-6.c
> @@ -0,0 +1,24 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
> +/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
> +
> +struct O { long rc; };
> +extern void dealloc (struct O *);
> +void
> +f (struct O **v, long n)
> +{
> + asm volatile (".rept 10\n\t.byte 0x90\n\t.endr" ::: "memory");
> + while (--n >= 0)
> + {
> + struct O *x = v[n];
> + if (x)
> + {
> + if (x->rc < 0)
> + ;
> + else if (--x->rc == 0)
> + dealloc (x);
> + }
> + }
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/bundle-align-7.c b/gcc/testsuite/gcc.target/i386/bundle-align-7.c
> new file mode 100644
> index 00000000000..b713d25d0b7
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/bundle-align-7.c
> @@ -0,0 +1,17 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O1 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
> +/* { dg-require-effective-target gcc_bundle_align_loops } */
> +/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
> +/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
> +
> +static int A[1 << 20];
> +
> +int
> +nested_sum (int n, int m)
> +{
> + int sum = 0;
> + for (int i = 0; i < n; ++i)
> + for (int j = 0; j < m; ++j)
> + sum += A[i * m + j];
> + return sum;
> +}
> diff --git a/gcc/testsuite/lib/target-supports.exp b/gcc/testsuite/lib/target-supports.exp
> index d467e9fa3b7..3286c40b07d 100644
> --- a/gcc/testsuite/lib/target-supports.exp
> +++ b/gcc/testsuite/lib/target-supports.exp
> @@ -12491,6 +12491,22 @@ proc check_effective_target_gas { } {
> return $use_gas_saved
> }
>
> +# Return 1 if the compiler was configured with assembler support for
> +# the flex .bundle_lock argument used by -mbundle-align-loops.
> +
> +proc check_effective_target_gcc_bundle_align_loops { } {
> + return [check_no_messages_and_pattern gcc_bundle_align_loops \
> + "\\.bundle_lock flex" assembly {
> + int a[1024];
> + int foo (int n) {
> + int s = 0;
> + for (int i = 0; i < n; i++)
> + s += a[i];
> + return s;
> + }
> + } "-O2 -march=znver4 -mbundle-align-loops -falign-loops=1"]
> +}
> +
> # Return 1 if GNU ld is used.
>
> proc check_effective_target_gld { } {
> --
> 2.34.1
On Mon, Oct 12, 2026 at 10:57 PM, Hongtao Liu <crazylht@gmail.com> wrote:
> On Sat Oct 3, 2026 at 7:02 AM PDT, Sarvesh Chandra wrote:
> > + /* Bundle alignment replaces generic loop, jump, and label padding. */
> > + if (opts->x_ix86_bundle_align_loops > 0)
> > + {
> > + SET_OPTION_IF_UNSET (opts, opts_set, str_align_loops, "1");
> > + SET_OPTION_IF_UNSET (opts, opts_set, str_align_jumps, "1");
> > + SET_OPTION_IF_UNSET (opts, opts_set, str_align_labels, "1");
>
> If we just set loop/jump/label align to 1, and keep align tight loops,
> can we get similar code size reduction without performance drop?
We do get similar code size reduction, but not without a performance
drop on znver5. I am attaching the Cauldron page here, which contains
the slides referring to the impact of bundle align against tight align
in isolation:
https://conf.gnu-tools-cauldron.org/prg26/talk/HS9RPC/
> It sounds to me bundle alignments is an enhancement to current tightloop
> alignment with dynamic decision in assembler instead of hard-coded
> .palign. so what's the real gain of the new pass against the original
> align_tight_loop?
Bundle not only wins due to hardcoded palign, but also due to the
crossing hot regions present in layout which need smarter insertion of
padding. We saw the impact of this in the microbenchmarks in the
Appendix section as well as the tupledealloc loop of cpython_r, which
had commit-to-commit variation, fixed by bundle align which identifies
the entire region in one bundle, vs tight which chooses one of them.
> The pass manually setting all other alignments to 1, if the assembler
> doesn't support the new directive .bundle_{lock, unlock}, do we have a
> way to fallback? or just lose all alignment with some performance loss?
The fallback when the assembler doesn't support flex bundles is the
current default, that is, 16B loop/jump/label align + tight align,
because we could not rule out the impact of any of the four flags on
znver5. In the absence of bundle align, we can expect performance
impacted negatively on frontend bound benchmarks. There are cases when
setting loop/jump/label align to 1 also has a negative impact, in both
real world benchmarks and microbenchmarks. Bundle align is one way to
address all of these regressions under one umbrella.
On Mon, Oct 5, 2026 at 10:08 AM, Richard Biener <richard.guenther@gmail.com> wrote:
> On Sat, Oct 3, 2026 at 4:03 PM Sarvesh Chandra wrote:
> > +static void
> > +bundle_compute_layout (HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
> > + int *asm_lo, int *asm_hi)
> > +{
> > + HOST_WIDE_INT off = 0;
> > + int nasm = 0;
> > + basic_block bb;
> > + FOR_EACH_BB_FN (bb, cfun)
>
> can you avoid doing this for non-candidate loop blocks?
We need to do this for non-candidate blocks too, as the interval
[lo, hi] for a candidate may contain non-candidate blocks. But we
could start counting at the first block that has a valid loop father.
> > +bundle_collect_candidates (vec<bundle_cand> &feas, HOST_WIDE_INT k,
> > + HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
> > + int *asm_lo, int *asm_hi)
> > +{
> > + /* Step 1: identify feasible candidates from layout-backward loop edges. */
> > + basic_block bb;
> > + FOR_EACH_BB_FN (bb, cfun)
>
> I wonder if you can instead walk the loop tree?
This is a layout-based optimization, trying to find hot contiguous
layout regions which can fit in one bundle, so a loop-tree traversal
won't identify our candidates.
> > +ix86_do_bundle_align_loops ()
> > +{
> > + if (ix86_cost->prefetch_block == 0)
>
> || number_of_loops (cfun) <= 1
>
> > + return;
I will add this in v2, while also moving this from RFC to PATCH.
Thanks Richard and Hongtao!
Regards,
Sarvesh
@@ -1448,6 +1448,12 @@
#endif
+/* Define if your assembler accepts the flex argument to .bundle_lock. */
+#ifndef USED_FOR_TARGET
+#undef HAVE_GAS_BUNDLE_FLEX_LOCK
+#endif
+
+
/* Define 0/1 if your assembler supports CFI directives. */
#undef HAVE_GAS_CFI_DIRECTIVE
@@ -83,6 +83,7 @@ along with GCC; see the file COPYING3. If not see
#include "ifcvt.h"
#include "symbol-summary.h"
#include "sreal.h"
+#include "fibonacci_heap.h"
#include "ipa-cp.h"
#include "ipa-prop.h"
#include "ipa-fnsummary.h"
@@ -5646,12 +5647,314 @@ ix86_align_loops ()
free_dominance_info (CDI_DOMINATORS);
}
+/* ix86_bundle_align_loops pass.
+
+ 1. Compute layout offsets and identify feasible loop-backward edges.
+ 2. Add synthetic unions for candidates sharing a K-byte window.
+ 3. Weight candidates by the back-edge counts of loops contained in
+ them.
+ 4. Select a maximum-weight disjoint set with weighted interval
+ scheduling.
+ 5. Emit .bundle_lock flex for the selected regions.
+
+ K is param_x86_tight_loop_max_size, or ix86_cost->prefetch_block when
+ unset. */
+
+/* A candidate loop region: the byte interval [lo, hi] on the layout axis
+ (hi inclusive), weight w, interval-scheduling predecessor p, the lock and
+ unlock blocks, and a feasibility flag. A synthetic union has w == -1 until
+ the weight sweep replaces it with its contained weight. */
+
+struct bundle_cand
+{
+ class loop *loop;
+ basic_block lock, unlock;
+ HOST_WIDE_INT lo, hi;
+ gcov_type w;
+ int p;
+ bool feasible;
+};
+
+/* Sort by hi ascending. Break ties by lo descending (a contained candidate
+ before its container), then by real loops before a synthetic union of the
+ same span. */
+
+static int
+bundle_cand_cmp_hi (const void *a, const void *b)
+{
+ const bundle_cand *x = (const bundle_cand *)a, *y = (const bundle_cand *)b;
+ if (x->hi != y->hi)
+ return x->hi < y->hi ? -1 : 1;
+ if (x->lo != y->lo)
+ return x->lo > y->lo ? -1 : 1;
+ if ((x->w == -1) != (y->w == -1))
+ return x->w == -1 ? 1 : -1;
+ return 0;
+}
+
+/* qsort comparator for HOST_WIDE_INT, ascending (lo-value compression). */
+
+static int
+bundle_hwi_cmp (const void *a, const void *b)
+{
+ HOST_WIDE_INT x = *(const HOST_WIDE_INT *)a, y = *(const HOST_WIDE_INT *)b;
+ return x < y ? -1 : x > y ? 1 : 0;
+}
+
+/* Less-than on HOST_WIDE_INT, for vec::lower_bound over the compressed
+ lo-value list. */
+
+static bool
+bundle_hwi_less (const HOST_WIDE_INT &a, const HOST_WIDE_INT &b)
+{
+ return a < b;
+}
+
+/* Less-than on interval end, for vec::lower_bound in the WIS predecessor
+ search over the hi-sorted feasible set. */
+
+static bool
+bundle_cand_hi_less (const bundle_cand &a, const bundle_cand &b)
+{
+ return a.hi < b.hi;
+}
+
+static void
+bundle_compute_layout (HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
+ int *asm_lo, int *asm_hi)
+{
+ HOST_WIDE_INT off = 0;
+ int nasm = 0;
+ basic_block bb;
+ FOR_EACH_BB_FN (bb, cfun)
+ {
+ rtx_insn *insn;
+ bb_lo[bb->index] = off;
+ asm_lo[bb->index] = nasm;
+ FOR_BB_INSNS (bb, insn)
+ if (NONDEBUG_INSN_P (insn))
+ {
+ /* Inline asm has no reliable min size; count its presence so any
+ candidate spanning this block can be excluded outright. */
+ if (asm_noperands (PATTERN (insn)) >= 0)
+ nasm++;
+ else
+ off += ix86_min_insn_size (insn);
+ }
+ bb_hi[bb->index] = off;
+ asm_hi[bb->index] = nasm;
+ }
+}
+
+static void
+bundle_collect_candidates (vec<bundle_cand> &feas, HOST_WIDE_INT k,
+ HOST_WIDE_INT *bb_lo, HOST_WIDE_INT *bb_hi,
+ int *asm_lo, int *asm_hi)
+{
+ /* Step 1: identify feasible candidates from layout-backward loop edges. */
+ basic_block bb;
+ FOR_EACH_BB_FN (bb, cfun)
+ {
+ edge e;
+ edge_iterator ei;
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ if (e->count ().initialized_p ()
+ && e->dest->index >= NUM_FIXED_BLOCKS
+ && (e->dest == bb
+ || bb_lo[e->dest->index] < bb_lo[bb->index]))
+ {
+ bundle_cand c = {};
+ c.loop = bb->loop_father;
+ c.lock = e->dest;
+ c.unlock = bb;
+ c.lo = bb_lo[e->dest->index];
+ c.hi = bb_hi[bb->index];
+ c.w = e->count ().to_gcov_type ();
+ /* Both ends must lie in a real loop, not the function-wide fake
+ loop whose latch is EXIT. This rejects a layout-backward exit
+ edge whose destination is outside any loop. */
+ c.feasible = (c.loop
+ && c.loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
+ && e->dest->loop_father
+ && (e->dest->loop_father->latch
+ != EXIT_BLOCK_PTR_FOR_FN (cfun))
+ && asm_hi[bb->index] == asm_lo[e->dest->index]
+ && c.hi - c.lo <= k
+ && optimize_bb_for_speed_p (c.lock));
+ if (c.feasible)
+ feas.safe_push (c);
+ }
+ }
+}
+
+static void
+bundle_add_unions (vec<bundle_cand> &feas, HOST_WIDE_INT k)
+{
+ /* Step 2: add synthetic unions for candidates sharing a K-byte window. */
+ feas.qsort (bundle_cand_cmp_hi);
+ unsigned base = feas.length ();
+ /* At most one union is added per candidate, so reserve now; this keeps the
+ &feas[i] pointers held by the heap valid across the push below. */
+ feas.reserve (2 * base);
+ fibonacci_heap<HOST_WIDE_INT, bundle_cand> heap (HOST_WIDE_INT_MIN);
+ for (unsigned i = 0; i < base; i++)
+ {
+ heap.insert (feas[i].lo, &feas[i]);
+ HOST_WIDE_INT thr = feas[i].hi - k;
+ while (heap.min_key () < thr)
+ heap.extract_min ();
+ if (heap.nodes () >= 2)
+ {
+ bundle_cand *s = heap.min ();
+ bundle_cand c = {};
+ c.loop = s->loop;
+ c.lock = s->lock;
+ c.unlock = feas[i].unlock;
+ c.lo = s->lo;
+ c.hi = feas[i].hi;
+ c.w = -1; /* Union; the weight sweep fills this in. */
+ c.feasible = true;
+ feas.safe_push (c);
+ }
+ }
+}
+
+static void
+bundle_compute_weights (vec<bundle_cand> &feas)
+{
+ /* Step 3: compute containment weights with a Fenwick tree. */
+ feas.qsort (bundle_cand_cmp_hi);
+ unsigned n_all = feas.length ();
+ /* Compress lo values into a sorted, unique list; POS below maps a lo to its
+ reversed rank via lower_bound. */
+ auto_vec<HOST_WIDE_INT> los;
+ los.reserve (n_all);
+ for (unsigned p = 0; p < n_all; p++)
+ los.quick_push (feas[p].lo);
+ los.qsort (bundle_hwi_cmp);
+ int m = 0;
+ for (unsigned p = 0; p < n_all; p++)
+ if (m == 0 || los[m - 1] != los[p])
+ los[m++] = los[p];
+ los.truncate (m);
+ /* BIT[1..m] keyed by reversed lo-rank, so a prefix sum [1..pos] covers
+ exactly lo_j >= lo_i. */
+ gcov_type *bit = XCNEWVEC (gcov_type, m + 1);
+ for (unsigned p = 0; p < n_all; p++)
+ {
+ /* Reversed rank of feas[p].lo: LOWER_BOUND is the first LOS >= lo. */
+ int pos = m - (int) los.lower_bound (feas[p].lo, bundle_hwi_less);
+ /* A union is not a real loop, so do not insert it into the tree. Its
+ weight is the contained sum, less one to break ties toward it. */
+ bool u = feas[p].w == -1;
+ if (!u)
+ for (int i = pos; i <= m; i += i & -i)
+ bit[i] += feas[p].w;
+ gcov_type sum = 0;
+ for (int i = pos; i > 0; i -= i & -i)
+ sum += bit[i];
+ feas[p].w = u ? sum - 1 : sum;
+ }
+ XDELETEVEC (bit);
+}
+
+static void
+bundle_select_and_emit (vec<bundle_cand> &feas)
+{
+ /* Step 4: select the maximum-weight disjoint set. */
+ unsigned n = feas.length ();
+ /* dp[i+1] is the best weight over feas[0..i]. Whether i is taken is
+ recomputed while backtracking rather than stored. */
+ gcov_type *dp = XCNEWVEC (gcov_type, n + 1);
+
+ for (unsigned i = 0; i < n; i++)
+ {
+ /* Predecessor: the last candidate that ends before feas[i] starts.
+ hi is inclusive, so find the first candidate with hi >= lo. */
+ bundle_cand key = {};
+ key.hi = feas[i].lo;
+ int p = (int) feas.lower_bound (key, bundle_cand_hi_less) - 1;
+ feas[i].p = p;
+ gcov_type take = feas[i].w + dp[p + 1], skip = dp[i];
+ dp[i + 1] = take >= skip ? take : skip;
+ if (dump_file)
+ fprintf (dump_file,
+ "bundle-align: cand %u loop %d [bb%d..bb%d] "
+ "lo=%ld hi=%ld w=%ld p=%d take=%ld skip=%ld => %s\n",
+ i, feas[i].loop->num, feas[i].lock->index,
+ feas[i].unlock->index, (long) feas[i].lo,
+ (long) feas[i].hi, (long) feas[i].w, p,
+ (long) take, (long) skip, take >= skip ? "take" : "skip");
+ }
+
+ /* Step 5: emit the selected regions. */
+ for (int i = (int) n - 1; i >= 0;)
+ if (feas[i].w + dp[feas[i].p + 1] >= dp[i])
+ {
+ basic_block h = feas[i].lock, l = feas[i].unlock;
+ emit_insn_before (gen_bundle_lock (), BB_HEAD (h));
+ emit_insn_after (gen_bundle_unlock (), BB_END (l));
+ if (dump_file)
+ fprintf (dump_file,
+ "bundle-align: wrap loop %d [bb%d..bb%d] "
+ "w=%ld span=%ld\n", feas[i].loop->num, h->index,
+ l->index, (long) feas[i].w,
+ (long) (feas[i].hi - feas[i].lo));
+ i = feas[i].p;
+ }
+ else
+ i--;
+
+ XDELETEVEC (dp);
+}
+
+/* Wrap the selected disjoint hot loop regions in .bundle_lock/.bundle_unlock.
+ Each region is an interval on the layout byte axis, which handles rotated
+ (latch-before-header) loops uniformly. */
+
+static void
+ix86_do_bundle_align_loops ()
+{
+ if (ix86_cost->prefetch_block == 0)
+ return;
+
+ /* Region size cap: tuning param if set, else the CPU bundle size. */
+ HOST_WIDE_INT k = (param_x86_tight_loop_max_size > 0
+ ? (HOST_WIDE_INT)param_x86_tight_loop_max_size
+ : (HOST_WIDE_INT)ix86_cost->prefetch_block);
+
+ loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
+
+ /* Per-block layout byte offsets [bb_lo, bb_hi). */
+ int nbb = last_basic_block_for_fn (cfun);
+ HOST_WIDE_INT *bb_lo = XCNEWVEC (HOST_WIDE_INT, nbb);
+ HOST_WIDE_INT *bb_hi = XCNEWVEC (HOST_WIDE_INT, nbb);
+ /* Running count of inline-asm insns in layout order; [asm_lo, asm_hi) lets a
+ candidate test whether any block it spans contains an asm. */
+ int *asm_lo = XCNEWVEC (int, nbb);
+ int *asm_hi = XCNEWVEC (int, nbb);
+ bundle_compute_layout (bb_lo, bb_hi, asm_lo, asm_hi);
+
+ auto_vec<bundle_cand> feas;
+ bundle_collect_candidates (feas, k, bb_lo, bb_hi, asm_lo, asm_hi);
+ bundle_add_unions (feas, k);
+ bundle_compute_weights (feas);
+ if (!feas.is_empty ())
+ bundle_select_and_emit (feas);
+
+ XDELETEVEC (bb_lo);
+ XDELETEVEC (bb_hi);
+ XDELETEVEC (asm_lo);
+ XDELETEVEC (asm_hi);
+ loop_optimizer_finalize ();
+}
+
namespace {
-const pass_data pass_data_align_tight_loops =
+const pass_data pass_data_bundle_align_loops =
{
RTL_PASS, /* type */
- "align_tight_loops", /* name */
+ "bundle_align_loops", /* name */
OPTGROUP_NONE, /* optinfo_flags */
TV_MACH_DEP, /* tv_id */
0, /* properties_required */
@@ -5661,40 +5964,44 @@ const pass_data pass_data_align_tight_loops =
0, /* todo_flags_finish */
};
-class pass_align_tight_loops : public rtl_opt_pass
+class pass_bundle_align_loops : public rtl_opt_pass
{
public:
- pass_align_tight_loops (gcc::context *ctxt)
- : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
+ pass_bundle_align_loops (gcc::context *ctxt)
+ : rtl_opt_pass (pass_data_bundle_align_loops, ctxt)
{}
/* opt_pass methods: */
bool gate (function *) final override
{
- return TARGET_ALIGN_TIGHT_LOOPS
- && optimize
- && optimize_function_for_speed_p (cfun);
+ return optimize
+ && optimize_function_for_speed_p (cfun)
+ && (ix86_bundle_align_loops > 0 || TARGET_ALIGN_TIGHT_LOOPS);
}
unsigned int execute (function *) final override
{
timevar_push (TV_MACH_DEP);
+ if (ix86_bundle_align_loops > 0)
+ ix86_do_bundle_align_loops ();
#ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
- ix86_align_loops ();
+ else
+ ix86_align_loops ();
#endif
timevar_pop (TV_MACH_DEP);
return 0;
}
-}; // class pass_align_tight_loops
+}; // class pass_bundle_align_loops
} // anon namespace
rtl_opt_pass *
-make_pass_align_tight_loops (gcc::context *ctxt)
+make_pass_bundle_align_loops (gcc::context *ctxt)
{
- return new pass_align_tight_loops (ctxt);
+ return new pass_bundle_align_loops (ctxt);
}
+
/* This compares the priority of target features in function DECL1
and DECL2. It returns positive value if DECL1 is higher priority,
negative value if DECL2 is higher priority and 0 if they are the
@@ -1872,8 +1872,16 @@ set_ix86_tune_features (struct gcc_options *opts,
/* Default align_* from the processor table. */
static void
-ix86_default_align (struct gcc_options *opts)
+ix86_default_align (struct gcc_options *opts,
+ struct gcc_options *opts_set)
{
+ /* Bundle alignment replaces generic loop, jump, and label padding. */
+ if (opts->x_ix86_bundle_align_loops > 0)
+ {
+ SET_OPTION_IF_UNSET (opts, opts_set, str_align_loops, "1");
+ SET_OPTION_IF_UNSET (opts, opts_set, str_align_jumps, "1");
+ SET_OPTION_IF_UNSET (opts, opts_set, str_align_labels, "1");
+ }
/* -falign-foo without argument: supply one. */
if (opts->x_flag_align_loops && !opts->x_str_align_loops)
opts->x_str_align_loops = processor_cost_table[ix86_tune]->align_loop;
@@ -1885,6 +1893,16 @@ ix86_default_align (struct gcc_options *opts)
opts->x_str_align_functions = processor_cost_table[ix86_tune]->align_func;
}
+/* Default -mbundle-align-loops from the active tune model. */
+
+static void
+ix86_default_bundle_align (struct gcc_options *opts)
+{
+ if (opts->x_ix86_bundle_align_loops == -1)
+ opts->x_ix86_bundle_align_loops
+ = ix86_tune_features[X86_TUNE_BUNDLE_ALIGN_LOOPS] ? 1 : 0;
+}
+
#ifndef USE_IX86_FRAME_POINTER
#define USE_IX86_FRAME_POINTER 0
#endif
@@ -1986,7 +2004,8 @@ ix86_override_options_after_change_1 (struct gcc_options *opts,
void
ix86_override_options_after_change (void)
{
- ix86_default_align (&global_options);
+ ix86_default_bundle_align (&global_options);
+ ix86_default_align (&global_options, &global_options_set);
ix86_recompute_optlev_based_flags (&global_options, &global_options_set);
@@ -2619,7 +2638,8 @@ ix86_option_override_internal (bool main_args_p,
opts->x_ix86_regparm = REGPARM_MAX;
/* Default align_* from the processor table. */
- ix86_default_align (&global_options);
+ ix86_default_bundle_align (&global_options);
+ ix86_default_align (&global_options, &global_options_set);
/* Provide default for -mbranch-cost= value. */
SET_OPTION_IF_UNSET (opts, opts_set, ix86_branch_cost,
@@ -31,9 +31,9 @@ along with GCC; see the file COPYING3. If not see
INSERT_PASS_BEFORE (pass_cse2, 1, pass_stv, true /* timode_p */);
INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_insert_endbr_and_patchable_area);
- /* pass_align_tight_loops must be after pass_insert_endbr_and_patchable_area.
- PR116174. */
- INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_align_tight_loops);
+ /* Tight/bundle loop aligner; after pass_insert_endbr_and_patchable_area
+ (PR116174) and before pass_shorten_branches. */
+ INSERT_PASS_BEFORE (pass_shorten_branches, 1, pass_bundle_align_loops);
INSERT_PASS_AFTER (pass_late_combine, 1, pass_x86_cse);
INSERT_PASS_AFTER (pass_late_combine, 1, pass_remove_partial_avx_dependency);
@@ -434,7 +434,7 @@ extern rtl_opt_pass *make_pass_remove_partial_avx_dependency
(gcc::context *);
extern rtl_opt_pass *make_pass_x86_cse (gcc::context *);
extern rtl_opt_pass *make_pass_apx_nf_convert (gcc::context *);
-extern rtl_opt_pass *make_pass_align_tight_loops (gcc::context *);
+extern rtl_opt_pass *make_pass_bundle_align_loops (gcc::context *);
extern bool ix86_has_no_direct_extern_access;
extern bool ix86_rpad_gate ();
@@ -233,6 +233,8 @@
UNSPECV_STACK_PROBE
UNSPECV_PROBE_STACK_RANGE
UNSPECV_ALIGN
+ UNSPECV_BUNDLE_LOCK
+ UNSPECV_BUNDLE_UNLOCK
UNSPECV_PROLOGUE_USE
UNSPECV_SPLIT_STACK_RETURN
UNSPECV_CLD
@@ -21756,6 +21758,39 @@
}
[(set_attr "length" "16")])
+;; ix86_bundle_align_loops pass: align the locked region's start to the cache
+;; line (prefetch_block). Final instruction sizes may grow during assembly
+;; because of indirect calls, prefixes, and relaxation, so a selected region
+;; can exceed one bundle; flex .bundle_lock accepts such a region instead of
+;; raising an error. Requires flex .bundle_lock support.
+(define_insn "bundle_lock"
+ [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_LOCK)]
+ ""
+{
+#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
+ fprintf (asm_out_file, "\t.bundle_align_mode %d\n\t.bundle_lock flex\n",
+ ceil_log2 (ix86_cost->prefetch_block));
+#endif
+ return "";
+}
+ [(set_attr "length" "0")
+ (set_attr "length_immediate" "0")
+ (set_attr "modrm" "0")])
+
+;; Close the bundle and turn bundle alignment mode back off.
+(define_insn "bundle_unlock"
+ [(unspec_volatile [(const_int 0)] UNSPECV_BUNDLE_UNLOCK)]
+ ""
+{
+#ifdef HAVE_GAS_BUNDLE_FLEX_LOCK
+ fputs ("\t.bundle_unlock\n\t.bundle_align_mode 0\n", asm_out_file);
+#endif
+ return "";
+}
+ [(set_attr "length" "0")
+ (set_attr "length_immediate" "0")
+ (set_attr "modrm" "0")])
+
(define_expand "prologue"
[(const_int 0)]
""
@@ -240,6 +240,10 @@ malign-loops=
Target Undocumented RejectNegative Joined UInteger
Loop code aligned to this power of 2.
+mbundle-align-loops
+Target Var(ix86_bundle_align_loops) Init(-1) Save
+Wrap small hot loops with .bundle_lock so the assembler keeps them inside one cache line.
+
malign-stringops
Target RejectNegative InverseMask(NO_ALIGN_STRINGOPS, ALIGN_STRINGOPS) Save
Align destination of the string operations.
@@ -1221,6 +1225,14 @@ Do not use GOT to access external symbols.
Target Joined UInteger Var(x86_stlf_window_ninsns) Init(64) Param
Instructions number above which STFL stall penalty can be compensated.
+-param=x86-tight-loop-max-size=
+Target Joined UInteger Var(param_x86_tight_loop_max_size) Init(0) Param
+Maximum body size in bytes for the i386 tight-loop aligner; 0 picks the active tune prefetch_block.
+
+-param=x86-bundle-align-branch-limit=
+Target Joined UInteger Var(param_x86_bundle_align_branch_limit) Init(2) Param
+Maximum internal conditional branches allowed in a bundle-locked loop region (the back-edge branch is not counted).
+
mavxifma
Target Mask(ISA2_AVXIFMA) Var(ix86_isa_flags2) Save
Support MMX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2, AVX, AVX2, and
@@ -28,6 +28,9 @@ UrlSuffix(gcc/x86-Options.html#index-malign-double)
; skipping UrlSuffix for 'malign-loops=' due to finding no URLs
+mbundle-align-loops
+UrlSuffix(gcc/x86-Options.html#index-mbundle-align-loops)
+
malign-stringops
UrlSuffix(gcc/x86-Options.html#index-malign-stringops)
@@ -259,6 +259,13 @@ DEF_TUNE (X86_TUNE_ALIGN_TIGHT_LOOPS, "align_tight_loops",
~(m_ZHAOXIN | m_CASCADELAKE | m_SKYLAKE_AVX512
| m_CORE_ATOM))
+/* X86_TUNE_BUNDLE_ALIGN_LOOPS: when true, use the ix86_bundle_align_loops
+ pass with the assembler's .bundle_lock/.bundle_unlock directives instead
+ of static .p2align padding. Targets cores where loops straddling a 64-byte
+ instruction-cache line are particularly costly (AMD Zen4/Zen5/Zen6). */
+DEF_TUNE (X86_TUNE_BUNDLE_ALIGN_LOOPS, "bundle_align_loops",
+ m_ZNVER4 | m_ZNVER5 | m_ZNVER6)
+
/*****************************************************************************/
/* Integer instruction selection tuning */
/*****************************************************************************/
@@ -26306,6 +26306,44 @@ $as_echo "#define HAVE_GAS_MAX_SKIP_P2ALIGN 1" >>confdefs.h
fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .bundle_lock flex argument" >&5
+$as_echo_n "checking assembler for .bundle_lock flex argument... " >&6; }
+if ${gcc_cv_as_bundle_flex_lock+:} false; then :
+ $as_echo_n "(cached) " >&6
+else
+ gcc_cv_as_bundle_flex_lock=no
+ if test x"$gcc_cv_as" != x; then
+ $as_echo '.bundle_align_mode 6
+.bundle_lock flex
+.rept 100
+nop
+.endr
+.bundle_unlock' > conftest.s
+ if { ac_try='$gcc_cv_as $gcc_cv_as_flags -o conftest.o conftest.s >&5'
+ { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_try\""; } >&5
+ (eval $ac_try) 2>&5
+ ac_status=$?
+ $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5
+ test $ac_status = 0; }; }
+ then
+ gcc_cv_as_bundle_flex_lock=yes
+ else
+ echo "configure: failed program was" >&5
+ cat conftest.s >&5
+ fi
+ rm -f conftest.o conftest.s
+ fi
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $gcc_cv_as_bundle_flex_lock" >&5
+$as_echo "$gcc_cv_as_bundle_flex_lock" >&6; }
+if test $gcc_cv_as_bundle_flex_lock = yes; then
+
+$as_echo "#define HAVE_GAS_BUNDLE_FLEX_LOCK 1" >>confdefs.h
+
+fi
+
+
+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking assembler for .literal16" >&5
$as_echo_n "checking assembler for .literal16... " >&6; }
if ${gcc_cv_as_literal16+:} false; then :
@@ -3142,6 +3142,17 @@ gcc_GAS_CHECK_FEATURE([.p2align with maximum skip], gcc_cv_as_max_skip_p2align,,
[Define if your assembler supports specifying the maximum number
of bytes to skip when using the GAS .p2align command.])])
+gcc_GAS_CHECK_FEATURE([.bundle_lock flex argument], gcc_cv_as_bundle_flex_lock,,
+ [.bundle_align_mode 6
+.bundle_lock flex
+.rept 100
+nop
+.endr
+.bundle_unlock],,
+[AC_DEFINE(HAVE_GAS_BUNDLE_FLEX_LOCK, 1,
+ [Define if your assembler accepts the flex argument to .bundle_lock
+ that permits bundles larger than .bundle_align_mode alignment.])])
+
gcc_GAS_CHECK_FEATURE([.literal16], gcc_cv_as_literal16,,
[.literal16],,
[AC_DEFINE(HAVE_GAS_LITERAL16, 1,
@@ -9598,6 +9598,12 @@ operations and calling the library routines for large operations.
@itemx no-align-stringops
Do/do not align destination of inlined string operations.
+@atindex @code{target("bundle-align-loops")}, x86
+@item bundle-align-loops
+@itemx no-bundle-align-loops
+Enable/disable wrapping of small hot loops in assembler
+@code{.bundle_lock}/@code{.bundle_unlock} directives.
+
@atindex @code{target("recip")}, x86
@item recip
@itemx no-recip
@@ -1554,7 +1554,8 @@ See RS/6000 and PowerPC Options.
-mamx-complex -mavxvnniint16 -msm3 -msha512 -msm4 -mapxf
-musermsr -mavx10.1 -mavx10.2 -mamx-avx512 -mmovrs -mamx-movrs
-mavx512bmm -mavx10v2aux
--mcldemote -mms-bitfields -mno-align-stringops -minline-all-stringops
+-mcldemote -mms-bitfields -mbundle-align-loops -mno-align-stringops
+-minline-all-stringops
-minline-stringops-dynamically -mstringop-strategy=@var{alg}
-mkl -mwidekl
-mmemcpy-strategy=@var{strategy} -mmemset-strategy=@var{strategy}
@@ -37155,6 +37156,18 @@ Here, @code{t5} takes up 2 bytes.
@end enumerate
+@opindex mbundle-align-loops
+@opindex mno-bundle-align-loops
+@item -mbundle-align-loops
+@itemx -mno-bundle-align-loops
+Wrap small hot loops in assembler @code{.bundle_lock}/@code{.bundle_unlock}
+directives so the assembler keeps each loop within a single instruction-cache
+line, aligning the loop header without inserting executed padding. Enabled by
+default for AMD Zen4 and later. It defaults @option{-falign-loops},
+@option{-falign-jumps}, and @option{-falign-labels} to 1 unless explicitly set.
+The maximum loop-body size considered is set by the
+@option{x86-tight-loop-max-size} parameter.
+
@opindex mno-align-stringops
@opindex malign-stringops
@item -mno-align-stringops
@@ -3114,6 +3114,10 @@ Target supports @option{-fstack-protector}.
@item gas
Target uses GNU @command{as}.
+@item gcc_bundle_align_loops
+Target compiler and assembler support @option{-mbundle-align-loops} and
+@code{.bundle_lock flex}.
+
@item gc_sections
Target supports @option{--gc-sections}.
new file mode 100644
@@ -0,0 +1,16 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
+
+typedef int (*callee_t) (int, int *);
+
+int
+indirect_sum (callee_t callee, int n, int *array, long calls)
+{
+ int sink = 0;
+ for (long call = 0; call < calls; ++call)
+ sink += callee (n, array);
+ return sink;
+}
new file mode 100644
@@ -0,0 +1,21 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
+
+__attribute__((noinline,noipa))
+int
+f (int n, int *p)
+{
+ int i = 0, s = 0;
+ if (n & 1)
+ goto mid;
+ while (i < n)
+ {
+ s += p[i & 1023];
+ mid:
+ i++;
+ }
+ return s;
+}
new file mode 100644
@@ -0,0 +1,26 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
+
+int a[1024];
+__attribute__((hot,noinline,noipa))
+int
+two_backedges (int n, int m)
+{
+ int i = 0, s = 0;
+outer_head:
+ {
+ int j = 0;
+ inner_head:
+ s += a[(i + j) & 1023];
+ if (__builtin_expect (s < 0, 0))
+ goto outer_head;
+ if (++j < m)
+ goto inner_head;
+ }
+ if (++i < n)
+ goto outer_head;
+ return s;
+}
new file mode 100644
@@ -0,0 +1,22 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver5 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
+/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
+
+volatile int sink;
+int a[1024];
+int
+classify (int n)
+{
+ int s = 0;
+ asm volatile (".rept 0\n\t.byte 0x90\n\t.endr" ::: "memory");
+ for (int i = 0; i < n; i++)
+ {
+ if (a[i] & 1) sink = i;
+ if (a[i] & 2) sink = i + 1;
+ if (a[i] & 4) s += 4;
+ s += a[i];
+ }
+ return s;
+}
new file mode 100644
@@ -0,0 +1,17 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
+
+void *mc (const void *s, int c, unsigned long n)
+{
+ const unsigned char *p = s;
+ while (n--)
+ {
+ if (*p == (unsigned char) c)
+ return (void *) p;
+ p++;
+ }
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,24 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler-times "\\.bundle_lock flex" 1 } } */
+/* { dg-final { scan-assembler-times "\\.bundle_unlock" 1 } } */
+
+struct O { long rc; };
+extern void dealloc (struct O *);
+void
+f (struct O **v, long n)
+{
+ asm volatile (".rept 10\n\t.byte 0x90\n\t.endr" ::: "memory");
+ while (--n >= 0)
+ {
+ struct O *x = v[n];
+ if (x)
+ {
+ if (x->rc < 0)
+ ;
+ else if (--x->rc == 0)
+ dealloc (x);
+ }
+ }
+}
new file mode 100644
@@ -0,0 +1,17 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O1 -march=znver3 -falign-loops=1 -falign-jumps=1 -falign-labels=1 -mbundle-align-loops" } */
+/* { dg-require-effective-target gcc_bundle_align_loops } */
+/* { dg-final { scan-assembler "\\.bundle_lock flex" } } */
+/* { dg-final { scan-assembler "\\.bundle_unlock" } } */
+
+static int A[1 << 20];
+
+int
+nested_sum (int n, int m)
+{
+ int sum = 0;
+ for (int i = 0; i < n; ++i)
+ for (int j = 0; j < m; ++j)
+ sum += A[i * m + j];
+ return sum;
+}
@@ -12491,6 +12491,22 @@ proc check_effective_target_gas { } {
return $use_gas_saved
}
+# Return 1 if the compiler was configured with assembler support for
+# the flex .bundle_lock argument used by -mbundle-align-loops.
+
+proc check_effective_target_gcc_bundle_align_loops { } {
+ return [check_no_messages_and_pattern gcc_bundle_align_loops \
+ "\\.bundle_lock flex" assembly {
+ int a[1024];
+ int foo (int n) {
+ int s = 0;
+ for (int i = 0; i < n; i++)
+ s += a[i];
+ return s;
+ }
+ } "-O2 -march=znver4 -mbundle-align-loops -falign-loops=1"]
+}
+
# Return 1 if GNU ld is used.
proc check_effective_target_gld { } {