diff mbox series

[vect] : unshare header computation for more CSE

Message ID patch-20682-tamar@arm.com
State Committed
Commit 09021961ea027196cafca0d0308fd7c5e997bff8
Headers
Series [vect] : unshare header computation for more CSE |

Checks

Context Check Description
linaro-tcwg-bot/tcwg_gcc_build--master-arm success Build passed
linaro-tcwg-bot/tcwg_gcc_build--master-aarch64 success Build passed

Commit Message

Tamar Christina July 7, 2026, 10:12 a.m. UTC 
  The example loop 

#define N 512
#define START 1
#define END 505

int x[N] __attribute__((aligned(32)));

int __attribute__((noipa))
foo (void)
{
  for (int *p = x + START; p < x + END; ++p)
    if (*p == 0)
      return START;

  return -1;
}

shows that when we vectorize we generate the following pre-header when doing
peeling for alignment:

  _22 = (unsigned long) &MEM <int[512]> [(void *)&x + 4B];
  _23 = _22 & POLY_INT_CST [15, 16];
  _24 = _23 >> 2;
  _26 = 1 - _24;
  _27 = _26 * 4;
  vectp_x.6_25 = &x + _27;
  _36 = 504 + _24;
  max_mask_37 = .WHILE_ULT (0, _36, { 0, ... });
  _38 = .WHILE_ULT (0, _24, { 0, ... });
  _39 = ~_38;
  _40 = max_mask_37 & _39;

notice how _22 and and the actual vector pointer vectp_x.6_25 start at the
same offset but have a different base.  This happens because in
vect_create_addr_base_for_vector_ref we fold the offset into the base very
early and so we can't form or share the address computation anymore with the
scalar address which needs to be there for the alignment checks.

The patch delays this and forms an explicit base + offset and keeping base as
a separate value.  This allows VN at the end of vect to share the computations
and we get

  _22 = (unsigned long) &MEM <int[512]> [(void *)&x + 4B];
  _23 = _22 & POLY_INT_CST [15, 16];
  _24 = _23 >> 2;
  _27 = _24 * 4;
  _28 = -_27;
  vectp_x.6_26 = &MEM <int[512]> [(void *)&x + 4B] + _28;
  _37 = 504 + _24;
  max_mask_38 = .WHILE_ULT (0, _37, { 0, ... });
  _39 = .WHILE_ULT (0, _24, { 0, ... });
  _40 = ~_39;
  _41 = max_mask_38 & _40;

Notice how they now share the same base address.  This drops two instructions
off the loop pre-header:

from

foo:
        cntb    x0
        sub     x3, x0, #1
        cmp     x0, 4096
        and     x0, x0, x3
        ccmp    x0, 0, 0, ls
        bne     .L7
        adrp    x4, .LANCHOR0
        add     x4, x4, :lo12:.LANCHOR0
        add     x1, x4, 4
        mov     w2, 1
        and     x1, x1, x3
        mov     w0, 0
        cntw    x3
        lsr     x1, x1, 2
        whilelo p15.s, xzr, x1
        sub     x2, x2, x1
        add     x1, x1, 504
        whilelo p7.s, xzr, x1
        not     p7.b, p7/z, p15.b
        add     x2, x4, x2, lsl 2
        b       .L4

to

foo:
        cntb    x0
        sub     x1, x0, #1
        cmp     x0, 4096
        and     x0, x0, x1
        adrp    x2, .LANCHOR0
        ccmp    x0, 0, 0, ls
        add     x2, x2, :lo12:.LANCHOR0
        bne     .L7
        add     x2, x2, 4
        mov     w0, 0
        and     x1, x2, x1
        cntw    x3
        sub     x2, x2, x1
        lsr     x1, x1, 2
        whilelo p15.s, xzr, x1
        add     x1, x1, 504
        whilelo p7.s, xzr, x1
        not     p7.b, p7/z, p15.b
        b       .L4

Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.

Any feedback?

Thanks,
Tamar

gcc/ChangeLog:

	* tree-vect-data-refs.cc (vect_create_addr_base_for_vector_ref):
	Restructure computations to force separate base.

gcc/testsuite/ChangeLog:

	* gcc.target/aarch64/sve/vect-early-break-cbranch_18.c: New test.

---


--

Comments

Richard Biener July 7, 2026, 11:15 a.m. UTC | #1 
On Tue, 7 Jul 2026, Tamar Christina wrote:

> The example loop 
> 
> #define N 512
> #define START 1
> #define END 505
> 
> int x[N] __attribute__((aligned(32)));
> 
> int __attribute__((noipa))
> foo (void)
> {
>   for (int *p = x + START; p < x + END; ++p)
>     if (*p == 0)
>       return START;
> 
>   return -1;
> }
> 
> shows that when we vectorize we generate the following pre-header when doing
> peeling for alignment:
> 
>   _22 = (unsigned long) &MEM <int[512]> [(void *)&x + 4B];
>   _23 = _22 & POLY_INT_CST [15, 16];
>   _24 = _23 >> 2;
>   _26 = 1 - _24;
>   _27 = _26 * 4;
>   vectp_x.6_25 = &x + _27;
>   _36 = 504 + _24;
>   max_mask_37 = .WHILE_ULT (0, _36, { 0, ... });
>   _38 = .WHILE_ULT (0, _24, { 0, ... });
>   _39 = ~_38;
>   _40 = max_mask_37 & _39;
> 
> notice how _22 and and the actual vector pointer vectp_x.6_25 start at the
> same offset but have a different base.  This happens because in
> vect_create_addr_base_for_vector_ref we fold the offset into the base very
> early and so we can't form or share the address computation anymore with the
> scalar address which needs to be there for the alignment checks.
> 
> The patch delays this and forms an explicit base + offset and keeping base as
> a separate value.  This allows VN at the end of vect to share the computations
> and we get
> 
>   _22 = (unsigned long) &MEM <int[512]> [(void *)&x + 4B];
>   _23 = _22 & POLY_INT_CST [15, 16];
>   _24 = _23 >> 2;
>   _27 = _24 * 4;
>   _28 = -_27;
>   vectp_x.6_26 = &MEM <int[512]> [(void *)&x + 4B] + _28;
>   _37 = 504 + _24;
>   max_mask_38 = .WHILE_ULT (0, _37, { 0, ... });
>   _39 = .WHILE_ULT (0, _24, { 0, ... });
>   _40 = ~_39;
>   _41 = max_mask_38 & _40;
> 
> Notice how they now share the same base address.  This drops two instructions
> off the loop pre-header:
> 
> from
> 
> foo:
>         cntb    x0
>         sub     x3, x0, #1
>         cmp     x0, 4096
>         and     x0, x0, x3
>         ccmp    x0, 0, 0, ls
>         bne     .L7
>         adrp    x4, .LANCHOR0
>         add     x4, x4, :lo12:.LANCHOR0
>         add     x1, x4, 4
>         mov     w2, 1
>         and     x1, x1, x3
>         mov     w0, 0
>         cntw    x3
>         lsr     x1, x1, 2
>         whilelo p15.s, xzr, x1
>         sub     x2, x2, x1
>         add     x1, x1, 504
>         whilelo p7.s, xzr, x1
>         not     p7.b, p7/z, p15.b
>         add     x2, x4, x2, lsl 2
>         b       .L4
> 
> to
> 
> foo:
>         cntb    x0
>         sub     x1, x0, #1
>         cmp     x0, 4096
>         and     x0, x0, x1
>         adrp    x2, .LANCHOR0
>         ccmp    x0, 0, 0, ls
>         add     x2, x2, :lo12:.LANCHOR0
>         bne     .L7
>         add     x2, x2, 4
>         mov     w0, 0
>         and     x1, x2, x1
>         cntw    x3
>         sub     x2, x2, x1
>         lsr     x1, x1, 2
>         whilelo p15.s, xzr, x1
>         add     x1, x1, 504
>         whilelo p7.s, xzr, x1
>         not     p7.b, p7/z, p15.b
>         b       .L4
> 
> Bootstrapped Regtested on aarch64-none-linux-gnu,
> arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
> -m32, -m64 and no issues.
> 
> Any feedback?

OK.

Richard.

> Thanks,
> Tamar
> 
> gcc/ChangeLog:
> 
> 	* tree-vect-data-refs.cc (vect_create_addr_base_for_vector_ref):
> 	Restructure computations to force separate base.
> 
> gcc/testsuite/ChangeLog:
> 
> 	* gcc.target/aarch64/sve/vect-early-break-cbranch_18.c: New test.
> 
> ---
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c b/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c
> new file mode 100644
> index 0000000000000000000000000000000000000000..9129f82cffe5e0028bbd6b7f0a9ae434388331e3
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c
> @@ -0,0 +1,22 @@
> +/* { dg-do compile } */
> +/* { dg-options "-Ofast -msve-vector-bits=256 -mautovec-preference=sve-only -fdump-tree-vect-details" } */
> +
> +#define N 512
> +#define START 1
> +#define END 505
> +
> +int x[N] __attribute__((aligned(32)));
> +
> +int __attribute__((noipa))
> +foo (void)
> +{
> +  for (int *p = x + START; p < x + END; ++p)
> +    if (*p == 0)
> +      return START;
> +
> +  return -1;
> +}
> +
> +/* { dg-final { scan-tree-dump "Alignment of access forced using peeling" "vect" } } */
> +/* { dg-final { scan-tree-dump-times {Value numbering stmt = vectp_x\.[0-9]+_[0-9]+ = &x \+ 4;} 1 "vect" } } */
> +/* { dg-final { scan-tree-dump-times {Value numbering stmt = vectp_x\.[0-9]+_[0-9]+ = vectp_x\.[0-9]+_[0-9]+ \+ 18446744073709551612;} 1 "vect" } } */
> diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
> index ab850de024bf41ea7635ce7381bcc6acd8fe0c28..8c71865aa0b1c2db7f0b0167dc7ed1db6c6da260 100644
> --- a/gcc/tree-vect-data-refs.cc
> +++ b/gcc/tree-vect-data-refs.cc
> @@ -5729,7 +5729,13 @@ vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
>    innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
>  
>    tree data_ref_base = unshare_expr (drb->base_address);
> -  tree base_offset = unshare_expr (get_dr_vinfo_offset (vinfo, dr_info, true));
> +  tree vector_offset = NULL_TREE;
> +  if (loop_vinfo && dr_info->offset)
> +    vector_offset = unshare_expr (dr_info->offset);
> +  tree base_offset = unshare_expr (vector_offset
> +				   ? drb->offset
> +				   : get_dr_vinfo_offset (vinfo, dr_info,
> +							  true));
>    tree init = unshare_expr (drb->init);
>  
>    if (loop_vinfo)
> @@ -5767,7 +5773,27 @@ vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
>  
>    vect_ptr_type = build_pointer_type (TREE_TYPE (DR_REF (dr)));
>    dest = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var, base_name);
> -  addr_base = force_gimple_operand (addr_base, &seq, true, dest);
> +
> +  /* Keep vectorizer-added offsets separate from the original scalar access
> +     address.  Forming "base + scalar offset" first gives the target a better
> +     chance of sharing it with other address calculations, such as the
> +     misalignment check used for masked alignment peeling.  */
> +  if (vector_offset)
> +    {
> +      tree scalar_dest = vect_get_new_vect_var (vect_ptr_type,
> +						vect_pointer_var, base_name);
> +      gimple_seq addr_seq = NULL;
> +      addr_base = force_gimple_operand (addr_base, &addr_seq, true,
> +					scalar_dest);
> +      gimple_seq_add_seq (&seq, addr_seq);
> +      addr_base = fold_build_pointer_plus (addr_base,
> +					   fold_convert (sizetype,
> +							 vector_offset));
> +    }
> +
> +  gimple_seq addr_seq = NULL;
> +  addr_base = force_gimple_operand (addr_base, &addr_seq, true, dest);
> +  gimple_seq_add_seq (&seq, addr_seq);
>    gimple_seq_add_seq (new_stmt_list, seq);
>  
>    if (TREE_CODE (addr_base) == SSA_NAME
> 
> 
>
diff mbox series

Patch

diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c b/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c
new file mode 100644
index 0000000000000000000000000000000000000000..9129f82cffe5e0028bbd6b7f0a9ae434388331e3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c
@@ -0,0 +1,22 @@ 
+/* { dg-do compile } */
+/* { dg-options "-Ofast -msve-vector-bits=256 -mautovec-preference=sve-only -fdump-tree-vect-details" } */
+
+#define N 512
+#define START 1
+#define END 505
+
+int x[N] __attribute__((aligned(32)));
+
+int __attribute__((noipa))
+foo (void)
+{
+  for (int *p = x + START; p < x + END; ++p)
+    if (*p == 0)
+      return START;
+
+  return -1;
+}
+
+/* { dg-final { scan-tree-dump "Alignment of access forced using peeling" "vect" } } */
+/* { dg-final { scan-tree-dump-times {Value numbering stmt = vectp_x\.[0-9]+_[0-9]+ = &x \+ 4;} 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times {Value numbering stmt = vectp_x\.[0-9]+_[0-9]+ = vectp_x\.[0-9]+_[0-9]+ \+ 18446744073709551612;} 1 "vect" } } */
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index ab850de024bf41ea7635ce7381bcc6acd8fe0c28..8c71865aa0b1c2db7f0b0167dc7ed1db6c6da260 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -5729,7 +5729,13 @@  vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
   innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
 
   tree data_ref_base = unshare_expr (drb->base_address);
-  tree base_offset = unshare_expr (get_dr_vinfo_offset (vinfo, dr_info, true));
+  tree vector_offset = NULL_TREE;
+  if (loop_vinfo && dr_info->offset)
+    vector_offset = unshare_expr (dr_info->offset);
+  tree base_offset = unshare_expr (vector_offset
+				   ? drb->offset
+				   : get_dr_vinfo_offset (vinfo, dr_info,
+							  true));
   tree init = unshare_expr (drb->init);
 
   if (loop_vinfo)
@@ -5767,7 +5773,27 @@  vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
 
   vect_ptr_type = build_pointer_type (TREE_TYPE (DR_REF (dr)));
   dest = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var, base_name);
-  addr_base = force_gimple_operand (addr_base, &seq, true, dest);
+
+  /* Keep vectorizer-added offsets separate from the original scalar access
+     address.  Forming "base + scalar offset" first gives the target a better
+     chance of sharing it with other address calculations, such as the
+     misalignment check used for masked alignment peeling.  */
+  if (vector_offset)
+    {
+      tree scalar_dest = vect_get_new_vect_var (vect_ptr_type,
+						vect_pointer_var, base_name);
+      gimple_seq addr_seq = NULL;
+      addr_base = force_gimple_operand (addr_base, &addr_seq, true,
+					scalar_dest);
+      gimple_seq_add_seq (&seq, addr_seq);
+      addr_base = fold_build_pointer_plus (addr_base,
+					   fold_convert (sizetype,
+							 vector_offset));
+    }
+
+  gimple_seq addr_seq = NULL;
+  addr_base = force_gimple_operand (addr_base, &addr_seq, true, dest);
+  gimple_seq_add_seq (&seq, addr_seq);
   gimple_seq_add_seq (new_stmt_list, seq);
 
   if (TREE_CODE (addr_base) == SSA_NAME