[3/3] AArch64: use ranger to fold WHILE_ULT to ptrue
Checks
| Context |
Check |
Description |
| linaro-tcwg-bot/tcwg_gcc_build--master-arm |
success
|
Build passed
|
| linaro-tcwg-bot/tcwg_gcc_build--master-aarch64 |
success
|
Build passed
|
Commit Message
Consider the following simple loop:
void foo (int *x)
{
for (int i = 0; i < 1000; i++)
x[i] *= 2;
}
compiled at -Ofast -march=armv8-a+sve generates
foo:
mov w1, 0
cntw x3
mov w2, 1000
whilelo p7.s, wzr, w2
.L2:
ld1w z31.s, p7/z, [x0, x1, lsl 2]
add z31.s, z31.s, z31.s
st1w z31.s, p7, [x0, x1, lsl 2]
add x1, x1, x3
whilelo p7.s, w1, w2
b.any .L2
ret
Which is nice, but the whilelo in the pre-header is unneeded. Due to the
architecturally defined minimum and maximum vector lengths[1] we know that at
any vector length the predicate is an all lanes active predicate, i.e. p7 is
always ptrue.
We can use gimple-isel these days while we still have range information on
the operands of .WHILE_ULTs to do this folding.
As such this patch folds whenever possible WHILE_ULTs into ptrue which are
cheaper to execute and so lowers our costs for entering the loops. i.e. the
above generates:
foo:
mov w1, 0
cntw x3
mov w2, 1000
ptrue p7.b, all
.L2:
ld1w z31.s, p7/z, [x0, x1, lsl 2]
add z31.s, z31.s, z31.s
st1w z31.s, p7, [x0, x1, lsl 2]
add x1, x1, x3
whilelo p7.s, w1, w2
b.any .L2
ret
[1] https://developer.arm.com/documentation/102476/0101/Introducing-SVE
Bootstrapped Regtested on aarch64-none-linux-gnu and no issues.
Any feedback? otherwise will commit early next week.
Thanks,
Tamar
gcc/ChangeLog:
* config/aarch64/aarch64.cc (aarch64_fold_while_ult_to_ptrue): New.
(aarch64_instruction_selection): Use it.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/fmaxnm_2.c: Update output.
* gcc.target/aarch64/sve/fmaxnm_3.c: Likewise.
* gcc.target/aarch64/sve/fminnm_2.c: Likewise.
* gcc.target/aarch64/sve/fminnm_3.c: Likewise.
* gcc.target/aarch64/sve/slp_12.c: Likewise.
* gcc.target/aarch64/sve/unpacked_fadd_2.c: Likewise.
* gcc.target/aarch64/sve/unpacked_fmul_2.c: Likewise.
* gcc.target/aarch64/sve/unpacked_fsubr_2.c: Likewise.
* gfortran.dg/pr88833.f90: Likewise.
* gcc.target/aarch64/sve/while_ult_1.c: New test.
* gcc.target/aarch64/sve/while_ult_1_run.c: New test.
---
--
Comments
Hi Tamar,
> On 7 Jul 2026, at 12:14, Tamar Christina <tamar.christina@arm.com> wrote:
>
> Consider the following simple loop:
>
> void foo (int *x)
> {
> for (int i = 0; i < 1000; i++)
> x[i] *= 2;
> }
>
> compiled at -Ofast -march=armv8-a+sve generates
>
> foo:
> mov w1, 0
> cntw x3
> mov w2, 1000
> whilelo p7.s, wzr, w2
> .L2:
> ld1w z31.s, p7/z, [x0, x1, lsl 2]
> add z31.s, z31.s, z31.s
> st1w z31.s, p7, [x0, x1, lsl 2]
> add x1, x1, x3
> whilelo p7.s, w1, w2
> b.any .L2
> ret
>
> Which is nice, but the whilelo in the pre-header is unneeded. Due to the
> architecturally defined minimum and maximum vector lengths[1] we know that at
> any vector length the predicate is an all lanes active predicate, i.e. p7 is
> always ptrue.
>
> We can use gimple-isel these days while we still have range information on
> the operands of .WHILE_ULTs to do this folding.
>
> As such this patch folds whenever possible WHILE_ULTs into ptrue which are
> cheaper to execute and so lowers our costs for entering the loops. i.e. the
> above generates:
>
> foo:
> mov w1, 0
> cntw x3
> mov w2, 1000
> ptrue p7.b, all
> .L2:
> ld1w z31.s, p7/z, [x0, x1, lsl 2]
> add z31.s, z31.s, z31.s
> st1w z31.s, p7, [x0, x1, lsl 2]
> add x1, x1, x3
> whilelo p7.s, w1, w2
> b.any .L2
> ret
>
> [1] https://developer.arm.com/documentation/102476/0101/Introducing-SVE
>
> Bootstrapped Regtested on aarch64-none-linux-gnu and no issues.
>
> Any feedback? otherwise will commit early next week.
>
> Thanks,
> Tamar
>
> gcc/ChangeLog:
>
> * config/aarch64/aarch64.cc (aarch64_fold_while_ult_to_ptrue): New.
> (aarch64_instruction_selection): Use it.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/aarch64/sve/fmaxnm_2.c: Update output.
> * gcc.target/aarch64/sve/fmaxnm_3.c: Likewise.
> * gcc.target/aarch64/sve/fminnm_2.c: Likewise.
> * gcc.target/aarch64/sve/fminnm_3.c: Likewise.
> * gcc.target/aarch64/sve/slp_12.c: Likewise.
> * gcc.target/aarch64/sve/unpacked_fadd_2.c: Likewise.
> * gcc.target/aarch64/sve/unpacked_fmul_2.c: Likewise.
> * gcc.target/aarch64/sve/unpacked_fsubr_2.c: Likewise.
> * gfortran.dg/pr88833.f90: Likewise.
> * gcc.target/aarch64/sve/while_ult_1.c: New test.
> * gcc.target/aarch64/sve/while_ult_1_run.c: New test.
>
> ---
> diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
> index 7a81be8dcbe942ca113f7db2fbab4c357c40f462..630856f95281b4e2c290353b3b491404b5dfe633 100644
> --- a/gcc/config/aarch64/aarch64.cc
> +++ b/gcc/config/aarch64/aarch64.cc
> @@ -4442,15 +4442,87 @@ aarch64_fold_sve_ptrue_vl (tree vectype, unsigned int vl,
> return builder.build ();
> }
>
> +/* Fold a WHILE_ULT whose bounds are known enough to be expressed as a PTRUE.
> + Generic folding cannot do this because it cannot assume an architectural
> + maximum for a scalable vector. */
> +
> +static bool
> +aarch64_fold_while_ult_to_ptrue (function *fun, gcall *call,
> + gimple_stmt_iterator *gsi)
> +{
> + if (!gimple_call_internal_p (call, IFN_WHILE_ULT))
> + return false;
> +
> + tree lhs = gimple_call_lhs (call);
> + if (!lhs)
> + return false;
> +
> + tree lhs_type = TREE_TYPE (lhs);
> + if (!VECTOR_BOOLEAN_TYPE_P (lhs_type)
> + || !aarch64_sve_pred_mode_p (TYPE_MODE (lhs_type)))
> + return false;
> +
> + int_range_max min, max;
> + range_query *query = get_range_query (fun);
> + if (!query->range_of_expr (min, gimple_call_arg (call, 0), call)
> + || !query->range_of_expr (max, gimple_call_arg (call, 1), call)
> + || min.undefined_p ()
> + || max.undefined_p ()
> + || !min.nonnegative_p ()
> + || !max.nonnegative_p ())
> + return false;
> +
> + widest_int min_upper = widest_int::from (min.upper_bound (), UNSIGNED);
> + widest_int min_lower = widest_int::from (min.lower_bound (), UNSIGNED);
> + widest_int max_lower = widest_int::from (max.lower_bound (), UNSIGNED);
> + widest_int max_upper = widest_int::from (max.upper_bound (), UNSIGNED);
> + if (wi::leu_p (max_lower, min_upper))
> + return false;
> +
> + unsigned int min_nelts
> + = constant_lower_bound (TYPE_VECTOR_SUBPARTS (lhs_type));
> + widest_int max_sve_nelts = min_nelts * 16;
For VLS (when TYPE_VECTOR_SUBPARTS (lhs_type).is_constant ()) can we use the exact number of elements rather than the VLA minimum 16?
> + widest_int min_gap = max_lower - min_upper;
> + widest_int max_gap = max_upper - min_lower;
> +
> + tree pred_cst = NULL_TREE;
> + unsigned HOST_WIDE_INT gap = 0;
> + if (min_gap == max_gap
> + && wi::fits_uhwi_p (min_gap))
> + {
> + gap = min_gap.to_uhwi ();
> + machine_mode pred_mode = TYPE_MODE (lhs_type);
> + if (gap <= UINT_MAX
> + && (aarch64_svpattern_for_vl (pred_mode, (int) gap)
I think this is unsafe for gap = 0x80000000 as that will get cast to a negative int.
Consider the test case:
void
large_exact_gap (int *__restrict a, const int *__restrict b)
{
for (unsigned long long i = 0; i < 0x80000000ULL; ++i)
a[i] = b[i] + 1;
}
Maybe handling min_gap >= max_sve_nelts first to produce all-ones is better?
> + != AARCH64_NUM_SVPATTERNS))
> + pred_cst = aarch64_fold_sve_ptrue_vl (lhs_type,
> + (unsigned int) gap, 1);
> + }
> +
> + if (!pred_cst)
> + {
> + if (wi::ltu_p (min_gap, max_sve_nelts))
> + return false;
> + pred_cst = build_all_ones_cst (lhs_type);
> + }
> +
> + gassign *assign = gimple_build_assign (lhs, pred_cst);
> + gsi_replace (gsi, assign, false);
> + return true;
> +}
> +
> /* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used to
> change generic sequences to a form AArch64 has an easier time expanding
> instructions for. It's not supposed to be used for generic rewriting that
> all targets would benefit from. */
>
> static bool
> -aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator *gsi)
> +aarch64_instruction_selection (function *fun, gimple_stmt_iterator *gsi)
> {
> auto stmt = gsi_stmt (*gsi);
> + if (gcall *call = dyn_cast<gcall *> (stmt))
> + return aarch64_fold_while_ult_to_ptrue (fun, call, gsi);
> +
> gassign *assign = dyn_cast<gassign *> (stmt);
>
> if (!assign)
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
> index ee3cdc20f96550ceeb1e6c11dd464c59eb4b3e6f..3f1dfdd60d238945e2c4fbf4676d3e943cfb3e30 100644
> --- a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
> @@ -16,7 +16,7 @@ f2 (double x, double *ptr)
> return x;
> }
>
> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
> /* { dg-final { scan-assembler-times {\tfmaxnmv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> /* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
> index a8eee0f4b2698f0425f55ea91c523e462ca134bd..2548e55f2841d370d5d30bce23350d3750f094a9 100644
> --- a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
> @@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
> res[1] = x1;
> }
>
> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> /* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 2 } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
> index 10aced05f1afe69f6e646ac23bac8196c13f8357..ed657d58fdfe663f9ddf5b013aa73e14868dbe03 100644
> --- a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
> @@ -16,7 +16,7 @@ f2 (double x, double *ptr)
> return x;
> }
>
> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
> /* { dg-final { scan-assembler-times {\tfminnmv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> /* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
> index 80ad01602491d07b338113f34d372b5f3970a3be..1f2fc76640fb6c5389c0185a793ca94446ff0352 100644
> --- a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
> @@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
> res[1] = x1;
> }
>
> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> /* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 2 } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c b/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
> index 0b9f8d9bdfaaf868ba0bd73ae83894c68e1613f8..6ba4c9651b67d7e2495b58ffc0ed01f563c428b6 100644
> --- a/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
> @@ -47,9 +47,9 @@ TEST_ALL (VEC_PERM)
>
> /* We should use WHILEs for all accesses. */
> /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b} 20 } } */
> -/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 20 } } */
> -/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 30 } } */
> -/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 30 } } */
> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 18 } } */
> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 27 } } */
> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 24 } } */
>
> /* 6 for the 8-bit types and 2 for the 16-bit types. */
> /* { dg-final { scan-assembler-times {\tuqdecb\t} 8 } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
> index 7a74efd129cb1c2bab6f60b26f371d9a0deb3b1d..7ebee0fff77602c0d63a43f00c4be958c1fe5036 100644
> --- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
> @@ -5,7 +5,7 @@
>
> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
> -/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 12 } } */
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 15 } } */
>
> /* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
> /* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
> index eb05600e98a052fcd21f9c7f0f29502bbb5484a5..98617a90260a5d0508fe7c59a76a4359c9e18283 100644
> --- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
> @@ -5,7 +5,7 @@
>
> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
> -/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
>
> /* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
> /* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
> index de9325ccacb00a970ecddd90fb25d609467f74c3..286a1282c402b16bc0733b9c5ca9dbe1c1fb8538 100644
> --- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
> @@ -5,7 +5,7 @@
>
> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
> -/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
>
> /* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
> /* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
> new file mode 100644
> index 0000000000000000000000000000000000000000..ddc28029117dadac0610da3ba2753bc9cc4cc385
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
> @@ -0,0 +1,99 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve -msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only -fdump-tree-vect-details --save-temps" } */
> +/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 14 "vect" } } */
> +
> +#include <arm_sve.h>
> +#include <stdint.h>
> +
> +#define NOIPA __attribute__ ((noipa))
> +
> +#define ADD_LOOP(TYPE, NAME, N) \
> + void NOIPA \
> + NAME (TYPE *__restrict a, const TYPE *__restrict b) \
> + { \
> + for (unsigned int i = 0; i < N; ++i) \
> + a[i] = b[i] + (TYPE) 1; \
> + }
> +
> +ADD_LOOP (uint8_t, full_b, 300)
> +ADD_LOOP (uint16_t, full_h, 1000)
> +ADD_LOOP (uint32_t, full_s, 100)
> +ADD_LOOP (uint64_t, full_d, 40)
> +
> +ADD_LOOP (uint8_t, exact_b, 256)
> +ADD_LOOP (uint16_t, exact_h, 128)
> +ADD_LOOP (uint32_t, exact_s, 64)
> +ADD_LOOP (uint64_t, exact_d, 32)
> +
> +svbool_t NOIPA
> +finite_b (void)
> +{
> + return svptrue_pat_b8 (SV_VL8);
> +}
These direct svptrue_pat_* calls exercise the existing ACLE folding path moved by patch 2.
They do not exercise the new finite-pattern path in aarch64_fold_while_ult_to_ptrue unless I’m missing something?.
Thanks,
Kyrill
> +
> +svbool_t NOIPA
> +finite_h (void)
> +{
> + return svptrue_pat_b16 (SV_VL4);
> +}
> +
> +svbool_t NOIPA
> +finite_s (void)
> +{
> + return svptrue_pat_b32 (SV_VL2);
> +}
> +
> +svbool_t NOIPA
> +finite_d (void)
> +{
> + return svptrue_pat_b64 (SV_VL1);
> +}
> +
> +
> +ADD_LOOP (uint8_t, partial_b, 255)
> +ADD_LOOP (uint16_t, partial_h, 127)
> +ADD_LOOP (uint32_t, partial_s, 63)
> +ADD_LOOP (uint64_t, partial_d, 31)
> +
> +void NOIPA
> +full_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
> + unsigned int n)
> +{
> + if (n < 1000)
> + __builtin_unreachable ();
> + for (unsigned int i = 0; i < n; ++i)
> + a[i] = b[i] + (uint16_t) 1;
> +}
> +
> +void NOIPA
> +partial_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
> + unsigned int n)
> +{
> + if (n < 100 || n > 127)
> + __builtin_unreachable ();
> + for (unsigned int i = 0; i < n; ++i)
> + a[i] = b[i] + (uint16_t) 1;
> +}
> +
> +/* The first WHILELO in the "full" loops should fold to an all-true
> + predicate. GCC prints all-true predicates as .b, regardless of the
> + element size of the consuming instruction. */
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, all\n} 9 } } */
> +
> +/* The finite loops should fold to finite PTRUE patterns. */
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl8\n} 1 } } */
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl4\n} 1 } } */
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s, vl2\n} 1 } } */
> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl1\n} 1 } } */
> +
> +/* The "partial" loops should keep their initial WHILELO. */
> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, wzr, w[0-9]+} 1 } } */
> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, wzr, w[0-9]+} 2 } } */
> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, wzr, w[0-9]+} 1 } } */
> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, wzr, w[0-9]+} 1 } } */
> +
> +/* All loops still need a latch WHILELO. */
> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, [wx][0-9]+, [wx][0-9]+} 3 } } */
> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, [wx][0-9]+, [wx][0-9]+} 5 } } */
> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, [wx][0-9]+, [wx][0-9]+} 3 } } */
> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, [wx][0-9]+, [wx][0-9]+} 3 } } */
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
> new file mode 100644
> index 0000000000000000000000000000000000000000..342b11afce92f567c90ab822f7bdc256c51de7cb
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
> @@ -0,0 +1,66 @@
> +/* { dg-do run { target aarch64_sve_hw } } */
> +/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve -msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only" } */
> +
> +#include "while_ult_1.c"
> +
> +#define CHECK_LOOP(TYPE, NAME, N) \
> + do \
> + { \
> + TYPE a[N], b[N]; \
> + for (unsigned int i = 0; i < N; ++i) \
> + { \
> + a[i] = 0; \
> + b[i] = (TYPE) (i * 3 + i % 7); \
> + } \
> + NAME (a, b); \
> + for (unsigned int i = 0; i < N; ++i) \
> + if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
> + __builtin_abort (); \
> + } \
> + while (0)
> +
> +#define CHECK_LOOP_ARG(TYPE, NAME, N) \
> + do \
> + { \
> + TYPE a[N], b[N]; \
> + for (unsigned int i = 0; i < N; ++i) \
> + { \
> + a[i] = 0; \
> + b[i] = (TYPE) (i * 3 + i % 7); \
> + } \
> + NAME (a, b, N); \
> + for (unsigned int i = 0; i < N; ++i) \
> + if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
> + __builtin_abort (); \
> + } \
> + while (0)
> +
> +int __attribute__ ((optimize (1)))
> +main (void)
> +{
> + CHECK_LOOP (uint8_t, full_b, 300);
> + CHECK_LOOP (uint16_t, full_h, 1000);
> + CHECK_LOOP (uint32_t, full_s, 100);
> + CHECK_LOOP (uint64_t, full_d, 40);
> +
> + CHECK_LOOP (uint8_t, exact_b, 256);
> + CHECK_LOOP (uint16_t, exact_h, 128);
> + CHECK_LOOP (uint32_t, exact_s, 64);
> + CHECK_LOOP (uint64_t, exact_d, 32);
> +
> + svbool_t pg = svptrue_b8 ();
> + if (svcntp_b8 (pg, finite_b ()) != 8
> + || svcntp_b16 (pg, finite_h ()) != 4
> + || svcntp_b32 (pg, finite_s ()) != 2
> + || svcntp_b64 (pg, finite_d ()) != 1)
> + __builtin_abort ();
> +
> + CHECK_LOOP (uint8_t, partial_b, 255);
> + CHECK_LOOP (uint16_t, partial_h, 127);
> + CHECK_LOOP (uint32_t, partial_s, 63);
> + CHECK_LOOP (uint64_t, partial_d, 31);
> +
> + CHECK_LOOP_ARG (uint16_t, full_range_h, 1000);
> + CHECK_LOOP_ARG (uint16_t, partial_range_h, 127);
> + return 0;
> +}
> diff --git a/gcc/testsuite/gfortran.dg/pr88833.f90 b/gcc/testsuite/gfortran.dg/pr88833.f90
> index 224e6ce5f3d53135c588b6ac0ab6ee1733eab852..5959cb1d16bf3da90bbbdedcb57631610cfc17d0 100644
> --- a/gcc/testsuite/gfortran.dg/pr88833.f90
> +++ b/gcc/testsuite/gfortran.dg/pr88833.f90
> @@ -6,4 +6,4 @@ subroutine foo(x)
> x = x + 10
> end subroutine foo
>
> -! { dg-final { scan-assembler {\twhilelo\tp[0-9]+\.s, wzr, (w[0-9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }
> +! { dg-final { scan-assembler-not {\twhilelo\tp[0-9]+\.s, wzr, (w[0-9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }
>
>
> --
> <rb20686.patch>
> -----Original Message-----
> From: Kyrylo Tkachov <ktkachov@nvidia.com>
> Sent: 09 July 2026 10:06
> To: Tamar Christina <Tamar.Christina@arm.com>
> Cc: gcc-patches@gcc.gnu.org; nd <nd@arm.com>; Richard Earnshaw
> <Richard.Earnshaw@arm.com>; ktkachov@gcc.gnu.org; Alex Coplan
> <Alex.Coplan@arm.com>; andrew.pinski@oss.qualcomm.com; Wilco Dijkstra
> <Wilco.Dijkstra@arm.com>; Alice Carlotti <Alice.Carlotti@arm.com>
> Subject: Re: [patch 3/3]AArch64: use ranger to fold WHILE_ULT to ptrue
>
> Hi Tamar,
>
> > On 7 Jul 2026, at 12:14, Tamar Christina <tamar.christina@arm.com> wrote:
> >
> > Consider the following simple loop:
> >
> > void foo (int *x)
> > {
> > for (int i = 0; i < 1000; i++)
> > x[i] *= 2;
> > }
> >
> > compiled at -Ofast -march=armv8-a+sve generates
> >
> > foo:
> > mov w1, 0
> > cntw x3
> > mov w2, 1000
> > whilelo p7.s, wzr, w2
> > .L2:
> > ld1w z31.s, p7/z, [x0, x1, lsl 2]
> > add z31.s, z31.s, z31.s
> > st1w z31.s, p7, [x0, x1, lsl 2]
> > add x1, x1, x3
> > whilelo p7.s, w1, w2
> > b.any .L2
> > ret
> >
> > Which is nice, but the whilelo in the pre-header is unneeded. Due to the
> > architecturally defined minimum and maximum vector lengths[1] we know
> that at
> > any vector length the predicate is an all lanes active predicate, i.e. p7 is
> > always ptrue.
> >
> > We can use gimple-isel these days while we still have range information on
> > the operands of .WHILE_ULTs to do this folding.
> >
> > As such this patch folds whenever possible WHILE_ULTs into ptrue which are
> > cheaper to execute and so lowers our costs for entering the loops. i.e. the
> > above generates:
> >
> > foo:
> > mov w1, 0
> > cntw x3
> > mov w2, 1000
> > ptrue p7.b, all
> > .L2:
> > ld1w z31.s, p7/z, [x0, x1, lsl 2]
> > add z31.s, z31.s, z31.s
> > st1w z31.s, p7, [x0, x1, lsl 2]
> > add x1, x1, x3
> > whilelo p7.s, w1, w2
> > b.any .L2
> > ret
> >
> > [1] https://developer.arm.com/documentation/102476/0101/Introducing-
> SVE
> >
> > Bootstrapped Regtested on aarch64-none-linux-gnu and no issues.
> >
> > Any feedback? otherwise will commit early next week.
> >
> > Thanks,
> > Tamar
> >
> > gcc/ChangeLog:
> >
> > * config/aarch64/aarch64.cc (aarch64_fold_while_ult_to_ptrue): New.
> > (aarch64_instruction_selection): Use it.
> >
> > gcc/testsuite/ChangeLog:
> >
> > * gcc.target/aarch64/sve/fmaxnm_2.c: Update output.
> > * gcc.target/aarch64/sve/fmaxnm_3.c: Likewise.
> > * gcc.target/aarch64/sve/fminnm_2.c: Likewise.
> > * gcc.target/aarch64/sve/fminnm_3.c: Likewise.
> > * gcc.target/aarch64/sve/slp_12.c: Likewise.
> > * gcc.target/aarch64/sve/unpacked_fadd_2.c: Likewise.
> > * gcc.target/aarch64/sve/unpacked_fmul_2.c: Likewise.
> > * gcc.target/aarch64/sve/unpacked_fsubr_2.c: Likewise.
> > * gfortran.dg/pr88833.f90: Likewise.
> > * gcc.target/aarch64/sve/while_ult_1.c: New test.
> > * gcc.target/aarch64/sve/while_ult_1_run.c: New test.
> >
> > ---
> > diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
> > index
> 7a81be8dcbe942ca113f7db2fbab4c357c40f462..630856f95281b4e2c2903
> 53b3b491404b5dfe633 100644
> > --- a/gcc/config/aarch64/aarch64.cc
> > +++ b/gcc/config/aarch64/aarch64.cc
> > @@ -4442,15 +4442,87 @@ aarch64_fold_sve_ptrue_vl (tree vectype,
> unsigned int vl,
> > return builder.build ();
> > }
> >
> > +/* Fold a WHILE_ULT whose bounds are known enough to be expressed as
> a PTRUE.
> > + Generic folding cannot do this because it cannot assume an architectural
> > + maximum for a scalable vector. */
> > +
> > +static bool
> > +aarch64_fold_while_ult_to_ptrue (function *fun, gcall *call,
> > + gimple_stmt_iterator *gsi)
> > +{
> > + if (!gimple_call_internal_p (call, IFN_WHILE_ULT))
> > + return false;
> > +
> > + tree lhs = gimple_call_lhs (call);
> > + if (!lhs)
> > + return false;
> > +
> > + tree lhs_type = TREE_TYPE (lhs);
> > + if (!VECTOR_BOOLEAN_TYPE_P (lhs_type)
> > + || !aarch64_sve_pred_mode_p (TYPE_MODE (lhs_type)))
> > + return false;
> > +
> > + int_range_max min, max;
> > + range_query *query = get_range_query (fun);
> > + if (!query->range_of_expr (min, gimple_call_arg (call, 0), call)
> > + || !query->range_of_expr (max, gimple_call_arg (call, 1), call)
> > + || min.undefined_p ()
> > + || max.undefined_p ()
> > + || !min.nonnegative_p ()
> > + || !max.nonnegative_p ())
> > + return false;
> > +
> > + widest_int min_upper = widest_int::from (min.upper_bound (),
> UNSIGNED);
> > + widest_int min_lower = widest_int::from (min.lower_bound (),
> UNSIGNED);
> > + widest_int max_lower = widest_int::from (max.lower_bound (),
> UNSIGNED);
> > + widest_int max_upper = widest_int::from (max.upper_bound (),
> UNSIGNED);
> > + if (wi::leu_p (max_lower, min_upper))
> > + return false;
> > +
> > + unsigned int min_nelts
> > + = constant_lower_bound (TYPE_VECTOR_SUBPARTS (lhs_type));
> > + widest_int max_sve_nelts = min_nelts * 16;
>
> For VLS (when TYPE_VECTOR_SUBPARTS (lhs_type).is_constant ()) can we use
> the exact number of elements rather than the VLA minimum 16?
>
We don't have WHILE_ULTs for fixed vector length though. When VLS the
mask is generated explicitly as a constant, .e.. you get a BImode vector with
{-1,-1,-1,-1,-1, ...}. So at the moment the expectation is that VLS never gets here.
You can get a whilelo synthesized by the predicate expansion code, i.e.
void foo (int *x)
{
for (int i = 0; i < 10; i++)
x[i] *= 2;
}
With -O3 -march=armv8.5-a+sve whilelt.c -S -o - -mautovec-preference=sve-only -msve-vector-bits=512
gets you a whilelo but that's done on expansion. While -msve-vector-bits=256 gets you
the correct ptrue p7.b, vl32.
But I can make it more explicit by rejecting
> > + widest_int min_gap = max_lower - min_upper;
> > + widest_int max_gap = max_upper - min_lower;
> > +
> > + tree pred_cst = NULL_TREE;
> > + unsigned HOST_WIDE_INT gap = 0;
> > + if (min_gap == max_gap
> > + && wi::fits_uhwi_p (min_gap))
> > + {
> > + gap = min_gap.to_uhwi ();
> > + machine_mode pred_mode = TYPE_MODE (lhs_type);
> > + if (gap <= UINT_MAX
> > + && (aarch64_svpattern_for_vl (pred_mode, (int) gap)
>
> I think this is unsafe for gap = 0x80000000 as that will get cast to a negative
> int.
> Consider the test case:
> void
> large_exact_gap (int *__restrict a, const int *__restrict b)
> {
> for (unsigned long long i = 0; i < 0x80000000ULL; ++i)
> a[i] = b[i] + 1;
> }
>
> Maybe handling min_gap >= max_sve_nelts first to produce all-ones is better?
Yeah that's a good shout. I had expected aarch64_fold_sve_ptrue_vl to return NULL
if it can't fold, but looks like it does no validation and so an invalid fold triggers an
ICE. Will fix.
>
> > + != AARCH64_NUM_SVPATTERNS))
> > + pred_cst = aarch64_fold_sve_ptrue_vl (lhs_type,
> > + (unsigned int) gap, 1);
> > + }
> > +
> > + if (!pred_cst)
> > + {
> > + if (wi::ltu_p (min_gap, max_sve_nelts))
> > + return false;
> > + pred_cst = build_all_ones_cst (lhs_type);
> > + }
> > +
> > + gassign *assign = gimple_build_assign (lhs, pred_cst);
> > + gsi_replace (gsi, assign, false);
> > + return true;
> > +}
> > +
> > /* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used
> to
> > change generic sequences to a form AArch64 has an easier time expanding
> > instructions for. It's not supposed to be used for generic rewriting that
> > all targets would benefit from. */
> >
> > static bool
> > -aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator
> *gsi)
> > +aarch64_instruction_selection (function *fun, gimple_stmt_iterator *gsi)
> > {
> > auto stmt = gsi_stmt (*gsi);
> > + if (gcall *call = dyn_cast<gcall *> (stmt))
> > + return aarch64_fold_while_ult_to_ptrue (fun, call, gsi);
> > +
> > gassign *assign = dyn_cast<gassign *> (stmt);
> >
> > if (!assign)
> > diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
> b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
> > index
> ee3cdc20f96550ceeb1e6c11dd464c59eb4b3e6f..3f1dfdd60d238945e2c4fb
> f4676d3e943cfb3e30 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
> > @@ -16,7 +16,7 @@ f2 (double x, double *ptr)
> > return x;
> > }
> >
> > -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-
> 9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
> > +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-
> 9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
> > /* { dg-final { scan-assembler-times {\tfmaxnmv\ts[0-9]+, p[0-7], z[0-
> 9]+\.s\n} 1 } } */
> > -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-
> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> > +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-
> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> > /* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-
> 9]+\.d\n} 1 } } */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
> b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
> > index
> a8eee0f4b2698f0425f55ea91c523e462ca134bd..2548e55f2841d370d5d3
> 0bce23350d3750f094a9 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
> > @@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
> > res[1] = x1;
> > }
> >
> > -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-
> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> > +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-
> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> > /* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-
> 9]+\.d\n} 2 } } */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
> b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
> > index
> 10aced05f1afe69f6e646ac23bac8196c13f8357..ed657d58fdfe663f9ddf5b0
> 13aa73e14868dbe03 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
> > @@ -16,7 +16,7 @@ f2 (double x, double *ptr)
> > return x;
> > }
> >
> > -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-
> 9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
> > +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-
> 9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
> > /* { dg-final { scan-assembler-times {\tfminnmv\ts[0-9]+, p[0-7], z[0-
> 9]+\.s\n} 1 } } */
> > -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-
> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> > +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-
> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> > /* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-
> 9]+\.d\n} 1 } } */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
> b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
> > index
> 80ad01602491d07b338113f34d372b5f3970a3be..1f2fc76640fb6c5389c01
> 85a793ca94446ff0352 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
> > @@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
> > res[1] = x1;
> > }
> >
> > -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-
> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> > +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-
> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
> > /* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-
> 9]+\.d\n} 2 } } */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
> b/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
> > index
> 0b9f8d9bdfaaf868ba0bd73ae83894c68e1613f8..6ba4c9651b67d7e2495b
> 58ffc0ed01f563c428b6 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
> > @@ -47,9 +47,9 @@ TEST_ALL (VEC_PERM)
> >
> > /* We should use WHILEs for all accesses. */
> > /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b} 20 } } */
> > -/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 20 } } */
> > -/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 30 } } */
> > -/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 30 } } */
> > +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 18 } } */
> > +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 27 } } */
> > +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 24 } } */
> >
> > /* 6 for the 8-bit types and 2 for the 16-bit types. */
> > /* { dg-final { scan-assembler-times {\tuqdecb\t} 8 } } */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
> b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
> > index
> 7a74efd129cb1c2bab6f60b26f371d9a0deb3b1d..7ebee0fff77602c0d63a43
> f00c4be958c1fe5036 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
> > @@ -5,7 +5,7 @@
> >
> > /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
> > /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
> > -/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 12 } } */
> > +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 15 } } */
> >
> > /* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
> > /* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
> b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
> > index
> eb05600e98a052fcd21f9c7f0f29502bbb5484a5..98617a90260a5d0508fe7
> c59a76a4359c9e18283 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
> > @@ -5,7 +5,7 @@
> >
> > /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
> > /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
> > -/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
> > +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
> >
> > /* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
> > /* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
> b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
> > index
> de9325ccacb00a970ecddd90fb25d609467f74c3..286a1282c402b16bc0733
> b9c5ca9dbe1c1fb8538 100644
> > --- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
> > +++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
> > @@ -5,7 +5,7 @@
> >
> > /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
> > /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
> > -/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
> > +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
> >
> > /* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
> > /* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
> b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
> > new file mode 100644
> > index
> 0000000000000000000000000000000000000000..ddc28029117dadac06
> 10da3ba2753bc9cc4cc385
> > --- /dev/null
> > +++ b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
> > @@ -0,0 +1,99 @@
> > +/* { dg-do compile } */
> > +/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve -
> msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only -
> fdump-tree-vect-details --save-temps" } */
> > +/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 14 "vect" } } */
> > +
> > +#include <arm_sve.h>
> > +#include <stdint.h>
> > +
> > +#define NOIPA __attribute__ ((noipa))
> > +
> > +#define ADD_LOOP(TYPE, NAME, N) \
> > + void NOIPA \
> > + NAME (TYPE *__restrict a, const TYPE *__restrict b) \
> > + { \
> > + for (unsigned int i = 0; i < N; ++i) \
> > + a[i] = b[i] + (TYPE) 1; \
> > + }
> > +
> > +ADD_LOOP (uint8_t, full_b, 300)
> > +ADD_LOOP (uint16_t, full_h, 1000)
> > +ADD_LOOP (uint32_t, full_s, 100)
> > +ADD_LOOP (uint64_t, full_d, 40)
> > +
> > +ADD_LOOP (uint8_t, exact_b, 256)
> > +ADD_LOOP (uint16_t, exact_h, 128)
> > +ADD_LOOP (uint32_t, exact_s, 64)
> > +ADD_LOOP (uint64_t, exact_d, 32)
> > +
> > +svbool_t NOIPA
> > +finite_b (void)
> > +{
> > + return svptrue_pat_b8 (SV_VL8);
> > +}
>
> These direct svptrue_pat_* calls exercise the existing ACLE folding path moved
> by patch 2.
> They do not exercise the new finite-pattern path in
> aarch64_fold_while_ult_to_ptrue unless I’m missing something?.
No that's correct, it's just checking that the code didn't break in a smaller testcase.
the aarch64_fold_while_ult_to_ptrue cases are tested by the ADD_LOOP macro ones.
I can remove them If you prefer.
Thanks,
Tamar
>
> Thanks,
> Kyrill
>
> > +
> > +svbool_t NOIPA
> > +finite_h (void)
> > +{
> > + return svptrue_pat_b16 (SV_VL4);
> > +}
> > +
> > +svbool_t NOIPA
> > +finite_s (void)
> > +{
> > + return svptrue_pat_b32 (SV_VL2);
> > +}
> > +
> > +svbool_t NOIPA
> > +finite_d (void)
> > +{
> > + return svptrue_pat_b64 (SV_VL1);
> > +}
> > +
> > +
> > +ADD_LOOP (uint8_t, partial_b, 255)
> > +ADD_LOOP (uint16_t, partial_h, 127)
> > +ADD_LOOP (uint32_t, partial_s, 63)
> > +ADD_LOOP (uint64_t, partial_d, 31)
> > +
> > +void NOIPA
> > +full_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
> > + unsigned int n)
> > +{
> > + if (n < 1000)
> > + __builtin_unreachable ();
> > + for (unsigned int i = 0; i < n; ++i)
> > + a[i] = b[i] + (uint16_t) 1;
> > +}
> > +
> > +void NOIPA
> > +partial_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
> > + unsigned int n)
> > +{
> > + if (n < 100 || n > 127)
> > + __builtin_unreachable ();
> > + for (unsigned int i = 0; i < n; ++i)
> > + a[i] = b[i] + (uint16_t) 1;
> > +}
> > +
> > +/* The first WHILELO in the "full" loops should fold to an all-true
> > + predicate. GCC prints all-true predicates as .b, regardless of the
> > + element size of the consuming instruction. */
> > +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, all\n} 9 } } */
> > +
> > +/* The finite loops should fold to finite PTRUE patterns. */
> > +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl8\n} 1 } } */
> > +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl4\n} 1 } } */
> > +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s, vl2\n} 1 } } */
> > +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl1\n} 1 } } */
> > +
> > +/* The "partial" loops should keep their initial WHILELO. */
> > +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, wzr, w[0-9]+} 1 }
> } */
> > +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, wzr, w[0-9]+} 2 }
> } */
> > +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, wzr, w[0-9]+} 1 }
> } */
> > +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, wzr, w[0-9]+} 1 }
> } */
> > +
> > +/* All loops still need a latch WHILELO. */
> > +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, [wx][0-9]+,
> [wx][0-9]+} 3 } } */
> > +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, [wx][0-9]+,
> [wx][0-9]+} 5 } } */
> > +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, [wx][0-9]+,
> [wx][0-9]+} 3 } } */
> > +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, [wx][0-9]+,
> [wx][0-9]+} 3 } } */
> > diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
> b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
> > new file mode 100644
> > index
> 0000000000000000000000000000000000000000..342b11afce92f567c90
> ab822f7bdc256c51de7cb
> > --- /dev/null
> > +++ b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
> > @@ -0,0 +1,66 @@
> > +/* { dg-do run { target aarch64_sve_hw } } */
> > +/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve -
> msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only" } */
> > +
> > +#include "while_ult_1.c"
> > +
> > +#define CHECK_LOOP(TYPE, NAME, N) \
> > + do \
> > + { \
> > + TYPE a[N], b[N]; \
> > + for (unsigned int i = 0; i < N; ++i) \
> > + { \
> > + a[i] = 0; \
> > + b[i] = (TYPE) (i * 3 + i % 7); \
> > + } \
> > + NAME (a, b); \
> > + for (unsigned int i = 0; i < N; ++i) \
> > + if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
> > + __builtin_abort (); \
> > + } \
> > + while (0)
> > +
> > +#define CHECK_LOOP_ARG(TYPE, NAME, N) \
> > + do \
> > + { \
> > + TYPE a[N], b[N]; \
> > + for (unsigned int i = 0; i < N; ++i) \
> > + { \
> > + a[i] = 0; \
> > + b[i] = (TYPE) (i * 3 + i % 7); \
> > + } \
> > + NAME (a, b, N); \
> > + for (unsigned int i = 0; i < N; ++i) \
> > + if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
> > + __builtin_abort (); \
> > + } \
> > + while (0)
> > +
> > +int __attribute__ ((optimize (1)))
> > +main (void)
> > +{
> > + CHECK_LOOP (uint8_t, full_b, 300);
> > + CHECK_LOOP (uint16_t, full_h, 1000);
> > + CHECK_LOOP (uint32_t, full_s, 100);
> > + CHECK_LOOP (uint64_t, full_d, 40);
> > +
> > + CHECK_LOOP (uint8_t, exact_b, 256);
> > + CHECK_LOOP (uint16_t, exact_h, 128);
> > + CHECK_LOOP (uint32_t, exact_s, 64);
> > + CHECK_LOOP (uint64_t, exact_d, 32);
> > +
> > + svbool_t pg = svptrue_b8 ();
> > + if (svcntp_b8 (pg, finite_b ()) != 8
> > + || svcntp_b16 (pg, finite_h ()) != 4
> > + || svcntp_b32 (pg, finite_s ()) != 2
> > + || svcntp_b64 (pg, finite_d ()) != 1)
> > + __builtin_abort ();
> > +
> > + CHECK_LOOP (uint8_t, partial_b, 255);
> > + CHECK_LOOP (uint16_t, partial_h, 127);
> > + CHECK_LOOP (uint32_t, partial_s, 63);
> > + CHECK_LOOP (uint64_t, partial_d, 31);
> > +
> > + CHECK_LOOP_ARG (uint16_t, full_range_h, 1000);
> > + CHECK_LOOP_ARG (uint16_t, partial_range_h, 127);
> > + return 0;
> > +}
> > diff --git a/gcc/testsuite/gfortran.dg/pr88833.f90
> b/gcc/testsuite/gfortran.dg/pr88833.f90
> > index
> 224e6ce5f3d53135c588b6ac0ab6ee1733eab852..5959cb1d16bf3da90bbb
> dedcb57631610cfc17d0 100644
> > --- a/gcc/testsuite/gfortran.dg/pr88833.f90
> > +++ b/gcc/testsuite/gfortran.dg/pr88833.f90
> > @@ -6,4 +6,4 @@ subroutine foo(x)
> > x = x + 10
> > end subroutine foo
> >
> > -! { dg-final { scan-assembler {\twhilelo\tp[0-9]+\.s, wzr, (w[0-
> 9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }
> > +! { dg-final { scan-assembler-not {\twhilelo\tp[0-9]+\.s, wzr, (w[0-
> 9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }
> >
> >
> > --
> > <rb20686.patch>
> On 9 Jul 2026, at 12:44, Tamar Christina <tamar.christina@arm.com> wrote:
>
>> -----Original Message-----
>> From: Kyrylo Tkachov <ktkachov@nvidia.com>
>> Sent: 09 July 2026 10:06
>> To: Tamar Christina <Tamar.Christina@arm.com>
>> Cc: gcc-patches@gcc.gnu.org; nd <nd@arm.com>; Richard Earnshaw
>> <Richard.Earnshaw@arm.com>; ktkachov@gcc.gnu.org; Alex Coplan
>> <Alex.Coplan@arm.com>; andrew.pinski@oss.qualcomm.com; Wilco Dijkstra
>> <Wilco.Dijkstra@arm.com>; Alice Carlotti <Alice.Carlotti@arm.com>
>> Subject: Re: [patch 3/3]AArch64: use ranger to fold WHILE_ULT to ptrue
>>
>> Hi Tamar,
>>
>>> On 7 Jul 2026, at 12:14, Tamar Christina <tamar.christina@arm.com> wrote:
>>>
>>> Consider the following simple loop:
>>>
>>> void foo (int *x)
>>> {
>>> for (int i = 0; i < 1000; i++)
>>> x[i] *= 2;
>>> }
>>>
>>> compiled at -Ofast -march=armv8-a+sve generates
>>>
>>> foo:
>>> mov w1, 0
>>> cntw x3
>>> mov w2, 1000
>>> whilelo p7.s, wzr, w2
>>> .L2:
>>> ld1w z31.s, p7/z, [x0, x1, lsl 2]
>>> add z31.s, z31.s, z31.s
>>> st1w z31.s, p7, [x0, x1, lsl 2]
>>> add x1, x1, x3
>>> whilelo p7.s, w1, w2
>>> b.any .L2
>>> ret
>>>
>>> Which is nice, but the whilelo in the pre-header is unneeded. Due to the
>>> architecturally defined minimum and maximum vector lengths[1] we know
>> that at
>>> any vector length the predicate is an all lanes active predicate, i.e. p7 is
>>> always ptrue.
>>>
>>> We can use gimple-isel these days while we still have range information on
>>> the operands of .WHILE_ULTs to do this folding.
>>>
>>> As such this patch folds whenever possible WHILE_ULTs into ptrue which are
>>> cheaper to execute and so lowers our costs for entering the loops. i.e. the
>>> above generates:
>>>
>>> foo:
>>> mov w1, 0
>>> cntw x3
>>> mov w2, 1000
>>> ptrue p7.b, all
>>> .L2:
>>> ld1w z31.s, p7/z, [x0, x1, lsl 2]
>>> add z31.s, z31.s, z31.s
>>> st1w z31.s, p7, [x0, x1, lsl 2]
>>> add x1, x1, x3
>>> whilelo p7.s, w1, w2
>>> b.any .L2
>>> ret
>>>
>>> [1] https://developer.arm.com/documentation/102476/0101/Introducing-
>> SVE
>>>
>>> Bootstrapped Regtested on aarch64-none-linux-gnu and no issues.
>>>
>>> Any feedback? otherwise will commit early next week.
>>>
>>> Thanks,
>>> Tamar
>>>
>>> gcc/ChangeLog:
>>>
>>> * config/aarch64/aarch64.cc (aarch64_fold_while_ult_to_ptrue): New.
>>> (aarch64_instruction_selection): Use it.
>>>
>>> gcc/testsuite/ChangeLog:
>>>
>>> * gcc.target/aarch64/sve/fmaxnm_2.c: Update output.
>>> * gcc.target/aarch64/sve/fmaxnm_3.c: Likewise.
>>> * gcc.target/aarch64/sve/fminnm_2.c: Likewise.
>>> * gcc.target/aarch64/sve/fminnm_3.c: Likewise.
>>> * gcc.target/aarch64/sve/slp_12.c: Likewise.
>>> * gcc.target/aarch64/sve/unpacked_fadd_2.c: Likewise.
>>> * gcc.target/aarch64/sve/unpacked_fmul_2.c: Likewise.
>>> * gcc.target/aarch64/sve/unpacked_fsubr_2.c: Likewise.
>>> * gfortran.dg/pr88833.f90: Likewise.
>>> * gcc.target/aarch64/sve/while_ult_1.c: New test.
>>> * gcc.target/aarch64/sve/while_ult_1_run.c: New test.
>>>
>>> ---
>>> diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
>>> index
>> 7a81be8dcbe942ca113f7db2fbab4c357c40f462..630856f95281b4e2c2903
>> 53b3b491404b5dfe633 100644
>>> --- a/gcc/config/aarch64/aarch64.cc
>>> +++ b/gcc/config/aarch64/aarch64.cc
>>> @@ -4442,15 +4442,87 @@ aarch64_fold_sve_ptrue_vl (tree vectype,
>> unsigned int vl,
>>> return builder.build ();
>>> }
>>>
>>> +/* Fold a WHILE_ULT whose bounds are known enough to be expressed as
>> a PTRUE.
>>> + Generic folding cannot do this because it cannot assume an architectural
>>> + maximum for a scalable vector. */
>>> +
>>> +static bool
>>> +aarch64_fold_while_ult_to_ptrue (function *fun, gcall *call,
>>> + gimple_stmt_iterator *gsi)
>>> +{
>>> + if (!gimple_call_internal_p (call, IFN_WHILE_ULT))
>>> + return false;
>>> +
>>> + tree lhs = gimple_call_lhs (call);
>>> + if (!lhs)
>>> + return false;
>>> +
>>> + tree lhs_type = TREE_TYPE (lhs);
>>> + if (!VECTOR_BOOLEAN_TYPE_P (lhs_type)
>>> + || !aarch64_sve_pred_mode_p (TYPE_MODE (lhs_type)))
>>> + return false;
>>> +
>>> + int_range_max min, max;
>>> + range_query *query = get_range_query (fun);
>>> + if (!query->range_of_expr (min, gimple_call_arg (call, 0), call)
>>> + || !query->range_of_expr (max, gimple_call_arg (call, 1), call)
>>> + || min.undefined_p ()
>>> + || max.undefined_p ()
>>> + || !min.nonnegative_p ()
>>> + || !max.nonnegative_p ())
>>> + return false;
>>> +
>>> + widest_int min_upper = widest_int::from (min.upper_bound (),
>> UNSIGNED);
>>> + widest_int min_lower = widest_int::from (min.lower_bound (),
>> UNSIGNED);
>>> + widest_int max_lower = widest_int::from (max.lower_bound (),
>> UNSIGNED);
>>> + widest_int max_upper = widest_int::from (max.upper_bound (),
>> UNSIGNED);
>>> + if (wi::leu_p (max_lower, min_upper))
>>> + return false;
>>> +
>>> + unsigned int min_nelts
>>> + = constant_lower_bound (TYPE_VECTOR_SUBPARTS (lhs_type));
>>> + widest_int max_sve_nelts = min_nelts * 16;
>>
>> For VLS (when TYPE_VECTOR_SUBPARTS (lhs_type).is_constant ()) can we use
>> the exact number of elements rather than the VLA minimum 16?
>>
>
> We don't have WHILE_ULTs for fixed vector length though. When VLS the
> mask is generated explicitly as a constant, .e.. you get a BImode vector with
> {-1,-1,-1,-1,-1, ...}. So at the moment the expectation is that VLS never gets here.
>
> You can get a whilelo synthesized by the predicate expansion code, i.e.
>
> void foo (int *x)
> {
> for (int i = 0; i < 10; i++)
> x[i] *= 2;
> }
>
> With -O3 -march=armv8.5-a+sve whilelt.c -S -o - -mautovec-preference=sve-only -msve-vector-bits=512
> gets you a whilelo but that's done on expansion. While -msve-vector-bits=256 gets you
> the correct ptrue p7.b, vl32.
>
> But I can make it more explicit by rejecting
>
>>> + widest_int min_gap = max_lower - min_upper;
>>> + widest_int max_gap = max_upper - min_lower;
>>> +
>>> + tree pred_cst = NULL_TREE;
>>> + unsigned HOST_WIDE_INT gap = 0;
>>> + if (min_gap == max_gap
>>> + && wi::fits_uhwi_p (min_gap))
>>> + {
>>> + gap = min_gap.to_uhwi ();
>>> + machine_mode pred_mode = TYPE_MODE (lhs_type);
>>> + if (gap <= UINT_MAX
>>> + && (aarch64_svpattern_for_vl (pred_mode, (int) gap)
>>
>> I think this is unsafe for gap = 0x80000000 as that will get cast to a negative
>> int.
>> Consider the test case:
>> void
>> large_exact_gap (int *__restrict a, const int *__restrict b)
>> {
>> for (unsigned long long i = 0; i < 0x80000000ULL; ++i)
>> a[i] = b[i] + 1;
>> }
>>
>> Maybe handling min_gap >= max_sve_nelts first to produce all-ones is better?
>
> Yeah that's a good shout. I had expected aarch64_fold_sve_ptrue_vl to return NULL
> if it can't fold, but looks like it does no validation and so an invalid fold triggers an
> ICE. Will fix.
>
>>
>>> + != AARCH64_NUM_SVPATTERNS))
>>> + pred_cst = aarch64_fold_sve_ptrue_vl (lhs_type,
>>> + (unsigned int) gap, 1);
>>> + }
>>> +
>>> + if (!pred_cst)
>>> + {
>>> + if (wi::ltu_p (min_gap, max_sve_nelts))
>>> + return false;
>>> + pred_cst = build_all_ones_cst (lhs_type);
>>> + }
>>> +
>>> + gassign *assign = gimple_build_assign (lhs, pred_cst);
>>> + gsi_replace (gsi, assign, false);
>>> + return true;
>>> +}
>>> +
>>> /* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used
>> to
>>> change generic sequences to a form AArch64 has an easier time expanding
>>> instructions for. It's not supposed to be used for generic rewriting that
>>> all targets would benefit from. */
>>>
>>> static bool
>>> -aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator
>> *gsi)
>>> +aarch64_instruction_selection (function *fun, gimple_stmt_iterator *gsi)
>>> {
>>> auto stmt = gsi_stmt (*gsi);
>>> + if (gcall *call = dyn_cast<gcall *> (stmt))
>>> + return aarch64_fold_while_ult_to_ptrue (fun, call, gsi);
>>> +
>>> gassign *assign = dyn_cast<gassign *> (stmt);
>>>
>>> if (!assign)
>>> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
>> b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
>>> index
>> ee3cdc20f96550ceeb1e6c11dd464c59eb4b3e6f..3f1dfdd60d238945e2c4fb
>> f4676d3e943cfb3e30 100644
>>> --- a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
>>> +++ b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_2.c
>>> @@ -16,7 +16,7 @@ f2 (double x, double *ptr)
>>> return x;
>>> }
>>>
>>> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-
>> 9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
>>> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-
>> 9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
>>> /* { dg-final { scan-assembler-times {\tfmaxnmv\ts[0-9]+, p[0-7], z[0-
>> 9]+\.s\n} 1 } } */
>>> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-
>> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
>>> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-
>> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
>>> /* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-
>> 9]+\.d\n} 1 } } */
>>> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
>> b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
>>> index
>> a8eee0f4b2698f0425f55ea91c523e462ca134bd..2548e55f2841d370d5d3
>> 0bce23350d3750f094a9 100644
>>> --- a/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
>>> +++ b/gcc/testsuite/gcc.target/aarch64/sve/fmaxnm_3.c
>>> @@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
>>> res[1] = x1;
>>> }
>>>
>>> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-
>> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
>>> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-
>> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
>>> /* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-
>> 9]+\.d\n} 2 } } */
>>> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
>> b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
>>> index
>> 10aced05f1afe69f6e646ac23bac8196c13f8357..ed657d58fdfe663f9ddf5b0
>> 13aa73e14868dbe03 100644
>>> --- a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
>>> +++ b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_2.c
>>> @@ -16,7 +16,7 @@ f2 (double x, double *ptr)
>>> return x;
>>> }
>>>
>>> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-
>> 9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
>>> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-
>> 9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
>>> /* { dg-final { scan-assembler-times {\tfminnmv\ts[0-9]+, p[0-7], z[0-
>> 9]+\.s\n} 1 } } */
>>> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-
>> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
>>> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-
>> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
>>> /* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-
>> 9]+\.d\n} 1 } } */
>>> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
>> b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
>>> index
>> 80ad01602491d07b338113f34d372b5f3970a3be..1f2fc76640fb6c5389c01
>> 85a793ca94446ff0352 100644
>>> --- a/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
>>> +++ b/gcc/testsuite/gcc.target/aarch64/sve/fminnm_3.c
>>> @@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
>>> res[1] = x1;
>>> }
>>>
>>> -/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-
>> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
>>> +/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-
>> 9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
>>> /* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-
>> 9]+\.d\n} 2 } } */
>>> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
>> b/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
>>> index
>> 0b9f8d9bdfaaf868ba0bd73ae83894c68e1613f8..6ba4c9651b67d7e2495b
>> 58ffc0ed01f563c428b6 100644
>>> --- a/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
>>> +++ b/gcc/testsuite/gcc.target/aarch64/sve/slp_12.c
>>> @@ -47,9 +47,9 @@ TEST_ALL (VEC_PERM)
>>>
>>> /* We should use WHILEs for all accesses. */
>>> /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b} 20 } } */
>>> -/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 20 } } */
>>> -/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 30 } } */
>>> -/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 30 } } */
>>> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 18 } } */
>>> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 27 } } */
>>> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 24 } } */
>>>
>>> /* 6 for the 8-bit types and 2 for the 16-bit types. */
>>> /* { dg-final { scan-assembler-times {\tuqdecb\t} 8 } } */
>>> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
>> b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
>>> index
>> 7a74efd129cb1c2bab6f60b26f371d9a0deb3b1d..7ebee0fff77602c0d63a43
>> f00c4be958c1fe5036 100644
>>> --- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
>>> +++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fadd_2.c
>>> @@ -5,7 +5,7 @@
>>>
>>> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
>>> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
>>> -/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 12 } } */
>>> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 15 } } */
>>>
>>> /* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
>>> /* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
>>> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
>> b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
>>> index
>> eb05600e98a052fcd21f9c7f0f29502bbb5484a5..98617a90260a5d0508fe7
>> c59a76a4359c9e18283 100644
>>> --- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
>>> +++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fmul_2.c
>>> @@ -5,7 +5,7 @@
>>>
>>> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
>>> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
>>> -/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
>>> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
>>>
>>> /* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
>>> /* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
>>> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
>> b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
>>> index
>> de9325ccacb00a970ecddd90fb25d609467f74c3..286a1282c402b16bc0733
>> b9c5ca9dbe1c1fb8538 100644
>>> --- a/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
>>> +++ b/gcc/testsuite/gcc.target/aarch64/sve/unpacked_fsubr_2.c
>>> @@ -5,7 +5,7 @@
>>>
>>> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
>>> /* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
>>> -/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
>>> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
>>>
>>> /* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
>>> /* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
>>> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
>> b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
>>> new file mode 100644
>>> index
>> 0000000000000000000000000000000000000000..ddc28029117dadac06
>> 10da3ba2753bc9cc4cc385
>>> --- /dev/null
>>> +++ b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1.c
>>> @@ -0,0 +1,99 @@
>>> +/* { dg-do compile } */
>>> +/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve -
>> msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only -
>> fdump-tree-vect-details --save-temps" } */
>>> +/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 14 "vect" } } */
>>> +
>>> +#include <arm_sve.h>
>>> +#include <stdint.h>
>>> +
>>> +#define NOIPA __attribute__ ((noipa))
>>> +
>>> +#define ADD_LOOP(TYPE, NAME, N) \
>>> + void NOIPA \
>>> + NAME (TYPE *__restrict a, const TYPE *__restrict b) \
>>> + { \
>>> + for (unsigned int i = 0; i < N; ++i) \
>>> + a[i] = b[i] + (TYPE) 1; \
>>> + }
>>> +
>>> +ADD_LOOP (uint8_t, full_b, 300)
>>> +ADD_LOOP (uint16_t, full_h, 1000)
>>> +ADD_LOOP (uint32_t, full_s, 100)
>>> +ADD_LOOP (uint64_t, full_d, 40)
>>> +
>>> +ADD_LOOP (uint8_t, exact_b, 256)
>>> +ADD_LOOP (uint16_t, exact_h, 128)
>>> +ADD_LOOP (uint32_t, exact_s, 64)
>>> +ADD_LOOP (uint64_t, exact_d, 32)
>>> +
>>> +svbool_t NOIPA
>>> +finite_b (void)
>>> +{
>>> + return svptrue_pat_b8 (SV_VL8);
>>> +}
>>
>> These direct svptrue_pat_* calls exercise the existing ACLE folding path moved
>> by patch 2.
>> They do not exercise the new finite-pattern path in
>> aarch64_fold_while_ult_to_ptrue unless I’m missing something?.
>
> No that's correct, it's just checking that the code didn't break in a smaller testcase.
> the aarch64_fold_while_ult_to_ptrue cases are tested by the ADD_LOOP macro ones.
>
> I can remove them If you prefer.
No that’s fine. The patch is okay with the issues above fixed as you see fit.
Thanks,
Kyrill
>
> Thanks,
> Tamar
>
>>
>> Thanks,
>> Kyrill
>>
>>> +
>>> +svbool_t NOIPA
>>> +finite_h (void)
>>> +{
>>> + return svptrue_pat_b16 (SV_VL4);
>>> +}
>>> +
>>> +svbool_t NOIPA
>>> +finite_s (void)
>>> +{
>>> + return svptrue_pat_b32 (SV_VL2);
>>> +}
>>> +
>>> +svbool_t NOIPA
>>> +finite_d (void)
>>> +{
>>> + return svptrue_pat_b64 (SV_VL1);
>>> +}
>>> +
>>> +
>>> +ADD_LOOP (uint8_t, partial_b, 255)
>>> +ADD_LOOP (uint16_t, partial_h, 127)
>>> +ADD_LOOP (uint32_t, partial_s, 63)
>>> +ADD_LOOP (uint64_t, partial_d, 31)
>>> +
>>> +void NOIPA
>>> +full_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
>>> + unsigned int n)
>>> +{
>>> + if (n < 1000)
>>> + __builtin_unreachable ();
>>> + for (unsigned int i = 0; i < n; ++i)
>>> + a[i] = b[i] + (uint16_t) 1;
>>> +}
>>> +
>>> +void NOIPA
>>> +partial_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
>>> + unsigned int n)
>>> +{
>>> + if (n < 100 || n > 127)
>>> + __builtin_unreachable ();
>>> + for (unsigned int i = 0; i < n; ++i)
>>> + a[i] = b[i] + (uint16_t) 1;
>>> +}
>>> +
>>> +/* The first WHILELO in the "full" loops should fold to an all-true
>>> + predicate. GCC prints all-true predicates as .b, regardless of the
>>> + element size of the consuming instruction. */
>>> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, all\n} 9 } } */
>>> +
>>> +/* The finite loops should fold to finite PTRUE patterns. */
>>> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl8\n} 1 } } */
>>> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl4\n} 1 } } */
>>> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s, vl2\n} 1 } } */
>>> +/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl1\n} 1 } } */
>>> +
>>> +/* The "partial" loops should keep their initial WHILELO. */
>>> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, wzr, w[0-9]+} 1 }
>> } */
>>> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, wzr, w[0-9]+} 2 }
>> } */
>>> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, wzr, w[0-9]+} 1 }
>> } */
>>> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, wzr, w[0-9]+} 1 }
>> } */
>>> +
>>> +/* All loops still need a latch WHILELO. */
>>> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, [wx][0-9]+,
>> [wx][0-9]+} 3 } } */
>>> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, [wx][0-9]+,
>> [wx][0-9]+} 5 } } */
>>> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, [wx][0-9]+,
>> [wx][0-9]+} 3 } } */
>>> +/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, [wx][0-9]+,
>> [wx][0-9]+} 3 } } */
>>> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
>> b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
>>> new file mode 100644
>>> index
>> 0000000000000000000000000000000000000000..342b11afce92f567c90
>> ab822f7bdc256c51de7cb
>>> --- /dev/null
>>> +++ b/gcc/testsuite/gcc.target/aarch64/sve/while_ult_1_run.c
>>> @@ -0,0 +1,66 @@
>>> +/* { dg-do run { target aarch64_sve_hw } } */
>>> +/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve -
>> msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only" } */
>>> +
>>> +#include "while_ult_1.c"
>>> +
>>> +#define CHECK_LOOP(TYPE, NAME, N) \
>>> + do \
>>> + { \
>>> + TYPE a[N], b[N]; \
>>> + for (unsigned int i = 0; i < N; ++i) \
>>> + { \
>>> + a[i] = 0; \
>>> + b[i] = (TYPE) (i * 3 + i % 7); \
>>> + } \
>>> + NAME (a, b); \
>>> + for (unsigned int i = 0; i < N; ++i) \
>>> + if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
>>> + __builtin_abort (); \
>>> + } \
>>> + while (0)
>>> +
>>> +#define CHECK_LOOP_ARG(TYPE, NAME, N) \
>>> + do \
>>> + { \
>>> + TYPE a[N], b[N]; \
>>> + for (unsigned int i = 0; i < N; ++i) \
>>> + { \
>>> + a[i] = 0; \
>>> + b[i] = (TYPE) (i * 3 + i % 7); \
>>> + } \
>>> + NAME (a, b, N); \
>>> + for (unsigned int i = 0; i < N; ++i) \
>>> + if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
>>> + __builtin_abort (); \
>>> + } \
>>> + while (0)
>>> +
>>> +int __attribute__ ((optimize (1)))
>>> +main (void)
>>> +{
>>> + CHECK_LOOP (uint8_t, full_b, 300);
>>> + CHECK_LOOP (uint16_t, full_h, 1000);
>>> + CHECK_LOOP (uint32_t, full_s, 100);
>>> + CHECK_LOOP (uint64_t, full_d, 40);
>>> +
>>> + CHECK_LOOP (uint8_t, exact_b, 256);
>>> + CHECK_LOOP (uint16_t, exact_h, 128);
>>> + CHECK_LOOP (uint32_t, exact_s, 64);
>>> + CHECK_LOOP (uint64_t, exact_d, 32);
>>> +
>>> + svbool_t pg = svptrue_b8 ();
>>> + if (svcntp_b8 (pg, finite_b ()) != 8
>>> + || svcntp_b16 (pg, finite_h ()) != 4
>>> + || svcntp_b32 (pg, finite_s ()) != 2
>>> + || svcntp_b64 (pg, finite_d ()) != 1)
>>> + __builtin_abort ();
>>> +
>>> + CHECK_LOOP (uint8_t, partial_b, 255);
>>> + CHECK_LOOP (uint16_t, partial_h, 127);
>>> + CHECK_LOOP (uint32_t, partial_s, 63);
>>> + CHECK_LOOP (uint64_t, partial_d, 31);
>>> +
>>> + CHECK_LOOP_ARG (uint16_t, full_range_h, 1000);
>>> + CHECK_LOOP_ARG (uint16_t, partial_range_h, 127);
>>> + return 0;
>>> +}
>>> diff --git a/gcc/testsuite/gfortran.dg/pr88833.f90
>> b/gcc/testsuite/gfortran.dg/pr88833.f90
>>> index
>> 224e6ce5f3d53135c588b6ac0ab6ee1733eab852..5959cb1d16bf3da90bbb
>> dedcb57631610cfc17d0 100644
>>> --- a/gcc/testsuite/gfortran.dg/pr88833.f90
>>> +++ b/gcc/testsuite/gfortran.dg/pr88833.f90
>>> @@ -6,4 +6,4 @@ subroutine foo(x)
>>> x = x + 10
>>> end subroutine foo
>>>
>>> -! { dg-final { scan-assembler {\twhilelo\tp[0-9]+\.s, wzr, (w[0-
>> 9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }
>>> +! { dg-final { scan-assembler-not {\twhilelo\tp[0-9]+\.s, wzr, (w[0-
>> 9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }
>>>
>>>
>>> --
>>> <rb20686.patch>
@@ -4442,15 +4442,87 @@ aarch64_fold_sve_ptrue_vl (tree vectype, unsigned int vl,
return builder.build ();
}
+/* Fold a WHILE_ULT whose bounds are known enough to be expressed as a PTRUE.
+ Generic folding cannot do this because it cannot assume an architectural
+ maximum for a scalable vector. */
+
+static bool
+aarch64_fold_while_ult_to_ptrue (function *fun, gcall *call,
+ gimple_stmt_iterator *gsi)
+{
+ if (!gimple_call_internal_p (call, IFN_WHILE_ULT))
+ return false;
+
+ tree lhs = gimple_call_lhs (call);
+ if (!lhs)
+ return false;
+
+ tree lhs_type = TREE_TYPE (lhs);
+ if (!VECTOR_BOOLEAN_TYPE_P (lhs_type)
+ || !aarch64_sve_pred_mode_p (TYPE_MODE (lhs_type)))
+ return false;
+
+ int_range_max min, max;
+ range_query *query = get_range_query (fun);
+ if (!query->range_of_expr (min, gimple_call_arg (call, 0), call)
+ || !query->range_of_expr (max, gimple_call_arg (call, 1), call)
+ || min.undefined_p ()
+ || max.undefined_p ()
+ || !min.nonnegative_p ()
+ || !max.nonnegative_p ())
+ return false;
+
+ widest_int min_upper = widest_int::from (min.upper_bound (), UNSIGNED);
+ widest_int min_lower = widest_int::from (min.lower_bound (), UNSIGNED);
+ widest_int max_lower = widest_int::from (max.lower_bound (), UNSIGNED);
+ widest_int max_upper = widest_int::from (max.upper_bound (), UNSIGNED);
+ if (wi::leu_p (max_lower, min_upper))
+ return false;
+
+ unsigned int min_nelts
+ = constant_lower_bound (TYPE_VECTOR_SUBPARTS (lhs_type));
+ widest_int max_sve_nelts = min_nelts * 16;
+ widest_int min_gap = max_lower - min_upper;
+ widest_int max_gap = max_upper - min_lower;
+
+ tree pred_cst = NULL_TREE;
+ unsigned HOST_WIDE_INT gap = 0;
+ if (min_gap == max_gap
+ && wi::fits_uhwi_p (min_gap))
+ {
+ gap = min_gap.to_uhwi ();
+ machine_mode pred_mode = TYPE_MODE (lhs_type);
+ if (gap <= UINT_MAX
+ && (aarch64_svpattern_for_vl (pred_mode, (int) gap)
+ != AARCH64_NUM_SVPATTERNS))
+ pred_cst = aarch64_fold_sve_ptrue_vl (lhs_type,
+ (unsigned int) gap, 1);
+ }
+
+ if (!pred_cst)
+ {
+ if (wi::ltu_p (min_gap, max_sve_nelts))
+ return false;
+ pred_cst = build_all_ones_cst (lhs_type);
+ }
+
+ gassign *assign = gimple_build_assign (lhs, pred_cst);
+ gsi_replace (gsi, assign, false);
+ return true;
+}
+
/* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used to
change generic sequences to a form AArch64 has an easier time expanding
instructions for. It's not supposed to be used for generic rewriting that
all targets would benefit from. */
static bool
-aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator *gsi)
+aarch64_instruction_selection (function *fun, gimple_stmt_iterator *gsi)
{
auto stmt = gsi_stmt (*gsi);
+ if (gcall *call = dyn_cast<gcall *> (stmt))
+ return aarch64_fold_while_ult_to_ptrue (fun, call, gsi);
+
gassign *assign = dyn_cast<gassign *> (stmt);
if (!assign)
@@ -16,7 +16,7 @@ f2 (double x, double *ptr)
return x;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.s,.*\tfmaxnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
/* { dg-final { scan-assembler-times {\tfmaxnmv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
/* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
@@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
res[1] = x1;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfmaxnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
/* { dg-final { scan-assembler-times {\tfmaxnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 2 } } */
@@ -16,7 +16,7 @@ f2 (double x, double *ptr)
return x;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.s,.*\tfminnm\tz[0-9]+\.s, \1/m, z[0-9]+\.s, z[0-9]+\.s\n} } } */
/* { dg-final { scan-assembler-times {\tfminnmv\ts[0-9]+, p[0-7], z[0-9]+\.s\n} 1 } } */
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
/* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 1 } } */
@@ -14,5 +14,5 @@ f (double *restrict res, double *restrict ptr)
res[1] = x1;
}
-/* { dg-final { scan-assembler {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
+/* { dg-final { scan-assembler-not {\twhilelo\t(p[0-7])\.d,.*\tfminnm\tz[0-9]+\.d, \1/m, z[0-9]+\.d, z[0-9]+\.d\n} } } */
/* { dg-final { scan-assembler-times {\tfminnmv\td[0-9]+, p[0-7], z[0-9]+\.d\n} 2 } } */
@@ -47,9 +47,9 @@ TEST_ALL (VEC_PERM)
/* We should use WHILEs for all accesses. */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b} 20 } } */
-/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 20 } } */
-/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 30 } } */
-/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 30 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h} 18 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s} 27 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d} 24 } } */
/* 6 for the 8-bit types and 2 for the 16-bit types. */
/* { dg-final { scan-assembler-times {\tuqdecb\t} 8 } } */
@@ -5,7 +5,7 @@
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 12 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 15 } } */
/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 11 } } */
/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 11 } } */
@@ -5,7 +5,7 @@
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 3 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 5 } } */
/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 5 } } */
@@ -5,7 +5,7 @@
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.s} } } */
/* { dg-final { scan-assembler-not {\tptrue\tp[0-7]\.d} } } */
-/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 6 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b} 9 } } */
/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.d} 7 } } */
/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.s} 7 } } */
new file mode 100644
@@ -0,0 +1,99 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve -msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only -fdump-tree-vect-details --save-temps" } */
+/* { dg-final { scan-tree-dump-times "LOOP VECTORIZED" 14 "vect" } } */
+
+#include <arm_sve.h>
+#include <stdint.h>
+
+#define NOIPA __attribute__ ((noipa))
+
+#define ADD_LOOP(TYPE, NAME, N) \
+ void NOIPA \
+ NAME (TYPE *__restrict a, const TYPE *__restrict b) \
+ { \
+ for (unsigned int i = 0; i < N; ++i) \
+ a[i] = b[i] + (TYPE) 1; \
+ }
+
+ADD_LOOP (uint8_t, full_b, 300)
+ADD_LOOP (uint16_t, full_h, 1000)
+ADD_LOOP (uint32_t, full_s, 100)
+ADD_LOOP (uint64_t, full_d, 40)
+
+ADD_LOOP (uint8_t, exact_b, 256)
+ADD_LOOP (uint16_t, exact_h, 128)
+ADD_LOOP (uint32_t, exact_s, 64)
+ADD_LOOP (uint64_t, exact_d, 32)
+
+svbool_t NOIPA
+finite_b (void)
+{
+ return svptrue_pat_b8 (SV_VL8);
+}
+
+svbool_t NOIPA
+finite_h (void)
+{
+ return svptrue_pat_b16 (SV_VL4);
+}
+
+svbool_t NOIPA
+finite_s (void)
+{
+ return svptrue_pat_b32 (SV_VL2);
+}
+
+svbool_t NOIPA
+finite_d (void)
+{
+ return svptrue_pat_b64 (SV_VL1);
+}
+
+
+ADD_LOOP (uint8_t, partial_b, 255)
+ADD_LOOP (uint16_t, partial_h, 127)
+ADD_LOOP (uint32_t, partial_s, 63)
+ADD_LOOP (uint64_t, partial_d, 31)
+
+void NOIPA
+full_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
+ unsigned int n)
+{
+ if (n < 1000)
+ __builtin_unreachable ();
+ for (unsigned int i = 0; i < n; ++i)
+ a[i] = b[i] + (uint16_t) 1;
+}
+
+void NOIPA
+partial_range_h (uint16_t *__restrict a, const uint16_t *__restrict b,
+ unsigned int n)
+{
+ if (n < 100 || n > 127)
+ __builtin_unreachable ();
+ for (unsigned int i = 0; i < n; ++i)
+ a[i] = b[i] + (uint16_t) 1;
+}
+
+/* The first WHILELO in the "full" loops should fold to an all-true
+ predicate. GCC prints all-true predicates as .b, regardless of the
+ element size of the consuming instruction. */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, all\n} 9 } } */
+
+/* The finite loops should fold to finite PTRUE patterns. */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl8\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.h, vl4\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.s, vl2\n} 1 } } */
+/* { dg-final { scan-assembler-times {\tptrue\tp[0-7]\.b, vl1\n} 1 } } */
+
+/* The "partial" loops should keep their initial WHILELO. */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, wzr, w[0-9]+} 1 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, wzr, w[0-9]+} 2 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, wzr, w[0-9]+} 1 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, wzr, w[0-9]+} 1 } } */
+
+/* All loops still need a latch WHILELO. */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.b, [wx][0-9]+, [wx][0-9]+} 3 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.h, [wx][0-9]+, [wx][0-9]+} 5 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, [wx][0-9]+, [wx][0-9]+} 3 } } */
+/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, [wx][0-9]+, [wx][0-9]+} 3 } } */
new file mode 100644
@@ -0,0 +1,66 @@
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O3 -ftree-vectorize -fopenmp-simd -march=armv8-a+sve -msve-vector-bits=scalable --param aarch64-autovec-preference=sve-only" } */
+
+#include "while_ult_1.c"
+
+#define CHECK_LOOP(TYPE, NAME, N) \
+ do \
+ { \
+ TYPE a[N], b[N]; \
+ for (unsigned int i = 0; i < N; ++i) \
+ { \
+ a[i] = 0; \
+ b[i] = (TYPE) (i * 3 + i % 7); \
+ } \
+ NAME (a, b); \
+ for (unsigned int i = 0; i < N; ++i) \
+ if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
+ __builtin_abort (); \
+ } \
+ while (0)
+
+#define CHECK_LOOP_ARG(TYPE, NAME, N) \
+ do \
+ { \
+ TYPE a[N], b[N]; \
+ for (unsigned int i = 0; i < N; ++i) \
+ { \
+ a[i] = 0; \
+ b[i] = (TYPE) (i * 3 + i % 7); \
+ } \
+ NAME (a, b, N); \
+ for (unsigned int i = 0; i < N; ++i) \
+ if (a[i] != (TYPE) (b[i] + (TYPE) 1)) \
+ __builtin_abort (); \
+ } \
+ while (0)
+
+int __attribute__ ((optimize (1)))
+main (void)
+{
+ CHECK_LOOP (uint8_t, full_b, 300);
+ CHECK_LOOP (uint16_t, full_h, 1000);
+ CHECK_LOOP (uint32_t, full_s, 100);
+ CHECK_LOOP (uint64_t, full_d, 40);
+
+ CHECK_LOOP (uint8_t, exact_b, 256);
+ CHECK_LOOP (uint16_t, exact_h, 128);
+ CHECK_LOOP (uint32_t, exact_s, 64);
+ CHECK_LOOP (uint64_t, exact_d, 32);
+
+ svbool_t pg = svptrue_b8 ();
+ if (svcntp_b8 (pg, finite_b ()) != 8
+ || svcntp_b16 (pg, finite_h ()) != 4
+ || svcntp_b32 (pg, finite_s ()) != 2
+ || svcntp_b64 (pg, finite_d ()) != 1)
+ __builtin_abort ();
+
+ CHECK_LOOP (uint8_t, partial_b, 255);
+ CHECK_LOOP (uint16_t, partial_h, 127);
+ CHECK_LOOP (uint32_t, partial_s, 63);
+ CHECK_LOOP (uint64_t, partial_d, 31);
+
+ CHECK_LOOP_ARG (uint16_t, full_range_h, 1000);
+ CHECK_LOOP_ARG (uint16_t, partial_range_h, 127);
+ return 0;
+}
@@ -6,4 +6,4 @@ subroutine foo(x)
x = x + 10
end subroutine foo
-! { dg-final { scan-assembler {\twhilelo\tp[0-9]+\.s, wzr, (w[0-9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }
+! { dg-final { scan-assembler-not {\twhilelo\tp[0-9]+\.s, wzr, (w[0-9]+).*\twhilelo\tp[0-9]+\.s, w[0-9]+, \1} } }