diff mbox series

[01/13] match-sat-alu.pd: Allow wider ranges for signed saturating truncation

Message ID 20260902145254.77832-3-ktkachov@nvidia.com
State New
Headers
Series Saturating arithmetic matching improvements |

Commit Message

Kyrylo Tkachov Sept. 2, 2026, 2:52 p.m. UTC 
  From: Kyrylo Tkachov <ktkachov@nvidia.com>

Signed saturating truncation currently requires the unsigned range type to
have the same precision as the signed input.  This rejects a wider range type
even though it preserves every input value.

Allow the range type to be at least as wide as the input.  Continue to require
a result narrower than the input.  Construct the offset and limit constants
at the range precision.  Use wide-int operations that do not depend on
host-width shifts.

Tests cover a valid 16-bit input with a 32-bit range and an 8-bit result.  They
also verify that a 17-bit range for a 32-bit input remains rejected because it
can discard input bits.

With AArch64 -O3 -march=armv8-a -mmax-vectorization and
--param=vect-epilogues-nomask=0:

before, 16 elements per iteration:

	ldp	q28, q27, [x3], 32
	uzp1	v26.16b, v28.16b, v27.16b
	saddl	v0.4s, v28.4h, v31.4h
	saddl2	v24.4s, v28.8h, v31.8h
	saddl	v23.4s, v27.4h, v31.4h
	saddl2	v25.4s, v27.8h, v31.8h
	ushr	v28.8h, v28.8h, 15
	ushr	v27.8h, v27.8h, 15
	cmhi	v0.4s, v0.4s, v30.4s
	cmhi	v24.4s, v24.4s, v30.4s
	cmhi	v23.4s, v23.4s, v30.4s
	cmhi	v25.4s, v25.4s, v30.4s
	uzp1	v27.16b, v28.16b, v27.16b
	uzp1	v24.8h, v0.8h, v24.8h
	uzp1	v25.8h, v23.8h, v25.8h
	neg	v27.16b, v27.16b
	uzp1	v25.16b, v24.16b, v25.16b
	eor	v27.16b, v27.16b, v29.16b
	bif	v27.16b, v26.16b, v25.16b
	str	q27, [x4], 16

after, 8 elements per iteration:

	ldr	q31, [x1, x4]
	sqxtn	v31.8b, v31.8h
	str	d31, [x0, x3]

Bootstrapped and tested on aarch64-none-linux-gnu.
Tested on x86_64-pc-linux-gnu.

Ok for trunk?

gcc/ChangeLog:

	* match-sat-alu.pd (signed_integer_sat_trunc): Allow a wider lossless
	unsigned range type.  Continue to require a narrowing result.  Construct
	the expected constants at the range precision.

gcc/testsuite/ChangeLog:

	* gcc.dg/vect/vect-sat-trunc-range-precision-1.c: New test.
	* gcc.target/aarch64/vect-sat-trunc-range-precision-1.c: Likewise.
	* gcc.target/aarch64/vect-sat-trunc-range-precision-negative-1.c:
	Likewise.

Signed-off-by: Kyrylo Tkachov <ktkachov@nvidia.com>
---
 gcc/match-sat-alu.pd                          |  29 ++---
 .../vect/vect-sat-trunc-range-precision-1.c   | 122 ++++++++++++++++++
 .../vect-sat-trunc-range-precision-1.c        |   7 +
 ...ect-sat-trunc-range-precision-negative-1.c |  24 ++++
 4 files changed, 167 insertions(+), 15 deletions(-)
 create mode 100644 gcc/testsuite/gcc.dg/vect/vect-sat-trunc-range-precision-1.c
 create mode 100644 gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-range-precision-1.c
 create mode 100644 gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-range-precision-negative-1.c
diff mbox series

Patch

diff --git a/gcc/match-sat-alu.pd b/gcc/match-sat-alu.pd
index 22855726d90..7156529f589 100644
--- a/gcc/match-sat-alu.pd
+++ b/gcc/match-sat-alu.pd
@@ -437,25 +437,24 @@  along with GCC; see the file COPYING3.  If not see
 		     (negate (nop_convert? (convert (lt @0 integer_zerop)))))
 		    INTEGER_CST@3)
 	 (convert @0))
-  /* The comparison has to be the unsigned reinterpretation of X, and the
-     conversion has to narrow, otherwise the constants below do not have the
-     precision the comparison is carried out at.  */
-  (if (!TYPE_UNSIGNED (TREE_TYPE (@0)) && TYPE_UNSIGNED (TREE_TYPE (@4))
-       && TYPE_PRECISION (TREE_TYPE (@4)) == TYPE_PRECISION (TREE_TYPE (@0))
-       && TYPE_PRECISION (type) < TYPE_PRECISION (TREE_TYPE (@0)))
+  /* The comparison type has to be unsigned and at least as wide as X.  A
+     narrower type would examine only the low bits of X.  The result
+     conversion has to narrow X.  */
+  (if (!TYPE_UNSIGNED (TREE_TYPE (@0))
+	&& TYPE_UNSIGNED (TREE_TYPE (@4))
+	&& (TYPE_PRECISION (TREE_TYPE (@4))
+	    >= TYPE_PRECISION (TREE_TYPE (@0)))
+	&& TYPE_PRECISION (type) < TYPE_PRECISION (TREE_TYPE (@0)))
    (with
     {
-     unsigned itype_prec = TYPE_PRECISION (TREE_TYPE (@0));
      unsigned otype_prec = TYPE_PRECISION (type);
-     wide_int offset = wi::uhwi (HOST_WIDE_INT_1U << (otype_prec - 1),
-				 itype_prec); // Aka 128 for int8_t
-     wide_int limit_0 = wi::mask (otype_prec, false, itype_prec); // Aka 255
-     wide_int limit_1 = wi::uhwi ((HOST_WIDE_INT_1U << otype_prec) - 3,
-				  itype_prec); // Aka 253
-     wide_int limit_2 = wi::uhwi ((HOST_WIDE_INT_1U << otype_prec) - 2,
-				  itype_prec); // Aka 254
+     unsigned range_prec = TYPE_PRECISION (TREE_TYPE (@4));
+     wide_int offset = wi::set_bit_in_zero (otype_prec - 1, range_prec);
+     wide_int limit_0 = wi::mask (otype_prec, false, range_prec); // Aka 255
+     wide_int limit_1 = limit_0 - 2; // Aka 253
+     wide_int limit_2 = limit_0 - 1; // Aka 254
      wide_int otype_max = wi::mask (otype_prec - 1, false, otype_prec);
-     wide_int itype_max = wi::mask (otype_prec - 1, false, itype_prec);
+     wide_int itype_max = wi::mask (otype_prec - 1, false, range_prec);
      wide_int int_cst_1 = wi::to_wide (@1);
      wide_int int_cst_2 = wi::to_wide (@2);
      wide_int int_cst_3 = wi::to_wide (@3);
diff --git a/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-range-precision-1.c b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-range-precision-1.c
new file mode 100644
index 00000000000..b4e11f3a973
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-sat-trunc-range-precision-1.c
@@ -0,0 +1,122 @@ 
+/* { dg-do run { target bitint } } */
+/* { dg-additional-options "-O3 -fvect-cost-model=unlimited" } */
+
+typedef __INT8_TYPE__ int8_t;
+typedef __INT16_TYPE__ int16_t;
+typedef __INT32_TYPE__ int32_t;
+typedef __UINT16_TYPE__ uint16_t;
+typedef __UINT32_TYPE__ uint32_t;
+typedef signed _BitInt(16) int16b_t;
+typedef unsigned _BitInt(17) uint17_t;
+
+#define N 259
+#define SAT_VALUE(OUT, X, MAX) ((OUT) (-((OUT) ((X) < 0)) ^ (OUT) MAX))
+
+__attribute__((noipa))
+static void
+wide_range (int8_t *__restrict out, const int16b_t *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      int16b_t source = in[i];
+      uint32_t range = (uint32_t) source + 128U;
+      out[i] = (range > 255U
+		? SAT_VALUE (int8_t, source, 127) : (int8_t) source);
+    }
+}
+
+__attribute__((noipa))
+static void
+narrow_range (int16_t *__restrict out, const int32_t *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      int32_t source = in[i];
+      uint17_t range = (uint17_t) source + (uint17_t) 32768;
+      out[i] = (range > (uint17_t) 65535
+		? SAT_VALUE (int16_t, source, 32767) : (int16_t) source);
+    }
+}
+
+__attribute__((noipa, optimize ("O0")))
+static void
+wide_range_ref (int8_t *out, const int16b_t *in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      int16b_t source = in[i];
+      if (source < -128)
+	out[i] = -128;
+      else if (source > 127)
+	out[i] = 127;
+      else
+	out[i] = (int8_t) source;
+    }
+}
+
+__attribute__((noipa, optimize ("O0")))
+static void
+narrow_range_ref (int16_t *out, const int32_t *in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      int32_t source = in[i];
+      uint17_t range = (uint17_t) source + (uint17_t) 32768;
+      out[i] = (range > (uint17_t) 65535
+		? SAT_VALUE (int16_t, source, 32767) : (int16_t) source);
+    }
+}
+
+static void
+check_wide_range (void)
+{
+  int8_t out[N];
+  int8_t ref[N];
+  int16b_t in[N];
+
+  for (int i = 0; i < N; ++i)
+    in[i] = (uint16_t) i * 40503U + 97U;
+
+  for (int n = 0; n <= N; ++n)
+    {
+      for (int i = 0; i < N; ++i)
+	out[i] = ref[i] = 23;
+      wide_range (out, in, n);
+      wide_range_ref (ref, in, n);
+      for (int i = 0; i < N; ++i)
+	if (out[i] != ref[i])
+	  __builtin_abort ();
+    }
+}
+
+static void
+check_narrow_range (void)
+{
+  int16_t out[N];
+  int16_t ref[N];
+  int32_t in[N];
+
+  for (int i = 0; i < N; ++i)
+    in[i] = (int32_t) ((uint32_t) i * 2654435761U + 1013904223U);
+  in[0] = 131072;
+
+  for (int n = 0; n <= N; ++n)
+    {
+      for (int i = 0; i < N; ++i)
+	out[i] = ref[i] = 23;
+      narrow_range (out, in, n);
+      narrow_range_ref (ref, in, n);
+      for (int i = 0; i < N; ++i)
+	if (out[i] != ref[i])
+	  __builtin_abort ();
+    }
+}
+
+int
+main (void)
+{
+  check_wide_range ();
+  check_narrow_range ();
+  return 0;
+}
+
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-range-precision-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-range-precision-1.c
new file mode 100644
index 00000000000..c52b309cbe1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-range-precision-1.c
@@ -0,0 +1,7 @@ 
+/* { dg-do compile { target bitint } } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+#include "../../gcc.dg/vect/vect-sat-trunc-range-precision-1.c"
+
+/* { dg-final { scan-tree-dump-times "sat_trunc pattern recognized" 1 "vect" } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-range-precision-negative-1.c b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-range-precision-negative-1.c
new file mode 100644
index 00000000000..8c30f8d7632
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/vect-sat-trunc-range-precision-negative-1.c
@@ -0,0 +1,24 @@ 
+/* { dg-do compile { target bitint } } */
+/* { dg-options "-O3 -march=armv8-a" } */
+/* { dg-additional-options "-mmax-vectorization --param=vect-epilogues-nomask=0 -fdump-tree-vect-details" } */
+
+typedef __INT16_TYPE__ int16_t;
+typedef __INT32_TYPE__ int32_t;
+typedef unsigned _BitInt(17) uint17_t;
+
+#define SAT_VALUE(X) ((int16_t) (-((int16_t) ((X) < 0)) ^ 32767))
+
+__attribute__((noipa))
+void
+narrow_range (int16_t *__restrict out, const int32_t *__restrict in, int n)
+{
+  for (int i = 0; i < n; ++i)
+    {
+      int32_t source = in[i];
+      uint17_t range = (uint17_t) source + (uint17_t) 32768;
+      out[i] = (range > (uint17_t) 65535
+		? SAT_VALUE (source) : (int16_t) source);
+    }
+}
+
+/* { dg-final { scan-tree-dump-not "sat_trunc pattern recognized" "vect" } } */