diff mbox series

[v3,11/12] math: Shorten the sinf/cosf/sincosf fast path

Message ID 20260923215118.2608685-12-adhemerval.zanella@linaro.org (mailing list archive)
State New
Headers
Series Add sinf/cosf/sincosf CORE-MATH implementations |

Checks

Context Check Description
redhat-pt-bot/TryBot-apply_patch success Patch applied to master at the time it was sent
linaro-tcwg-bot/tcwg_glibc_build--master-arm success Build passed
linaro-tcwg-bot/tcwg_glibc_build--master-aarch64 success Build passed
linaro-tcwg-bot/tcwg_glibc_check--master-aarch64 success Test passed
linaro-tcwg-bot/tcwg_glibc_check--master-arm success Test passed

Commit Message

Adhemerval Zanella Netto Sept. 23, 2026, 9:50 p.m. UTC 
  Converting the rounded value with (int) id gives the same low bits and
lets gcc on aarch64 fold the rounding and the conversion into a
single fcvtns of the scaled argument (and it s cvttsd2si on x86_64).

The polynomial coefficients and the sin/cos tables are also grouped i
one hidden object, so a function materializes one base address instead of
one per table.

On aarch64 with gcc 16 the fast path goes from 59 to 54 instructions
for sinf, 55 to 50 for cosf and 63 to 58 for sincosf, with 4 instead
of 8 register moves and 3 instead of 5 adrp.

Measured against the previous commit (gcc 16.2.1) with the benchtests
inputs that preceded the workload rewrite of this series (random
[-2pi, 2pi], large [-64, 64], huge [-1024, 1024]), geomean of the
reciprocal-throughput and latency, negative is faster:

                      sinf     cosf  sincosf
x86_64           +1.3%    -0.4%    +0.3%
x86_64-fma       +0.9%    -2.6%    -1.1%
aarch64          -0.1%    +0.4%    -6.0%
power10          +2.3%    +2.9%    +0.2%
loongarch        -0.6%    -1.4%    -0.5%

The aarch64 sincosf gain comes from the large and huge inputs (-8% and
-9% reciprocal-throughput).

Checked on x86_64-linux-gnu, aarch64-linux-gnu, powerpc64le-linux-gnu
and loongarch64-linux-gnuf64.
---
 sysdeps/ieee754/flt-32/s_sincosf_common.h |  4 +-
 sysdeps/ieee754/flt-32/s_sincosf_data.c   | 76 ++++++++++++-----------
 sysdeps/ieee754/flt-32/s_sincosf_data.h   | 21 ++++---
 3 files changed, 53 insertions(+), 48 deletions(-)
diff mbox series

Patch

diff --git a/sysdeps/ieee754/flt-32/s_sincosf_common.h b/sysdeps/ieee754/flt-32/s_sincosf_common.h
index a15a0afe806..c7f385d07eb 100644
--- a/sysdeps/ieee754/flt-32/s_sincosf_common.h
+++ b/sysdeps/ieee754/flt-32/s_sincosf_common.h
@@ -34,7 +34,7 @@  static inline double
 rltl0 (double x, int *q)
 {
   double idh = 0x1.45f306dc9c883p+2 * x, id = roundeven_finite (idh);
-  *q = asuint64 (0x1.8p52 + id);
+  *q = (int) id;
   return idh - id;
 }
 
@@ -44,7 +44,7 @@  rltl (float z, int *q)
   double x = z;
   double idl = -0x1.b1bbead603d8bp-29 * x, idh = 0x1.45f306ep+2 * x,
 	 id = roundeven_finite (idh);
-  *q = asuint64 (0x1.8p52 + id);
+  *q = (int) id;
   return (idh - id) + idl;
 }
 
diff --git a/sysdeps/ieee754/flt-32/s_sincosf_data.c b/sysdeps/ieee754/flt-32/s_sincosf_data.c
index a86640c4048..d248db8c08f 100644
--- a/sysdeps/ieee754/flt-32/s_sincosf_data.c
+++ b/sysdeps/ieee754/flt-32/s_sincosf_data.c
@@ -33,44 +33,46 @@  const uint64_t __sinf_ipi[] =
   0xa2f9836e4e441529
 };
 
-const double __sinf_b[] =
+const sincosf_tables_t __sincosf_tables =
 {
-   0x1.3bd3cc9be45dcp-6, -0x1.03c1f081b0833p-14,  0x1.55d3c6fc9ac1fp-24,
-  -0x1.e1d3ff281b40dp-35
-};
-const double __sinf_a[] =
-{
-   0x1.921fb54442d17p-3, -0x1.4abbce6256a39p-10,   0x1.466bc5a518c16p-19,
-  -0x1.32bdc61074ff6p-29
-};
-const double __sinf_tb[] =
-{
-   0x0p+0,                0x1.8f8b83c69a60bp-3,  0x1.87de2a6aea963p-2,
-   0x1.1c73b39ae68c8p-1,  0x1.6a09e667f3bcdp-1,  0x1.a9b66290ea1a3p-1,
-   0x1.d906bcf328d46p-1,  0x1.f6297cff75cbp-1,   0x1p+0,
-   0x1.f6297cff75cbp-1,   0x1.d906bcf328d46p-1,  0x1.a9b66290ea1a3p-1,
-   0x1.6a09e667f3bcdp-1,  0x1.1c73b39ae68c8p-1,  0x1.87de2a6aea963p-2,
-   0x1.8f8b83c69a60bp-3,  0x0p+0,               -0x1.8f8b83c69a60bp-3,
-  -0x1.87de2a6aea963p-2, -0x1.1c73b39ae68c8p-1, -0x1.6a09e667f3bcdp-1,
-  -0x1.a9b66290ea1a3p-1, -0x1.d906bcf328d46p-1, -0x1.f6297cff75cbp-1,
-  -0x1p+0,               -0x1.f6297cff75cbp-1,  -0x1.d906bcf328d46p-1,
-  -0x1.a9b66290ea1a3p-1, -0x1.6a09e667f3bcdp-1, -0x1.1c73b39ae68c8p-1,
-  -0x1.87de2a6aea963p-2, -0x1.8f8b83c69a60bp-3
-};
-
-const double __cosf_tb[] =
-{
-   0x1p+0,                0x1.f6297cff75cbp-1,   0x1.d906bcf328d46p-1,
-   0x1.a9b66290ea1a3p-1,  0x1.6a09e667f3bcdp-1,  0x1.1c73b39ae68c8p-1,
-   0x1.87de2a6aea963p-2,  0x1.8f8b83c69a60bp-3,  0x0p+0,
-  -0x1.8f8b83c69a60bp-3, -0x1.87de2a6aea963p-2, -0x1.1c73b39ae68c8p-1,
-  -0x1.6a09e667f3bcdp-1, -0x1.a9b66290ea1a3p-1, -0x1.d906bcf328d46p-1,
-  -0x1.f6297cff75cbp-1,  -0x1p+0,               -0x1.f6297cff75cbp-1,
-  -0x1.d906bcf328d46p-1, -0x1.a9b66290ea1a3p-1, -0x1.6a09e667f3bcdp-1,
-  -0x1.1c73b39ae68c8p-1, -0x1.87de2a6aea963p-2, -0x1.8f8b83c69a60bp-3,
-   0x0p+0,                0x1.8f8b83c69a60bp-3,  0x1.87de2a6aea963p-2,
-   0x1.1c73b39ae68c8p-1,  0x1.6a09e667f3bcdp-1,  0x1.a9b66290ea1a3p-1,
-   0x1.d906bcf328d46p-1,  0x1.f6297cff75cbp-1
+  .b =
+  {
+     0x1.3bd3cc9be45dcp-6, -0x1.03c1f081b0833p-14,  0x1.55d3c6fc9ac1fp-24,
+    -0x1.e1d3ff281b40dp-35
+  },
+  .a =
+  {
+     0x1.921fb54442d17p-3, -0x1.4abbce6256a39p-10,   0x1.466bc5a518c16p-19,
+    -0x1.32bdc61074ff6p-29
+  },
+  .tb =
+  {
+     0x0p+0,                0x1.8f8b83c69a60bp-3,  0x1.87de2a6aea963p-2,
+     0x1.1c73b39ae68c8p-1,  0x1.6a09e667f3bcdp-1,  0x1.a9b66290ea1a3p-1,
+     0x1.d906bcf328d46p-1,  0x1.f6297cff75cbp-1,   0x1p+0,
+     0x1.f6297cff75cbp-1,   0x1.d906bcf328d46p-1,  0x1.a9b66290ea1a3p-1,
+     0x1.6a09e667f3bcdp-1,  0x1.1c73b39ae68c8p-1,  0x1.87de2a6aea963p-2,
+     0x1.8f8b83c69a60bp-3,  0x0p+0,               -0x1.8f8b83c69a60bp-3,
+    -0x1.87de2a6aea963p-2, -0x1.1c73b39ae68c8p-1, -0x1.6a09e667f3bcdp-1,
+    -0x1.a9b66290ea1a3p-1, -0x1.d906bcf328d46p-1, -0x1.f6297cff75cbp-1,
+    -0x1p+0,               -0x1.f6297cff75cbp-1,  -0x1.d906bcf328d46p-1,
+    -0x1.a9b66290ea1a3p-1, -0x1.6a09e667f3bcdp-1, -0x1.1c73b39ae68c8p-1,
+    -0x1.87de2a6aea963p-2, -0x1.8f8b83c69a60bp-3
+  },
+  .tb_cosf =
+  {
+     0x1p+0,                0x1.f6297cff75cbp-1,   0x1.d906bcf328d46p-1,
+     0x1.a9b66290ea1a3p-1,  0x1.6a09e667f3bcdp-1,  0x1.1c73b39ae68c8p-1,
+     0x1.87de2a6aea963p-2,  0x1.8f8b83c69a60bp-3,  0x0p+0,
+    -0x1.8f8b83c69a60bp-3, -0x1.87de2a6aea963p-2, -0x1.1c73b39ae68c8p-1,
+    -0x1.6a09e667f3bcdp-1, -0x1.a9b66290ea1a3p-1, -0x1.d906bcf328d46p-1,
+    -0x1.f6297cff75cbp-1,  -0x1p+0,               -0x1.f6297cff75cbp-1,
+    -0x1.d906bcf328d46p-1, -0x1.a9b66290ea1a3p-1, -0x1.6a09e667f3bcdp-1,
+    -0x1.1c73b39ae68c8p-1, -0x1.87de2a6aea963p-2, -0x1.8f8b83c69a60bp-3,
+     0x0p+0,                0x1.8f8b83c69a60bp-3,  0x1.87de2a6aea963p-2,
+     0x1.1c73b39ae68c8p-1,  0x1.6a09e667f3bcdp-1,  0x1.a9b66290ea1a3p-1,
+     0x1.d906bcf328d46p-1,  0x1.f6297cff75cbp-1
+  },
 };
 
 const sincosf_database_t __sinf_st[] =
diff --git a/sysdeps/ieee754/flt-32/s_sincosf_data.h b/sysdeps/ieee754/flt-32/s_sincosf_data.h
index ca7164de52b..7d9a7028f39 100644
--- a/sysdeps/ieee754/flt-32/s_sincosf_data.h
+++ b/sysdeps/ieee754/flt-32/s_sincosf_data.h
@@ -24,15 +24,18 @@ 
 extern const uint64_t __sinf_ipi[] attribute_hidden;
 #define IPI __sinf_ipi
 
-extern const double __sinf_b[] attribute_hidden;
-#define B __sinf_b
-extern const double __sinf_a[] attribute_hidden;
-#define A __sinf_a
-extern const double __sinf_tb[] attribute_hidden;
-#define TB __sinf_tb
-
-extern const double __cosf_tb[] attribute_hidden;
-#define TB_COSF __cosf_tb
+typedef struct
+{
+  double b[4];
+  double a[4];
+  double tb[32];
+  double tb_cosf[32];
+} sincosf_tables_t;
+extern const sincosf_tables_t __sincosf_tables attribute_hidden;
+#define B __sincosf_tables.b
+#define A __sincosf_tables.a
+#define TB __sincosf_tables.tb
+#define TB_COSF __sincosf_tables.tb_cosf
 
 typedef struct
 {