From edf620281501fbf314855aa9b321688f280e9220 Mon Sep 17 00:00:00 2001 From: Dylan Fleming Date: Mon, 19 May 2025 12:44:20 +0000 Subject: [PATCH] AArch64: Implement AdvSIMD and SVE atanpi/f Implement double and single precision variants of the C23 routine atanpi for both AdvSIMD and SVE. Reviewed-by: Wilco Dijkstra --- bits/libm-simd-decl-stubs.h | 11 ++ math/bits/mathcalls.h | 1 + sysdeps/aarch64/fpu/Makefile | 1 + sysdeps/aarch64/fpu/Versions | 5 + sysdeps/aarch64/fpu/advsimd_f32_protos.h | 1 + sysdeps/aarch64/fpu/atanpi_advsimd.c | 117 ++++++++++++++++ sysdeps/aarch64/fpu/atanpi_sve.c | 127 ++++++++++++++++++ sysdeps/aarch64/fpu/atanpif_advsimd.c | 92 +++++++++++++ sysdeps/aarch64/fpu/atanpif_sve.c | 89 ++++++++++++ sysdeps/aarch64/fpu/bits/math-vector.h | 8 ++ .../fpu/test-double-advsimd-wrappers.c | 1 + .../aarch64/fpu/test-double-sve-wrappers.c | 1 + .../aarch64/fpu/test-float-advsimd-wrappers.c | 1 + sysdeps/aarch64/fpu/test-float-sve-wrappers.c | 1 + .../unix/sysv/linux/aarch64/libmvec.abilist | 5 + 15 files changed, 461 insertions(+) create mode 100644 sysdeps/aarch64/fpu/atanpi_advsimd.c create mode 100644 sysdeps/aarch64/fpu/atanpi_sve.c create mode 100644 sysdeps/aarch64/fpu/atanpif_advsimd.c create mode 100644 sysdeps/aarch64/fpu/atanpif_sve.c diff --git a/bits/libm-simd-decl-stubs.h b/bits/libm-simd-decl-stubs.h index e05f596459..f1add553a0 100644 --- a/bits/libm-simd-decl-stubs.h +++ b/bits/libm-simd-decl-stubs.h @@ -395,4 +395,15 @@ #define __DECL_SIMD_asinpif32x #define __DECL_SIMD_asinpif64x #define __DECL_SIMD_asinpif128x + +#define __DECL_SIMD_atanpi +#define __DECL_SIMD_atanpif +#define __DECL_SIMD_atanpil +#define __DECL_SIMD_atanpif16 +#define __DECL_SIMD_atanpif32 +#define __DECL_SIMD_atanpif64 +#define __DECL_SIMD_atanpif128 +#define __DECL_SIMD_atanpif32x +#define __DECL_SIMD_atanpif64x +#define __DECL_SIMD_atanpif128x #endif diff --git a/math/bits/mathcalls.h b/math/bits/mathcalls.h index 573cec9d49..1a06a8cad6 100644 --- a/math/bits/mathcalls.h +++ b/math/bits/mathcalls.h @@ -74,6 +74,7 @@ __MATHCALL (asinpi,, (_Mdouble_ __x)); __MATHCALL_VEC (asinpi,, (_Mdouble_ __x)); /* Arc tangent of X, divided by pi. */ __MATHCALL (atanpi,, (_Mdouble_ __x)); +__MATHCALL_VEC (atanpi,, (_Mdouble_ __x)); /* Arc tangent of Y/X, divided by pi. */ __MATHCALL (atan2pi,, (_Mdouble_ __y, _Mdouble_ __x)); diff --git a/sysdeps/aarch64/fpu/Makefile b/sysdeps/aarch64/fpu/Makefile index e9d5c1ae42..d2a52ae34f 100644 --- a/sysdeps/aarch64/fpu/Makefile +++ b/sysdeps/aarch64/fpu/Makefile @@ -6,6 +6,7 @@ libmvec-supported-funcs = acos \ asinpi \ atan \ atanh \ + atanpi \ atan2 \ cbrt \ cos \ diff --git a/sysdeps/aarch64/fpu/Versions b/sysdeps/aarch64/fpu/Versions index b9a09f8256..2d88a0a21b 100644 --- a/sysdeps/aarch64/fpu/Versions +++ b/sysdeps/aarch64/fpu/Versions @@ -168,5 +168,10 @@ libmvec { _ZGVnN4v_asinpif; _ZGVsMxv_asinpi; _ZGVsMxv_asinpif; + _ZGVnN2v_atanpi; + _ZGVnN2v_atanpif; + _ZGVnN4v_atanpif; + _ZGVsMxv_atanpi; + _ZGVsMxv_atanpif; } } diff --git a/sysdeps/aarch64/fpu/advsimd_f32_protos.h b/sysdeps/aarch64/fpu/advsimd_f32_protos.h index 215a5a19f4..26c250b2be 100644 --- a/sysdeps/aarch64/fpu/advsimd_f32_protos.h +++ b/sysdeps/aarch64/fpu/advsimd_f32_protos.h @@ -25,6 +25,7 @@ libmvec_hidden_proto (V_NAME_F1(asinh)); libmvec_hidden_proto (V_NAME_F1(asinpi)); libmvec_hidden_proto (V_NAME_F1(atan)); libmvec_hidden_proto (V_NAME_F1(atanh)); +libmvec_hidden_proto (V_NAME_F1(atanpi)); libmvec_hidden_proto (V_NAME_F1(cbrt)); libmvec_hidden_proto (V_NAME_F1(cos)); libmvec_hidden_proto (V_NAME_F1(cosh)); diff --git a/sysdeps/aarch64/fpu/atanpi_advsimd.c b/sysdeps/aarch64/fpu/atanpi_advsimd.c new file mode 100644 index 0000000000..9101419a5f --- /dev/null +++ b/sysdeps/aarch64/fpu/atanpi_advsimd.c @@ -0,0 +1,117 @@ +/* Double-Precision vector (Advanced SIMD) inverse tanpi function + + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include "v_math.h" + +static const struct data +{ + double c2, c4, c6, c8, c10, c12, c14, c16, c18, c20; + float64x2_t c0, c1, c3, c5, c7, c9, c11, c13, c15, c17, c19; +} data = { + /* Coefficients of polynomial P such that atanpi(x)~x*P(x^2) on + [2^-1022, 1.0]. */ + .c0 = V2 (0x1.45f306dc9c883p-2), .c1 = V2 (-0x1.b2995e7b7ba4ap-4), + .c2 = 0x1.04c26be3d2c1p-4, .c3 = V2 (-0x1.7483759c17ea1p-5), + .c4 = 0x1.21bb95c315d57p-5, .c5 = V2 (-0x1.da1bdc3d453f3p-6), + .c6 = 0x1.912d20459b4bfp-6, .c7 = V2 (-0x1.5bbd4545cad1fp-6), + .c8 = 0x1.331b83bec30a1p-6, .c9 = V2 (-0x1.13d6457f44de3p-6), + .c10 = 0x1.f8e802974db94p-7, .c11 = V2 (-0x1.d7e173ab04a1ap-7), + .c12 = 0x1.bdfa47d6a4f28p-7, .c13 = V2 (-0x1.9ba78f3232ceep-7), + .c14 = 0x1.5e6044590ab4fp-7, .c15 = V2 (-0x1.01ccfdeb9f77fp-7), + .c16 = 0x1.345cf0d4eb1c1p-8, .c17 = V2 (-0x1.19e5f00f67e3ap-9), + .c18 = 0x1.6d3035ac7625bp-11, .c19 = V2 (-0x1.286bb9ae4ed79p-13), + .c20 = 0x1.c37ec36da0e1ap-17, +}; + +#define SignMask v_u64 (0x8000000000000000) + +/* Fast implementation of vector atanpi. + atanpi(x) ~ shift + z * P(z^2) with reduction to [0,1] using + z=1/x and shift = +-1/2. Maximum observed error is 2.76 ulps: + _ZGVnN2v_atanpi(0x1.fa2d6912cd64fp-1) got 0x1.fc45a51bd497fp-3 + want 0x1.fc45a51bd497cp-3. */ +float64x2_t VPCS_ATTR V_NAME_D1 (atanpi) (float64x2_t x) +{ + const struct data *d = ptr_barrier (&data); + + uint64x2_t ix = vreinterpretq_u64_f64 (x); + uint64x2_t sign = vandq_u64 (ix, SignMask); + + /* Argument Reduction: + y := arctanpi(x) for |x| < 1 + y := arctanpi(-1/x) + 1/2 for x > 1 + y := arctanpi(-1/x) - 1/2 for x < -1 + Hence, use z=-1/a if |x|>=|-1|, otherwise z=a. */ + uint64x2_t red = vcagtq_f64 (x, v_f64 (-1.0)); + float64x2_t z = vbslq_f64 (red, vdivq_f64 (v_f64 (-1.0), x), x); + + /* Shift is calculated as +1/2 or 0, depending on the argument case. */ + float64x2_t shift = vreinterpretq_f64_u64 ( + vandq_u64 (red, vreinterpretq_u64_f64 (v_f64 (0.5)))); + + /* Reinsert sign bit from argument into the shift value. */ + shift = vreinterpretq_f64_u64 ( + veorq_u64 (vreinterpretq_u64_f64 (shift), sign)); + + /* Calculate polynomial approximation P(z^2) with deg(P)=19. */ + float64x2_t z2 = vmulq_f64 (z, z); + float64x2_t z4 = vmulq_f64 (z2, z2); + float64x2_t z8 = vmulq_f64 (z4, z4); + float64x2_t z16 = vmulq_f64 (z8, z8); + + float64x2_t c24 = vld1q_f64 (&d->c2); + float64x2_t c68 = vld1q_f64 (&d->c6); + + /* Order-7 Estrin. */ + float64x2_t p12 = vfmaq_laneq_f64 (d->c1, z2, c24, 0); + float64x2_t p34 = vfmaq_laneq_f64 (d->c3, z2, c24, 1); + float64x2_t p56 = vfmaq_laneq_f64 (d->c5, z2, c68, 0); + float64x2_t p78 = vfmaq_laneq_f64 (d->c7, z2, c68, 1); + + float64x2_t p14 = vfmaq_f64 (p12, z4, p34); + float64x2_t p58 = vfmaq_f64 (p56, z4, p78); + float64x2_t p18 = vfmaq_f64 (p14, z8, p58); + + /* Order-11 Estrin. */ + float64x2_t c1012 = vld1q_f64 (&d->c10); + float64x2_t c1416 = vld1q_f64 (&d->c14); + float64x2_t c1820 = vld1q_f64 (&d->c18); + + float64x2_t p910 = vfmaq_laneq_f64 (d->c9, z2, c1012, 0); + float64x2_t p1112 = vfmaq_laneq_f64 (d->c11, z2, c1012, 1); + float64x2_t p912 = vfmaq_f64 (p910, z4, p1112); + + float64x2_t p1314 = vfmaq_laneq_f64 (d->c13, z2, c1416, 0); + float64x2_t p1516 = vfmaq_laneq_f64 (d->c15, z2, c1416, 1); + float64x2_t p1316 = vfmaq_f64 (p1314, z4, p1516); + + float64x2_t p1718 = vfmaq_laneq_f64 (d->c17, z2, c1820, 0); + float64x2_t p1920 = vfmaq_laneq_f64 (d->c19, z2, c1820, 1); + float64x2_t p1720 = vfmaq_f64 (p1718, z4, p1920); + + float64x2_t p916 = vfmaq_f64 (p912, z8, p1316); + float64x2_t p920 = vfmaq_f64 (p916, z16, p1720); + + float64x2_t y = vfmaq_f64 (p18, p920, z16); + + y = vfmaq_f64 (d->c0, z2, y); + + /* y = shift + z * p(z^2). */ + return vfmaq_f64 (shift, z, y); +} diff --git a/sysdeps/aarch64/fpu/atanpi_sve.c b/sysdeps/aarch64/fpu/atanpi_sve.c new file mode 100644 index 0000000000..3f8f277981 --- /dev/null +++ b/sysdeps/aarch64/fpu/atanpi_sve.c @@ -0,0 +1,127 @@ +/* Double-Precision vector (SVE) inverse tanpi function + + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include "sv_math.h" + +static const struct data +{ + float64_t c2, c4, c6, c8, c10, c12, c14, c16, c18, c20; + float64_t c0, c1, c3, c5, c7, c9, c11, c13, c15, c17, c19; + float64_t shift_val, neg_one; +} data = { + /* Coefficients of polnomial P such that atan(x)~x+x*P(x^2) on + [2^-1022, 1.0]. */ + .c0 = 0x1.45f306dc9c883p-2, + .c1 = -0x1.b2995e7b7ba4ap-4, + .c2 = 0x1.04c26be3d2c1p-4, + .c3 = -0x1.7483759c17ea1p-5, + .c4 = 0x1.21bb95c315d57p-5, + .c5 = -0x1.da1bdc3d453f3p-6, + .c6 = 0x1.912d20459b4bfp-6, + .c7 = -0x1.5bbd4545cad1fp-6, + .c8 = 0x1.331b83bec30a1p-6, + .c9 = -0x1.13d6457f44de3p-6, + .c10 = 0x1.f8e802974db94p-7, + .c11 = -0x1.d7e173ab04a1ap-7, + .c12 = 0x1.bdfa47d6a4f28p-7, + .c13 = -0x1.9ba78f3232ceep-7, + .c14 = 0x1.5e6044590ab4fp-7, + .c15 = -0x1.01ccfdeb9f77fp-7, + .c16 = 0x1.345cf0d4eb1c1p-8, + .c17 = -0x1.19e5f00f67e3ap-9, + .c18 = 0x1.6d3035ac7625bp-11, + .c19 = -0x1.286bb9ae4ed79p-13, + .c20 = 0x1.c37ec36da0e1ap-17, + .shift_val = 1.5, + .neg_one = -1, +}; + +/* Fast implementation of SVE atan. + Based on atan(x) ~ shift + z + z^3 * P(z^2) with reduction to 0,1 using + z=1/x and shift = pi/2. Largest errors are close to 1. The maximum observed + error is 2.80 ulps: + _ZGVsMxv_atanpi(0x1.f19587d63c76fp-1) got 0x1.f6b1304817d02p-3 + want 0x1.f6b1304817d05p-3. */ +svfloat64_t SV_NAME_D1 (atanpi) (svfloat64_t x, const svbool_t pg) +{ + const struct data *d = ptr_barrier (&data); + + svbool_t ptrue = svptrue_b64 (); + svuint64_t ix = svreinterpret_u64 (x); + svuint64_t sign = svand_x (pg, ix, 0x8000000000000000); + + /* Argument reduction: + y := arctan(x) for x < 1 + y := pi/2 + arctan(-1/x) for x > 1 + Hence, use z=-1/a if x>=1, otherwise z=a. */ + svbool_t red = svacgt (pg, x, d->neg_one); + svfloat64_t z = svsel (red, svdiv_x (pg, sv_f64 (d->neg_one), x), x); + + /* Reuse of -1.0f to reduce constant loads, + We need a shift value of 1/2, which is created via -1 + (1 + 1/2). */ + svfloat64_t shift + = svadd_z (red, sv_f64 (d->neg_one), sv_f64 (d->shift_val)); + + /* Reinserts the sign bit of the argument to handle the case of x < -1. */ + shift = svreinterpret_f64 (sveor_x (pg, svreinterpret_u64 (shift), sign)); + + /* Use split Estrin scheme for P(z^2) with deg(P)=19. */ + svfloat64_t z2 = svmul_x (ptrue, z, z); + svfloat64_t z4 = svmul_x (ptrue, z2, z2); + svfloat64_t z8 = svmul_x (ptrue, z4, z4); + svfloat64_t z16 = svmul_x (ptrue, z8, z8); + + /* Order-7 Estrin. */ + svfloat64_t c24 = svld1rq (ptrue, &d->c2); + svfloat64_t c68 = svld1rq (ptrue, &d->c6); + + svfloat64_t p12 = svmla_lane (sv_f64 (d->c1), z2, c24, 0); + svfloat64_t p34 = svmla_lane (sv_f64 (d->c3), z2, c24, 1); + svfloat64_t p56 = svmla_lane (sv_f64 (d->c5), z2, c68, 0); + svfloat64_t p78 = svmla_lane (sv_f64 (d->c7), z2, c68, 1); + + svfloat64_t p14 = svmla_x (pg, p12, z4, p34); + svfloat64_t p58 = svmla_x (pg, p56, z4, p78); + svfloat64_t p18 = svmla_x (pg, p14, z8, p58); + + /* Order-11 Estrin. */ + svfloat64_t c1012 = svld1rq (ptrue, &d->c10); + svfloat64_t c1416 = svld1rq (ptrue, &d->c14); + svfloat64_t c1820 = svld1rq (ptrue, &d->c18); + + svfloat64_t p910 = svmla_lane (sv_f64 (d->c9), z2, c1012, 0); + svfloat64_t p1112 = svmla_lane (sv_f64 (d->c11), z2, c1012, 1); + svfloat64_t p912 = svmla_x (pg, p910, z4, p1112); + + svfloat64_t p1314 = svmla_lane (sv_f64 (d->c13), z2, c1416, 0); + svfloat64_t p1516 = svmla_lane (sv_f64 (d->c15), z2, c1416, 1); + svfloat64_t p1316 = svmla_x (pg, p1314, z4, p1516); + + svfloat64_t p1718 = svmla_lane (sv_f64 (d->c17), z2, c1820, 0); + svfloat64_t p1920 = svmla_lane (sv_f64 (d->c19), z2, c1820, 1); + svfloat64_t p1720 = svmla_x (pg, p1718, z4, p1920); + + svfloat64_t p916 = svmla_x (pg, p912, z8, p1316); + svfloat64_t p920 = svmla_x (pg, p916, z16, p1720); + + svfloat64_t y = svmla_x (pg, p18, z16, p920); + + y = svmla_x (pg, sv_f64 (d->c0), z2, y); + return svmla_x (pg, shift, z, y); +} diff --git a/sysdeps/aarch64/fpu/atanpif_advsimd.c b/sysdeps/aarch64/fpu/atanpif_advsimd.c new file mode 100644 index 0000000000..92951564f4 --- /dev/null +++ b/sysdeps/aarch64/fpu/atanpif_advsimd.c @@ -0,0 +1,92 @@ +/* Single-Precision vector (Advanced SIMD) inverse tanpi function + + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include "v_math.h" + +static const struct data +{ + uint32x4_t half; + float32x4_t neg_one; + float32x4_t c0, c1, c3, c5, c7; + float c2, c4, c6, c8; +} data = { + /* Polynomial coefficients generated using Remez algorithm, + see atanpi.sollya for details. */ + .c0 = V4 (0x1.45f306p-2), .c1 = V4 (-0x1.b2975ep-4), + .c2 = 0x1.0490e4p-4, .c3 = V4 (-0x1.70c272p-5), + .c4 = 0x1.0eef52p-5, .c5 = V4 (-0x1.6abbbap-6), + .c6 = 0x1.78157p-7, .c7 = V4 (-0x1.f0b406p-9), + .c8 = 0x1.2ae7fep-11, .half = V4 (0x3f000000), + .neg_one = V4 (-1.0f), +}; + +#define SignMask v_u32 (0x80000000) + +/* Fast implementation of vector atanpif based on + atanpi(x) ~ shift + z * P(z^2) with reduction to [0,1] + using z=-1/x and shift = +-1/2. + Maximum observed error is 2.59ulps: + _ZGVnN4v_atanpif (0x1.f2a89cp-1) got 0x1.f76524p-3 + want 0x1.f7651ep-3. */ +float32x4_t VPCS_ATTR NOINLINE V_NAME_F1 (atanpi) (float32x4_t x) +{ + const struct data *d = ptr_barrier (&data); + + uint32x4_t ix = vreinterpretq_u32_f32 (x); + uint32x4_t sign = vandq_u32 (ix, SignMask); + + /* Argument Reduction: + y := arctanpi(x) for |x| < 1 + y := arctanpi(-1/x) + 1/2 for x > 1 + y := arctanpi(-1/x) - 1/2 for x < -1 + Hence, use z=-1/a if |x|>=|-1|, otherwise z=a. */ + uint32x4_t red = vcagtq_f32 (x, d->neg_one); + + float32x4_t z = vbslq_f32 (red, vdivq_f32 (d->neg_one, x), x); + + /* Shift is calculated as +1/2 or 0, depending on the argument case. */ + float32x4_t shift = vreinterpretq_f32_u32 (vandq_u32 (red, d->half)); + + /* Reinsert sign bit from argument into the shift value. */ + shift = vreinterpretq_f32_u32 ( + veorq_u32 (vreinterpretq_u32_f32 (shift), sign)); + + /* Uses an Estrin scheme for polynomial approximation. */ + float32x4_t z2 = vmulq_f32 (z, z); + float32x4_t z4 = vmulq_f32 (z2, z2); + float32x4_t z8 = vmulq_f32 (z4, z4); + + float32x4_t even_coeffs = vld1q_f32 (&d->c2); + + float32x4_t p12 = vfmaq_laneq_f32 (d->c1, z2, even_coeffs, 0); + float32x4_t p34 = vfmaq_laneq_f32 (d->c3, z2, even_coeffs, 1); + float32x4_t p56 = vfmaq_laneq_f32 (d->c5, z2, even_coeffs, 2); + float32x4_t p78 = vfmaq_laneq_f32 (d->c7, z2, even_coeffs, 3); + + float32x4_t p14 = vfmaq_f32 (p12, z4, p34); + float32x4_t p58 = vfmaq_f32 (p56, z4, p78); + + float32x4_t y = vfmaq_f32 (p14, z8, p58); + y = vfmaq_f32 (d->c0, z2, y); + + /* y = shift + z * P(z^2). */ + return vfmaq_f32 (shift, z, y); +} +libmvec_hidden_def (V_NAME_F1 (atanpi)) +HALF_WIDTH_ALIAS_F1 (atanpi) diff --git a/sysdeps/aarch64/fpu/atanpif_sve.c b/sysdeps/aarch64/fpu/atanpif_sve.c new file mode 100644 index 0000000000..2abd788c41 --- /dev/null +++ b/sysdeps/aarch64/fpu/atanpif_sve.c @@ -0,0 +1,89 @@ +/* Single-Precision vector (SVE) inverse tanpi function + + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include "sv_math.h" + +static const struct data +{ + float32_t c2, c4, c6, c8; + float32_t c0, c1, c3, c5, c7; + float32_t shift_val, neg_one; +} data = { + /* Coefficients of polynomial P such that atan(x)~x+x*P(x^2) on + [2**-128, 1.0]. */ + .c0 = 0x1.45f306p-2, .c1 = -0x1.b2975ep-4, .c2 = 0x1.0490e4p-4, + .c3 = -0x1.70c272p-5, .c4 = 0x1.0eef52p-5, .c5 = -0x1.6abbbap-6, + .c6 = 0x1.78157p-7, .c7 = -0x1.f0b406p-9, .c8 = 0x1.2ae7fep-11, + .shift_val = 1.5f, .neg_one = -1.0f, +}; + +#define SignMask (0x80000000) + +/* Fast implementation of SVE atanpif based on + atan(x) ~ shift + z * P(z^2) with reduction to [0,1] using + z=-1/x and shift = 1/2. + Largest observed error is 2.59 ULP, close to +/-1.0: + _ZGVsMxv_atanpif(0x1.f2a89cp-1) got 0x1.f76524p-3 + want 0x1.f7651ep-3. */ +svfloat32_t SV_NAME_F1 (atanpi) (svfloat32_t x, const svbool_t pg) +{ + const struct data *d = ptr_barrier (&data); + svbool_t ptrue = svptrue_b32 (); + + /* No need to trigger special case. Small cases, infs and nans + are supported by our approximation technique. */ + svuint32_t ix = svreinterpret_u32 (x); + svuint32_t sign = svand_x (pg, ix, SignMask); + + /* Argument reduction: + y := arctan(x) for x < 1 + y := arctan(-1/x) + 1/2 for x > +1 + y := arctan(-1/x) - 1/2 for x < -1 + Hence, use z=-1/a if |x|>=|-1|, otherwise z=a. */ + svbool_t red = svacgt (pg, x, d->neg_one); + svfloat32_t z = svsel (red, svdiv_x (ptrue, sv_f32 (d->neg_one), x), x); + + /* Reuse of -1.0f to reduce constant loads, + We need a shift value of 1/2, which is created via -1 + (1 + 1/2). */ + svfloat32_t shift + = svadd_z (red, sv_f32 (d->neg_one), sv_f32 (d->shift_val)); + + /* Reinserts the sign bit of the argument to handle the case of x < -1. */ + shift = svreinterpret_f32 (sveor_x (pg, svreinterpret_u32 (shift), sign)); + + svfloat32_t z2 = svmul_x (ptrue, z, z); + svfloat32_t z4 = svmul_x (ptrue, z2, z2); + svfloat32_t z8 = svmul_x (ptrue, z4, z4); + + svfloat32_t even_coeffs = svld1rq (ptrue, &d->c2); + + svfloat32_t p12 = svmla_lane (sv_f32 (d->c1), z2, even_coeffs, 0); + svfloat32_t p34 = svmla_lane (sv_f32 (d->c3), z2, even_coeffs, 1); + svfloat32_t p56 = svmla_lane (sv_f32 (d->c5), z2, even_coeffs, 2); + svfloat32_t p78 = svmla_lane (sv_f32 (d->c7), z2, even_coeffs, 3); + + svfloat32_t p14 = svmad_x (pg, z4, p34, p12); + svfloat32_t p58 = svmad_x (pg, z4, p78, p56); + + svfloat32_t p18 = svmad_x (pg, z8, p58, p14); + svfloat32_t y = svmad_x (pg, z2, p18, d->c0); + + /* shift + z * P(z^2). */ + return svmad_x (pg, y, z, shift); +} diff --git a/sysdeps/aarch64/fpu/bits/math-vector.h b/sysdeps/aarch64/fpu/bits/math-vector.h index a67d565d8a..7667e0a633 100644 --- a/sysdeps/aarch64/fpu/bits/math-vector.h +++ b/sysdeps/aarch64/fpu/bits/math-vector.h @@ -61,6 +61,10 @@ # define __DECL_SIMD_atanh __DECL_SIMD_aarch64 # undef __DECL_SIMD_atanhf # define __DECL_SIMD_atanhf __DECL_SIMD_aarch64 +# undef __DECL_SIMD_atanpi +# define __DECL_SIMD_atanpi __DECL_SIMD_aarch64 +# undef __DECL_SIMD_atanpif +# define __DECL_SIMD_atanpif __DECL_SIMD_aarch64 # undef __DECL_SIMD_atan2 # define __DECL_SIMD_atan2 __DECL_SIMD_aarch64 # undef __DECL_SIMD_atan2f @@ -192,6 +196,7 @@ __vpcs __f32x4_t _ZGVnN4v_asinhf (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_asinpif (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_atanf (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_atanhf (__f32x4_t); +__vpcs __f32x4_t _ZGVnN4v_atanpif (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_cbrtf (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_cosf (__f32x4_t); __vpcs __f32x4_t _ZGVnN4v_coshf (__f32x4_t); @@ -225,6 +230,7 @@ __vpcs __f64x2_t _ZGVnN2v_asinh (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_asinpi (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_atan (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_atanh (__f64x2_t); +__vpcs __f64x2_t _ZGVnN2v_atanpi (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_cbrt (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_cos (__f64x2_t); __vpcs __f64x2_t _ZGVnN2v_cosh (__f64x2_t); @@ -263,6 +269,7 @@ __sv_f32_t _ZGVsMxv_asinhf (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_asinpif (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_atanf (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_atanhf (__sv_f32_t, __sv_bool_t); +__sv_f32_t _ZGVsMxv_atanpif (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_cbrtf (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_cosf (__sv_f32_t, __sv_bool_t); __sv_f32_t _ZGVsMxv_coshf (__sv_f32_t, __sv_bool_t); @@ -296,6 +303,7 @@ __sv_f64_t _ZGVsMxv_asinh (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_asinpi (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_atan (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_atanh (__sv_f64_t, __sv_bool_t); +__sv_f64_t _ZGVsMxv_atanpi (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_cbrt (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_cos (__sv_f64_t, __sv_bool_t); __sv_f64_t _ZGVsMxv_cosh (__sv_f64_t, __sv_bool_t); diff --git a/sysdeps/aarch64/fpu/test-double-advsimd-wrappers.c b/sysdeps/aarch64/fpu/test-double-advsimd-wrappers.c index ab86b7c949..b7a0538b44 100644 --- a/sysdeps/aarch64/fpu/test-double-advsimd-wrappers.c +++ b/sysdeps/aarch64/fpu/test-double-advsimd-wrappers.c @@ -31,6 +31,7 @@ VPCS_VECTOR_WRAPPER (asinh_advsimd, _ZGVnN2v_asinh) VPCS_VECTOR_WRAPPER (asinpi_advsimd, _ZGVnN2v_asinpi) VPCS_VECTOR_WRAPPER (atan_advsimd, _ZGVnN2v_atan) VPCS_VECTOR_WRAPPER (atanh_advsimd, _ZGVnN2v_atanh) +VPCS_VECTOR_WRAPPER (atanpi_advsimd, _ZGVnN2v_atanpi) VPCS_VECTOR_WRAPPER_ff (atan2_advsimd, _ZGVnN2vv_atan2) VPCS_VECTOR_WRAPPER (cbrt_advsimd, _ZGVnN2v_cbrt) VPCS_VECTOR_WRAPPER (cos_advsimd, _ZGVnN2v_cos) diff --git a/sysdeps/aarch64/fpu/test-double-sve-wrappers.c b/sysdeps/aarch64/fpu/test-double-sve-wrappers.c index 8cbb1d9aaa..aec60db616 100644 --- a/sysdeps/aarch64/fpu/test-double-sve-wrappers.c +++ b/sysdeps/aarch64/fpu/test-double-sve-wrappers.c @@ -50,6 +50,7 @@ SVE_VECTOR_WRAPPER (asinh_sve, _ZGVsMxv_asinh) SVE_VECTOR_WRAPPER (asinpi_sve, _ZGVsMxv_asinpi) SVE_VECTOR_WRAPPER (atan_sve, _ZGVsMxv_atan) SVE_VECTOR_WRAPPER (atanh_sve, _ZGVsMxv_atanh) +SVE_VECTOR_WRAPPER (atanpi_sve, _ZGVsMxv_atanpi) SVE_VECTOR_WRAPPER_ff (atan2_sve, _ZGVsMxvv_atan2) SVE_VECTOR_WRAPPER (cbrt_sve, _ZGVsMxv_cbrt) SVE_VECTOR_WRAPPER (cos_sve, _ZGVsMxv_cos) diff --git a/sysdeps/aarch64/fpu/test-float-advsimd-wrappers.c b/sysdeps/aarch64/fpu/test-float-advsimd-wrappers.c index a77aadbec1..60cffd5035 100644 --- a/sysdeps/aarch64/fpu/test-float-advsimd-wrappers.c +++ b/sysdeps/aarch64/fpu/test-float-advsimd-wrappers.c @@ -31,6 +31,7 @@ VPCS_VECTOR_WRAPPER (asinhf_advsimd, _ZGVnN4v_asinhf) VPCS_VECTOR_WRAPPER (asinpif_advsimd, _ZGVnN4v_asinpif) VPCS_VECTOR_WRAPPER (atanf_advsimd, _ZGVnN4v_atanf) VPCS_VECTOR_WRAPPER (atanhf_advsimd, _ZGVnN4v_atanhf) +VPCS_VECTOR_WRAPPER (atanpif_advsimd, _ZGVnN4v_atanpif) VPCS_VECTOR_WRAPPER_ff (atan2f_advsimd, _ZGVnN4vv_atan2f) VPCS_VECTOR_WRAPPER (cbrtf_advsimd, _ZGVnN4v_cbrtf) VPCS_VECTOR_WRAPPER (cosf_advsimd, _ZGVnN4v_cosf) diff --git a/sysdeps/aarch64/fpu/test-float-sve-wrappers.c b/sysdeps/aarch64/fpu/test-float-sve-wrappers.c index 117cd5c41d..ab50ae9ad0 100644 --- a/sysdeps/aarch64/fpu/test-float-sve-wrappers.c +++ b/sysdeps/aarch64/fpu/test-float-sve-wrappers.c @@ -50,6 +50,7 @@ SVE_VECTOR_WRAPPER (asinhf_sve, _ZGVsMxv_asinhf) SVE_VECTOR_WRAPPER (asinpif_sve, _ZGVsMxv_asinpif) SVE_VECTOR_WRAPPER (atanf_sve, _ZGVsMxv_atanf) SVE_VECTOR_WRAPPER (atanhf_sve, _ZGVsMxv_atanhf) +SVE_VECTOR_WRAPPER (atanpif_sve, _ZGVsMxv_atanpif) SVE_VECTOR_WRAPPER_ff (atan2f_sve, _ZGVsMxvv_atan2f) SVE_VECTOR_WRAPPER (cbrtf_sve, _ZGVsMxv_cbrtf) SVE_VECTOR_WRAPPER (cosf_sve, _ZGVsMxv_cosf) diff --git a/sysdeps/unix/sysv/linux/aarch64/libmvec.abilist b/sysdeps/unix/sysv/linux/aarch64/libmvec.abilist index 4ca8e77d49..2714648371 100644 --- a/sysdeps/unix/sysv/linux/aarch64/libmvec.abilist +++ b/sysdeps/unix/sysv/linux/aarch64/libmvec.abilist @@ -152,9 +152,14 @@ GLIBC_2.42 _ZGVnN2v_acospi F GLIBC_2.42 _ZGVnN2v_acospif F GLIBC_2.42 _ZGVnN2v_asinpi F GLIBC_2.42 _ZGVnN2v_asinpif F +GLIBC_2.42 _ZGVnN2v_atanpi F +GLIBC_2.42 _ZGVnN2v_atanpif F GLIBC_2.42 _ZGVnN4v_acospif F GLIBC_2.42 _ZGVnN4v_asinpif F +GLIBC_2.42 _ZGVnN4v_atanpif F GLIBC_2.42 _ZGVsMxv_acospi F GLIBC_2.42 _ZGVsMxv_acospif F GLIBC_2.42 _ZGVsMxv_asinpi F GLIBC_2.42 _ZGVsMxv_asinpif F +GLIBC_2.42 _ZGVsMxv_atanpi F +GLIBC_2.42 _ZGVsMxv_atanpif F