git: cf46693f1351 - stable/15 - lib/msun amd64: use AVX FMA instructions for fma(3) and fmaf(3) when available

The branch stable/15 has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=cf46693f1351fc42e84cacbf81b7b625a5bdab72

commit cf46693f1351fc42e84cacbf81b7b625a5bdab72
Author:     Konstantin Belousov <kib@FreeBSD.org>
AuthorDate: 2026-09-06 05:01:35 +0000
Commit:     Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2026-09-14 00:44:42 +0000

    lib/msun amd64: use AVX FMA instructions for fma(3) and fmaf(3) when available
    
    (cherry picked from commit 18a5549bcb81ec084f284a98046631db277ad47c)
---
 lib/msun/amd64/Makefile.inc |  9 ++++++++-
 lib/msun/amd64/s_fma_avx.S  | 24 ++++++++++++++++++++++++
 lib/msun/amd64/s_fmaf_avx.S | 24 ++++++++++++++++++++++++
 lib/msun/src/s_fma.c        | 11 ++++++++++-
 lib/msun/src/s_fmaf.c       | 11 ++++++++++-
 5 files changed, 76 insertions(+), 3 deletions(-)

diff --git a/lib/msun/amd64/Makefile.inc b/lib/msun/amd64/Makefile.inc
index f1e6aef552ca..0efb16314af3 100644
--- a/lib/msun/amd64/Makefile.inc
+++ b/lib/msun/amd64/Makefile.inc
@@ -1,8 +1,15 @@
-ARCH_SRCS = e_fmod.S e_fmodf.S e_fmodl.S \
+.PATH:	${LIBC_SRCTOP}/amd64/string
+
+ARCH_SRCS = amd64_archlevel.c \
+	    e_fmod.S e_fmodf.S e_fmodl.S \
 	    e_remainder.S e_remainderf.S e_remainderl.S \
 	    e_sqrt.S e_sqrtf.S e_sqrtl.S \
+	    s_fma_avx.S s_fmaf_avx.S \
 	    s_llrint.S s_llrintf.S s_llrintl.S \
 	    s_logbl.S s_lrint.S s_lrintf.S s_lrintl.S \
 	    s_remquo.S s_remquof.S s_remquol.S \
 	    s_rintl.S s_scalbn.S s_scalbnf.S s_scalbnl.S
 LDBL_PREC = 64
+
+CFLAGS+=-D_RENAME_FMA
+CFLAGS+=-D_RENAME_FMAF
diff --git a/lib/msun/amd64/s_fma_avx.S b/lib/msun/amd64/s_fma_avx.S
new file mode 100644
index 000000000000..6585d0ffb074
--- /dev/null
+++ b/lib/msun/amd64/s_fma_avx.S
@@ -0,0 +1,24 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright 2026 The FreeBSD Foundation
+ *
+ * This software were developed by
+ * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
+ * the FreeBSD Foundation.
+ */
+
+#include "amd64_archlevel.h"
+
+	.text
+ARCHENTRY(fma, avx)
+	vfmadd132sd	%xmm1, %xmm2, %xmm0
+	ret
+ARCHEND(fma, avx)
+
+ARCHFUNCS(fma)
+	ARCHFUNC(fma, sw)
+	NOARCHFUNC
+	NOARCHFUNC
+	ARCHFUNC(fma, avx)
+ENDARCHFUNCS(fma)
diff --git a/lib/msun/amd64/s_fmaf_avx.S b/lib/msun/amd64/s_fmaf_avx.S
new file mode 100644
index 000000000000..61645693b20d
--- /dev/null
+++ b/lib/msun/amd64/s_fmaf_avx.S
@@ -0,0 +1,24 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright 2026 The FreeBSD Foundation
+ *
+ * This software were developed by
+ * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from
+ * the FreeBSD Foundation.
+ */
+
+#include "amd64_archlevel.h"
+
+	.text
+ARCHENTRY(fmaf, avx):
+	vfmadd132ss	%xmm1, %xmm2, %xmm0
+	ret
+ARCHEND(fmaf, avx):
+
+ARCHFUNCS(fmaf)
+	ARCHFUNC(fmaf, sw)
+	NOARCHFUNC
+	NOARCHFUNC
+	ARCHFUNC(fmaf, avx)
+ENDARCHFUNCS(fmaf)
diff --git a/lib/msun/src/s_fma.c b/lib/msun/src/s_fma.c
index 23a84491dd2a..107d78543887 100644
--- a/lib/msun/src/s_fma.c
+++ b/lib/msun/src/s_fma.c
@@ -163,6 +163,10 @@ dd_mul(double a, double b)
 	return (ret);
 }
 
+#ifdef _RENAME_FMA
+double fma_sw(double, double, double);
+#endif
+
 /*
  * Fused multiply-add: Compute x * y + z with a single rounding error.
  *
@@ -181,7 +185,12 @@ dd_mul(double a, double b)
  * since this implementation will likely be several times slower.
  */
 double
-fma(double x, double y, double z)
+#ifdef _RENAME_FMA
+fma_sw
+#else
+fma
+#endif
+(double x, double y, double z)
 {
 	double xs, ys, zs, adj;
 	struct dd xy, r;
diff --git a/lib/msun/src/s_fmaf.c b/lib/msun/src/s_fmaf.c
index 5f3d5d1b1b99..666d2bb26d91 100644
--- a/lib/msun/src/s_fmaf.c
+++ b/lib/msun/src/s_fmaf.c
@@ -38,6 +38,10 @@ fmaf(float x, float y, float z)
 	return (__builtin_fmaf(x, y, z));
 }
 #else
+#ifdef _RENAME_FMAF
+float fmaf_sw(float, float, float);
+#endif
+
 /*
  * Fused multiply-add: Compute x * y + z with a single rounding error.
  *
@@ -46,7 +50,12 @@ fmaf(float x, float y, float z)
  * rounding occurs.
  */
 float
-fmaf(float x, float y, float z)
+#ifdef _RENAME_FMAF
+fmaf_sw
+#else
+fmaf
+#endif
+(float x, float y, float z)
 {
 	double xy, result;
 	uint32_t hr, lr;