From c0c39a765b0714aed36fced6fbba452a6619acb0 Mon Sep 17 00:00:00 2001
From: Jakub Jelinek <jakub@redhat.com>
Date: Wed, 13 May 2020 11:21:02 +0200
Subject: [PATCH] Fold single imm use of a FMA if it is a negation [PR95060]

match.pd already has simplifications for negation of a FMA (FMS, FNMA, FNMS)
call if it is single use, but when the widening_mul pass discovers FMAs,
nothing folds the statements anymore.

So, the following patch adjusts the widening_mul pass to handle that.

I had to adjust quite a lot of tests, because they have in them nested FMAs
(one FMA feeding another one) and the patch results in some (equivalent) changes
in the chosen instructions, previously the negation of one FMA's result
would result in the dependent FMA being adjusted for the negation, but now
instead the first FMA is adjusted.

2020-05-13  Jakub Jelinek  <jakub@redhat.com>

	PR tree-optimization/95060
	* tree-ssa-math-opts.c (convert_mult_to_fma_1): Fold a NEGATE_EXPR
	if it is the single use of the FMA internal builtin.

	* gcc.target/i386/avx512f-pr95060.c: New test.
	* gcc.target/i386/fma_double_1.c: Adjust expected insn counts.
	* gcc.target/i386/fma_double_2.c: Likewise.
	* gcc.target/i386/fma_double_3.c: Likewise.
	* gcc.target/i386/fma_double_4.c: Likewise.
	* gcc.target/i386/fma_double_5.c: Likewise.
	* gcc.target/i386/fma_double_6.c: Likewise.
	* gcc.target/i386/fma_float_1.c: Likewise.
	* gcc.target/i386/fma_float_2.c: Likewise.
	* gcc.target/i386/fma_float_3.c: Likewise.
	* gcc.target/i386/fma_float_4.c: Likewise.
	* gcc.target/i386/fma_float_5.c: Likewise.
	* gcc.target/i386/fma_float_6.c: Likewise.
	* gcc.target/i386/l_fma_double_1.c: Likewise.
	* gcc.target/i386/l_fma_double_2.c: Likewise.
	* gcc.target/i386/l_fma_double_3.c: Likewise.
	* gcc.target/i386/l_fma_double_4.c: Likewise.
	* gcc.target/i386/l_fma_double_5.c: Likewise.
	* gcc.target/i386/l_fma_double_6.c: Likewise.
	* gcc.target/i386/l_fma_float_1.c: Likewise.
	* gcc.target/i386/l_fma_float_2.c: Likewise.
	* gcc.target/i386/l_fma_float_3.c: Likewise.
	* gcc.target/i386/l_fma_float_4.c: Likewise.
	* gcc.target/i386/l_fma_float_5.c: Likewise.
	* gcc.target/i386/l_fma_float_6.c: Likewise.
---
 gcc/ChangeLog                                 |  6 ++++
 gcc/testsuite/ChangeLog                       | 29 +++++++++++++++++++
 .../gcc.target/i386/avx512f-pr95060.c         | 22 ++++++++++++++
 gcc/testsuite/gcc.target/i386/fma_double_1.c  |  6 ++--
 gcc/testsuite/gcc.target/i386/fma_double_2.c  |  8 ++---
 gcc/testsuite/gcc.target/i386/fma_double_3.c  |  8 ++---
 gcc/testsuite/gcc.target/i386/fma_double_4.c  |  8 ++---
 gcc/testsuite/gcc.target/i386/fma_double_5.c  |  8 ++---
 gcc/testsuite/gcc.target/i386/fma_double_6.c  |  8 ++---
 gcc/testsuite/gcc.target/i386/fma_float_1.c   |  6 ++--
 gcc/testsuite/gcc.target/i386/fma_float_2.c   |  8 ++---
 gcc/testsuite/gcc.target/i386/fma_float_3.c   |  8 ++---
 gcc/testsuite/gcc.target/i386/fma_float_4.c   |  8 ++---
 gcc/testsuite/gcc.target/i386/fma_float_5.c   |  8 ++---
 gcc/testsuite/gcc.target/i386/fma_float_6.c   |  8 ++---
 .../gcc.target/i386/l_fma_double_1.c          | 16 +++++-----
 .../gcc.target/i386/l_fma_double_2.c          | 16 +++++-----
 .../gcc.target/i386/l_fma_double_3.c          | 16 +++++-----
 .../gcc.target/i386/l_fma_double_4.c          | 16 +++++-----
 .../gcc.target/i386/l_fma_double_5.c          | 16 +++++-----
 .../gcc.target/i386/l_fma_double_6.c          | 16 +++++-----
 gcc/testsuite/gcc.target/i386/l_fma_float_1.c | 16 +++++-----
 gcc/testsuite/gcc.target/i386/l_fma_float_2.c | 16 +++++-----
 gcc/testsuite/gcc.target/i386/l_fma_float_3.c | 16 +++++-----
 gcc/testsuite/gcc.target/i386/l_fma_float_4.c | 16 +++++-----
 gcc/testsuite/gcc.target/i386/l_fma_float_5.c | 16 +++++-----
 gcc/testsuite/gcc.target/i386/l_fma_float_6.c | 16 +++++-----
 gcc/tree-ssa-math-opts.c                      | 29 +++++++++++++++++++
 28 files changed, 226 insertions(+), 144 deletions(-)
 create mode 100644 gcc/testsuite/gcc.target/i386/avx512f-pr95060.c

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 4234a72432c..ba230b42121 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,9 @@
+2020-05-13  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/95060
+	* tree-ssa-math-opts.c (convert_mult_to_fma_1): Fold a NEGATE_EXPR
+	if it is the single use of the FMA internal builtin.
+
 2020-05-13  Bin Cheng  <bin.cheng@linux.alibaba.com>
 
 	PR tree-optimization/94969
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index 3e5fe75b60b..02878c0647c 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -1,3 +1,32 @@
+2020-05-13  Jakub Jelinek  <jakub@redhat.com>
+
+	PR tree-optimization/95060
+	* gcc.target/i386/avx512f-pr95060.c: New test.
+	* gcc.target/i386/fma_double_1.c: Adjust expected insn counts.
+	* gcc.target/i386/fma_double_2.c: Likewise.
+	* gcc.target/i386/fma_double_3.c: Likewise.
+	* gcc.target/i386/fma_double_4.c: Likewise.
+	* gcc.target/i386/fma_double_5.c: Likewise.
+	* gcc.target/i386/fma_double_6.c: Likewise.
+	* gcc.target/i386/fma_float_1.c: Likewise.
+	* gcc.target/i386/fma_float_2.c: Likewise.
+	* gcc.target/i386/fma_float_3.c: Likewise.
+	* gcc.target/i386/fma_float_4.c: Likewise.
+	* gcc.target/i386/fma_float_5.c: Likewise.
+	* gcc.target/i386/fma_float_6.c: Likewise.
+	* gcc.target/i386/l_fma_double_1.c: Likewise.
+	* gcc.target/i386/l_fma_double_2.c: Likewise.
+	* gcc.target/i386/l_fma_double_3.c: Likewise.
+	* gcc.target/i386/l_fma_double_4.c: Likewise.
+	* gcc.target/i386/l_fma_double_5.c: Likewise.
+	* gcc.target/i386/l_fma_double_6.c: Likewise.
+	* gcc.target/i386/l_fma_float_1.c: Likewise.
+	* gcc.target/i386/l_fma_float_2.c: Likewise.
+	* gcc.target/i386/l_fma_float_3.c: Likewise.
+	* gcc.target/i386/l_fma_float_4.c: Likewise.
+	* gcc.target/i386/l_fma_float_5.c: Likewise.
+	* gcc.target/i386/l_fma_float_6.c: Likewise.
+
 2020-05-13  Martin Liska  <mliska@suse.cz>
 
 	PR sanitizer/95051
diff --git a/gcc/testsuite/gcc.target/i386/avx512f-pr95060.c b/gcc/testsuite/gcc.target/i386/avx512f-pr95060.c
new file mode 100644
index 00000000000..b38dc4f1e13
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/avx512f-pr95060.c
@@ -0,0 +1,22 @@
+/* PR tree-optimization/95060 */
+/* { dg-do compile } */
+/* { dg-options "-O3 -ffast-math -mavx512f" } */
+/* { dg-final { scan-assembler "\tvfnmsub" } } */
+/* { dg-final { scan-assembler-not "\tvfmadd" } } */
+
+#define N 32
+float r[N], a[N], b[N], c[N];
+
+void
+foo (void)
+{
+  for (int i = 0; i < N; i++)
+    r[i] = -(a[i] * b[i]) - c[i];
+}
+
+void
+bar (void)
+{
+  for (int i = 0; i < N; i++)
+    r[i] = -(a[i] * b[i] + c[i]);
+}
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_1.c b/gcc/testsuite/gcc.target/i386/fma_double_1.c
index c3aa3e83c02..767ee5ca63e 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_1.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_1.c
@@ -8,11 +8,9 @@
 
 #include "fma_1.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfmadd132sd" 8  } } */
 /* { dg-final { scan-assembler-times "vfmadd231sd" 4  } } */
-/* { dg-final { scan-assembler-times "vfmsub132sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfmsub132sd" 8  } } */
 /* { dg-final { scan-assembler-times "vfmsub231sd" 4  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132sd" 4  } } */
 /* { dg-final { scan-assembler-times "vfnmadd231sd" 4  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132sd" 4  } } */
 /* { dg-final { scan-assembler-times "vfnmsub231sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_2.c b/gcc/testsuite/gcc.target/i386/fma_double_2.c
index 843eff0a158..f15fb3b2cfa 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_2.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_2.c
@@ -8,7 +8,7 @@
 
 #include "fma_2.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132sd" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_3.c b/gcc/testsuite/gcc.target/i386/fma_double_3.c
index 3a04777c6bf..6b67774ca91 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_3.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_3.c
@@ -8,7 +8,7 @@
 
 #include "fma_3.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub\[132\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[132\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[132\]+sd" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub\[132\]+sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[132\]+sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[132\]+sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_4.c b/gcc/testsuite/gcc.target/i386/fma_double_4.c
index 51fc111adb2..267f6fd2d16 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_4.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_4.c
@@ -8,7 +8,7 @@
 
 #include "fma_4.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132sd" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_5.c b/gcc/testsuite/gcc.target/i386/fma_double_5.c
index 640b552b0f0..fd7fcabe5b5 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_5.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_5.c
@@ -8,7 +8,7 @@
 
 #include "fma_5.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[132\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub\[132\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[132\]+sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[132\]+sd" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd\[132\]+sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub\[132\]+sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[132\]+sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[132\]+sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_double_6.c b/gcc/testsuite/gcc.target/i386/fma_double_6.c
index 7b75a224f11..f7ab1ce5b75 100644
--- a/gcc/testsuite/gcc.target/i386/fma_double_6.c
+++ b/gcc/testsuite/gcc.target/i386/fma_double_6.c
@@ -8,7 +8,7 @@
 
 #include "fma_6.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132sd" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132sd" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132sd" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132sd" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132sd" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_1.c b/gcc/testsuite/gcc.target/i386/fma_float_1.c
index 67b1f3fe7cb..a4ac2cbbe00 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_1.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_1.c
@@ -8,11 +8,9 @@
 
 #include "fma_1.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfmadd132ss" 8  } } */
 /* { dg-final { scan-assembler-times "vfmadd231ss" 4  } } */
-/* { dg-final { scan-assembler-times "vfmsub132ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfmsub132ss" 8  } } */
 /* { dg-final { scan-assembler-times "vfmsub231ss" 4  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132ss" 4  } } */
 /* { dg-final { scan-assembler-times "vfnmadd231ss" 4  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132ss" 4  } } */
 /* { dg-final { scan-assembler-times "vfnmsub231ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_2.c b/gcc/testsuite/gcc.target/i386/fma_float_2.c
index a54644d0c70..a7509116b81 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_2.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_2.c
@@ -8,7 +8,7 @@
 
 #include "fma_2.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132ss" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_3.c b/gcc/testsuite/gcc.target/i386/fma_float_3.c
index 7986ce4ee76..d88a3bfcd25 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_3.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_3.c
@@ -8,7 +8,7 @@
 
 #include "fma_3.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[132\]+ss" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd\[132\]+ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub\[132\]+ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[132\]+ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[132\]+ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_4.c b/gcc/testsuite/gcc.target/i386/fma_float_4.c
index d9689d9a7af..cb1a81cbb26 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_4.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_4.c
@@ -8,7 +8,7 @@
 
 #include "fma_4.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132ss" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_5.c b/gcc/testsuite/gcc.target/i386/fma_float_5.c
index 2105ae627f0..3a62590d2a2 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_5.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_5.c
@@ -8,7 +8,7 @@
 
 #include "fma_5.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[132\]+ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[132\]+ss" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd\[132\]+ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub\[132\]+ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[132\]+ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[132\]+ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/fma_float_6.c b/gcc/testsuite/gcc.target/i386/fma_float_6.c
index c7580736834..30d8283bde3 100644
--- a/gcc/testsuite/gcc.target/i386/fma_float_6.c
+++ b/gcc/testsuite/gcc.target/i386/fma_float_6.c
@@ -8,7 +8,7 @@
 
 #include "fma_6.h"
 
-/* { dg-final { scan-assembler-times "vfmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfmsub132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmadd132ss" 8  } } */
-/* { dg-final { scan-assembler-times "vfnmsub132ss" 8  } } */
+/* { dg-final { scan-assembler-times "vfmadd132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfmsub132ss" 12  } } */
+/* { dg-final { scan-assembler-times "vfnmadd132ss" 4  } } */
+/* { dg-final { scan-assembler-times "vfnmsub132ss" 4  } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_1.c b/gcc/testsuite/gcc.target/i386/l_fma_double_1.c
index aa7c1307238..5089874faa5 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_1.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_1.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_1.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_2.c b/gcc/testsuite/gcc.target/i386/l_fma_double_2.c
index c59a89156bc..e4696204299 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_2.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_2.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_2.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_3.c b/gcc/testsuite/gcc.target/i386/l_fma_double_3.c
index c508d64d2e7..df986d0a633 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_3.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_3.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_3.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_4.c b/gcc/testsuite/gcc.target/i386/l_fma_double_4.c
index fec33e490af..ae065590f62 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_4.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_4.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_4.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_5.c b/gcc/testsuite/gcc.target/i386/l_fma_double_5.c
index 8bcfe4e68db..5d31abaa5a7 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_5.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_5.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_5.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_double_6.c b/gcc/testsuite/gcc.target/i386/l_fma_double_6.c
index 15da66b44cc..ff857fb02f1 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_double_6.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_double_6.c
@@ -11,11 +11,11 @@ typedef double adouble __attribute__((aligned(sizeof (double))));
 
 #include "l_fma_6.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 32 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 32 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+pd" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+pd" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+sd" 48 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+sd" 16 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+sd" 16 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_1.c b/gcc/testsuite/gcc.target/i386/l_fma_float_1.c
index 29eb77fc162..daef8767a5d 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_1.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_1.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_1.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_2.c b/gcc/testsuite/gcc.target/i386/l_fma_float_2.c
index 2943b2c68b2..ffa5c6f1b39 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_2.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_2.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_2.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_3.c b/gcc/testsuite/gcc.target/i386/l_fma_float_3.c
index c144dc1861a..a05ef5912f4 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_3.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_3.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_3.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_4.c b/gcc/testsuite/gcc.target/i386/l_fma_float_4.c
index a940dfd72f9..b0a37baafeb 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_4.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_4.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_4.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_5.c b/gcc/testsuite/gcc.target/i386/l_fma_float_5.c
index e7a12d47ea5..598275c54af 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_5.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_5.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_5.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/testsuite/gcc.target/i386/l_fma_float_6.c b/gcc/testsuite/gcc.target/i386/l_fma_float_6.c
index 82397a9d04f..1e7a2162c07 100644
--- a/gcc/testsuite/gcc.target/i386/l_fma_float_6.c
+++ b/gcc/testsuite/gcc.target/i386/l_fma_float_6.c
@@ -10,11 +10,11 @@
 
 #include "l_fma_6.h"
 
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 8 } } */
-/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 64 } } */
-/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 64 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ps" 12 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ps" 4 } } */
+/* { dg-final { scan-assembler-times "vfmadd\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfmsub\[123\]+ss" 96 } } */
+/* { dg-final { scan-assembler-times "vfnmadd\[123\]+ss" 32 } } */
+/* { dg-final { scan-assembler-times "vfnmsub\[123\]+ss" 32 } } */
diff --git a/gcc/tree-ssa-math-opts.c b/gcc/tree-ssa-math-opts.c
index 969c1a6b6c6..5fbaa24142e 100644
--- a/gcc/tree-ssa-math-opts.c
+++ b/gcc/tree-ssa-math-opts.c
@@ -2930,6 +2930,35 @@ convert_mult_to_fma_1 (tree mul_result, tree op1, tree op2)
 	  fprintf (dump_file, "\n");
 	}
 
+      /* If the FMA result is negated in a single use, fold the negation
+	 too.  */
+      orig_stmt = gsi_stmt (gsi);
+      use_operand_p use_p;
+      gimple *neg_stmt;
+      if (is_gimple_call (orig_stmt)
+	  && gimple_call_internal_p (orig_stmt)
+	  && gimple_call_lhs (orig_stmt)
+	  && TREE_CODE (gimple_call_lhs (orig_stmt)) == SSA_NAME
+	  && single_imm_use (gimple_call_lhs (orig_stmt), &use_p, &neg_stmt)
+	  && is_gimple_assign (neg_stmt)
+	  && gimple_assign_rhs_code (neg_stmt) == NEGATE_EXPR
+	  && !stmt_could_throw_p (cfun, neg_stmt))
+	{
+	  gsi = gsi_for_stmt (neg_stmt);
+	  if (fold_stmt (&gsi, follow_all_ssa_edges))
+	    {
+	      if (maybe_clean_or_replace_eh_stmt (neg_stmt, gsi_stmt (gsi)))
+		gcc_unreachable ();
+	      update_stmt (gsi_stmt (gsi));
+	      if (dump_file && (dump_flags & TDF_DETAILS))
+		{
+		  fprintf (dump_file, "Folded FMA negation ");
+		  print_gimple_stmt (dump_file, gsi_stmt (gsi), 0, TDF_NONE);
+		  fprintf (dump_file, "\n");
+		}
+	    }
+	}
+
       widen_mul_stats.fmas_inserted++;
     }
 }
-- 
2.30.2