[AArch64] Fix shrinkwrapping interactions with atomics (PR92692)

The separate shrinkwrapping pass may insert stores in the middle
of atomics loops which can cause issues on some implementations.
Avoid this by delaying splitting atomics patterns until after
prolog/epilog generation.

gcc/
	PR target/92692
	* config/aarch64/aarch64.c (aarch64_split_compare_and_swap)
	Add assert to ensure prolog has been emitted.
	(aarch64_split_atomic_op): Likewise.
	* config/aarch64/atomics.md (aarch64_compare_and_swap<mode>)
	Use epilogue_completed rather than reload_completed.
	(aarch64_atomic_exchange<mode>): Likewise.
	(aarch64_atomic_<atomic_optab><mode>): Likewise.
	(atomic_nand<mode>): Likewise.
	(aarch64_atomic_fetch_<atomic_optab><mode>): Likewise.
	(atomic_fetch_nand<mode>): Likewise.
	(aarch64_atomic_<atomic_optab>_fetch<mode>): Likewise.
	(atomic_nand_fetch<mode>): Likewise.

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 49ca5f92dec6e91765593935ccef0f9f813d597b..06bcb81acc841ab8ebe0c858ba61f38906c9afbd 100644 (file)
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,19 @@
+2020-01-17  Wilco Dijkstra  <wdijkstr@arm.com>
+
+       PR target/92692
+       * config/aarch64/aarch64.c (aarch64_split_compare_and_swap)
+       Add assert to ensure prolog has been emitted.
+       (aarch64_split_atomic_op): Likewise.
+       * config/aarch64/atomics.md (aarch64_compare_and_swap<mode>)
+       Use epilogue_completed rather than reload_completed.
+       (aarch64_atomic_exchange<mode>): Likewise.
+       (aarch64_atomic_<atomic_optab><mode>): Likewise.
+       (atomic_nand<mode>): Likewise.
+       (aarch64_atomic_fetch_<atomic_optab><mode>): Likewise.
+       (atomic_fetch_nand<mode>): Likewise.
+       (aarch64_atomic_<atomic_optab>_fetch<mode>): Likewise.
+       (atomic_nand_fetch<mode>): Likewise.
+
 2020-01-17  Richard Sandiford  <richard.sandiford@arm.com>
 
        PR target/93133

diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c
index 3660ce7bde09256668185a1f6a37ff0a13cdeb5d..ea9d6a12b12f9065ff9c40d005d71b0280bce312 100644 (file)
--- a/gcc/config/aarch64/aarch64.c
+++ b/gcc/config/aarch64/aarch64.c
@@ -18375,6 +18375,9 @@ aarch64_emit_post_barrier (enum memmodel model)
 void
 aarch64_split_compare_and_swap (rtx operands[])
 {
+  /* Split after prolog/epilog to avoid interactions with shrinkwrapping.  */
+  gcc_assert (epilogue_completed);
+
   rtx rval, mem, oldval, newval, scratch, x, model_rtx;
   machine_mode mode;
   bool is_weak;
@@ -18469,6 +18472,9 @@ void
 aarch64_split_atomic_op (enum rtx_code code, rtx old_out, rtx new_out, rtx mem,
                         rtx value, rtx model_rtx, rtx cond)
 {
+  /* Split after prolog/epilog to avoid interactions with shrinkwrapping.  */
+  gcc_assert (epilogue_completed);
+
   machine_mode mode = GET_MODE (mem);
   machine_mode wmode = (mode == DImode ? DImode : SImode);
   const enum memmodel model = memmodel_from_int (INTVAL (model_rtx));

diff --git a/gcc/config/aarch64/atomics.md b/gcc/config/aarch64/atomics.md
index c2bcabd0c3c2627b7222dcbc1af9c2e6b7ce6a76..996947799b5ef8445e9786b94e1ce62fd16e5b5c 100644 (file)
--- a/gcc/config/aarch64/atomics.md
+++ b/gcc/config/aarch64/atomics.md
@@ -56,7 +56,7 @@
    (clobber (match_scratch:SI 7 "=&r"))]
   ""
   "#"
-  "&& reload_completed"
+  "&& epilogue_completed"
   [(const_int 0)]
   {
     aarch64_split_compare_and_swap (operands);
@@ -80,7 +80,7 @@
    (clobber (match_scratch:SI 7 "=&r"))]
   ""
   "#"
-  "&& reload_completed"
+  "&& epilogue_completed"
   [(const_int 0)]
   {
     aarch64_split_compare_and_swap (operands);
@@ -104,7 +104,7 @@
    (clobber (match_scratch:SI 7 "=&r"))]
   ""
   "#"
-  "&& reload_completed"
+  "&& epilogue_completed"
   [(const_int 0)]
   {
     aarch64_split_compare_and_swap (operands);
@@ -223,7 +223,7 @@
    (clobber (match_scratch:SI 4 "=&r"))]
   ""
   "#"
-  "&& reload_completed"
+  "&& epilogue_completed"
   [(const_int 0)]
   {
     aarch64_split_atomic_op (SET, operands[0], NULL, operands[1],
@@ -344,7 +344,7 @@
   (clobber (match_scratch:SI 4 "=&r"))]
   ""
   "#"
-  "&& reload_completed"
+  "&& epilogue_completed"
   [(const_int 0)]
   {
     aarch64_split_atomic_op (<CODE>, NULL, operands[3], operands[0],
@@ -400,7 +400,7 @@
    (clobber (match_scratch:SI 4 "=&r"))]
   ""
   "#"
-  "&& reload_completed"
+  "&& epilogue_completed"
   [(const_int 0)]
   {
      aarch64_split_atomic_op (NOT, NULL, operands[3], operands[0],
@@ -504,7 +504,7 @@
    (clobber (match_scratch:SI 5 "=&r"))]
   ""
   "#"
-  "&& reload_completed"
+  "&& epilogue_completed"
   [(const_int 0)]
   {
     aarch64_split_atomic_op (<CODE>, operands[0], operands[4], operands[1],
@@ -551,7 +551,7 @@
    (clobber (match_scratch:SI 5 "=&r"))]
   ""
   "#"
-  "&& reload_completed"
+  "&& epilogue_completed"
   [(const_int 0)]
   {
     aarch64_split_atomic_op (NOT, operands[0], operands[4], operands[1],
@@ -604,7 +604,7 @@
    (clobber (match_scratch:SI 4 "=&r"))]
   ""
   "#"
-  "&& reload_completed"
+  "&& epilogue_completed"
   [(const_int 0)]
   {
     aarch64_split_atomic_op (<CODE>, NULL, operands[0], operands[1],
@@ -628,7 +628,7 @@
    (clobber (match_scratch:SI 4 "=&r"))]
   ""
   "#"
-  "&& reload_completed"
+  "&& epilogue_completed"
   [(const_int 0)]
   {
     aarch64_split_atomic_op (NOT, NULL, operands[0], operands[1],