From 9cd7b72071d5b1101de6b726391e8ba71a61638f Mon Sep 17 00:00:00 2001 From: Matthew Wahab Date: Tue, 22 Sep 2015 09:19:55 +0000 Subject: [PATCH] [AArch64] Use atomic instructions for swap and fetch-update operations. gcc/ 2015-09-22 Matthew Wahab * config/aarch64/aarch64-protos.h (aarch64_gen_atomic_ldop): Declare. * config/aarch64/aarch64.c (aarch64_emit_atomic_swap): New. (aarch64_gen_atomic_ldop): New. (aarch64_split_atomic_op): Fix whitespace and add a comment. * config/aarch64/atomics.md (UNSPECV_ATOMIC_SWP): New. (aarch64_compare_and_swap_lse): Fix some whitespace. (atomic_exchange): Replace with an expander. (aarch64_atomic_exchange): New. (aarch64_atomic_exchange_lse): New. (aarch64_atomic_): Fix some whitespace. (aarch64_atomic_swp): New. gcc/testsuite/ 2015-09-22 Matthew Wahab * gcc.target/aarch64/atomic-inst-ops.inc: (TEST_MODEL): New. (TEST_ONE): New. * gcc.target/aarch64/atomic-inst-swap.c: New. From-SVN: r227998 --- gcc/ChangeLog | 15 ++++ gcc/config/aarch64/aarch64-protos.h | 1 + gcc/config/aarch64/aarch64.c | 46 +++++++++++- gcc/config/aarch64/atomics.md | 71 +++++++++++++++++-- gcc/testsuite/ChangeLog | 6 ++ .../gcc.target/aarch64/atomic-inst-ops.inc | 13 ++++ .../gcc.target/aarch64/atomic-inst-swp.c | 44 ++++++++++++ 7 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 gcc/testsuite/gcc.target/aarch64/atomic-inst-swp.c diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 86b5c21f501..ae8f79404b8 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,18 @@ +2015-09-22 Matthew Wahab + + * config/aarch64/aarch64-protos.h (aarch64_gen_atomic_ldop): + Declare. + * config/aarch64/aarch64.c (aarch64_emit_atomic_swap): New. + (aarch64_gen_atomic_ldop): New. + (aarch64_split_atomic_op): Fix whitespace and add a comment. + * config/aarch64/atomics.md (UNSPECV_ATOMIC_SWP): New. + (aarch64_compare_and_swap_lse): Fix some whitespace. + (atomic_exchange): Replace with an expander. + (aarch64_atomic_exchange): New. + (aarch64_atomic_exchange_lse): New. + (aarch64_atomic_): Fix some whitespace. + (aarch64_atomic_swp): New. + 2015-09-22 Manuel López-Ibáñez * tree-inline.c (expand_call_inline): Use inform for extra note. diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h index ff1985137b3..eba4c76d4cc 100644 --- a/gcc/config/aarch64/aarch64-protos.h +++ b/gcc/config/aarch64/aarch64-protos.h @@ -378,6 +378,7 @@ rtx aarch64_load_tp (rtx); void aarch64_expand_compare_and_swap (rtx op[]); void aarch64_split_compare_and_swap (rtx op[]); void aarch64_gen_atomic_cas (rtx, rtx, rtx, rtx, rtx); +void aarch64_gen_atomic_ldop (enum rtx_code, rtx, rtx, rtx, rtx); void aarch64_split_atomic_op (enum rtx_code, rtx, rtx, rtx, rtx, rtx, rtx); bool aarch64_gen_adjusted_ldpstp (rtx *, bool, enum machine_mode, RTX_CODE); diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c index 39ed15e3b60..34c630b0875 100644 --- a/gcc/config/aarch64/aarch64.c +++ b/gcc/config/aarch64/aarch64.c @@ -10992,11 +10992,54 @@ aarch64_split_compare_and_swap (rtx operands[]) aarch64_emit_post_barrier (model); } +/* Emit an atomic swap. */ + +static void +aarch64_emit_atomic_swap (machine_mode mode, rtx dst, rtx value, + rtx mem, rtx model) +{ + rtx (*gen) (rtx, rtx, rtx, rtx); + + switch (mode) + { + case QImode: gen = gen_aarch64_atomic_swpqi; break; + case HImode: gen = gen_aarch64_atomic_swphi; break; + case SImode: gen = gen_aarch64_atomic_swpsi; break; + case DImode: gen = gen_aarch64_atomic_swpdi; break; + default: + gcc_unreachable (); + } + + emit_insn (gen (dst, mem, value, model)); +} + +/* Emit an atomic operation where the architecture supports it. */ + +void +aarch64_gen_atomic_ldop (enum rtx_code code, rtx out_data, + rtx mem, rtx value, rtx model_rtx) +{ + machine_mode mode = GET_MODE (mem); + + out_data = gen_lowpart (mode, out_data); + + switch (code) + { + case SET: + aarch64_emit_atomic_swap (mode, out_data, value, mem, model_rtx); + return; + + default: + /* The operation can't be done with atomic instructions. */ + gcc_unreachable (); + } +} + /* Split an atomic operation. */ void aarch64_split_atomic_op (enum rtx_code code, rtx old_out, rtx new_out, rtx mem, - rtx value, rtx model_rtx, rtx cond) + rtx value, rtx model_rtx, rtx cond) { machine_mode mode = GET_MODE (mem); machine_mode wmode = (mode == DImode ? DImode : SImode); @@ -11005,6 +11048,7 @@ aarch64_split_atomic_op (enum rtx_code code, rtx old_out, rtx new_out, rtx mem, rtx_code_label *label; rtx x; + /* Split the atomic operation into a sequence. */ label = gen_label_rtx (); emit_label (label); diff --git a/gcc/config/aarch64/atomics.md b/gcc/config/aarch64/atomics.md index 65d2cc911be..cb8053942c8 100644 --- a/gcc/config/aarch64/atomics.md +++ b/gcc/config/aarch64/atomics.md @@ -27,6 +27,7 @@ UNSPECV_ATOMIC_CMPSW ; Represent an atomic compare swap. UNSPECV_ATOMIC_EXCHG ; Represent an atomic exchange. UNSPECV_ATOMIC_CAS ; Represent an atomic CAS. + UNSPECV_ATOMIC_SWP ; Represent an atomic SWP. UNSPECV_ATOMIC_OP ; Represent an atomic operation. ]) @@ -134,7 +135,7 @@ (match_operand:SI 5 "const_int_operand") ;; mod_s (match_operand:SI 6 "const_int_operand")] ;; mod_f UNSPECV_ATOMIC_CMPSW))] - "TARGET_LSE " + "TARGET_LSE" "#" "&& reload_completed" [(const_int 0)] @@ -146,7 +147,28 @@ } ) -(define_insn_and_split "atomic_exchange" +(define_expand "atomic_exchange" + [(match_operand:ALLI 0 "register_operand" "") + (match_operand:ALLI 1 "aarch64_sync_memory_operand" "") + (match_operand:ALLI 2 "register_operand" "") + (match_operand:SI 3 "const_int_operand" "")] + "" + { + rtx (*gen) (rtx, rtx, rtx, rtx); + + /* Use an atomic SWP when available. */ + if (TARGET_LSE) + gen = gen_aarch64_atomic_exchange_lse; + else + gen = gen_aarch64_atomic_exchange; + + emit_insn (gen (operands[0], operands[1], operands[2], operands[3])); + + DONE; + } +) + +(define_insn_and_split "aarch64_atomic_exchange" [(set (match_operand:ALLI 0 "register_operand" "=&r") ;; output (match_operand:ALLI 1 "aarch64_sync_memory_operand" "+Q")) ;; memory (set (match_dup 1) @@ -162,7 +184,26 @@ [(const_int 0)] { aarch64_split_atomic_op (SET, operands[0], NULL, operands[1], - operands[2], operands[3], operands[4]); + operands[2], operands[3], operands[4]); + DONE; + } +) + +(define_insn_and_split "aarch64_atomic_exchange_lse" + [(set (match_operand:ALLI 0 "register_operand" "=&r") + (match_operand:ALLI 1 "aarch64_sync_memory_operand" "+Q")) + (set (match_dup 1) + (unspec_volatile:ALLI + [(match_operand:ALLI 2 "register_operand" "r") + (match_operand:SI 3 "const_int_operand" "")] + UNSPECV_ATOMIC_EXCHG))] + "TARGET_LSE" + "#" + "&& reload_completed" + [(const_int 0)] + { + aarch64_gen_atomic_ldop (SET, operands[0], operands[1], + operands[2], operands[3]); DONE; } ) @@ -183,7 +224,7 @@ [(const_int 0)] { aarch64_split_atomic_op (, NULL, operands[3], operands[0], - operands[1], operands[2], operands[4]); + operands[1], operands[2], operands[4]); DONE; } ) @@ -425,6 +466,28 @@ ;; ARMv8.1 LSE instructions. +;; Atomic swap with memory. +(define_insn "aarch64_atomic_swp" + [(set (match_operand:ALLI 0 "register_operand" "+&r") + (match_operand:ALLI 1 "aarch64_sync_memory_operand" "+Q")) + (set (match_dup 1) + (unspec_volatile:ALLI + [(match_operand:ALLI 2 "register_operand" "r") + (match_operand:SI 3 "const_int_operand" "")] + UNSPECV_ATOMIC_SWP))] + "TARGET_LSE && reload_completed" + { + enum memmodel model = memmodel_from_int (INTVAL (operands[3])); + if (is_mm_relaxed (model)) + return "swp\t%2, %0, %1"; + else if (is_mm_acquire (model) || is_mm_consume (model)) + return "swpa\t%2, %0, %1"; + else if (is_mm_release (model)) + return "swpl\t%2, %0, %1"; + else + return "swpal\t%2, %0, %1"; + }) + ;; Atomic compare-and-swap: HI and smaller modes. (define_insn "aarch64_atomic_cas" diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index e5c415e3b7d..c0e467fe0aa 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,9 @@ +2015-09-22 Matthew Wahab + + * gcc.target/aarch64/atomic-inst-ops.inc: (TEST_MODEL): New. + (TEST_ONE): New. + * gcc.target/aarch64/atomic-inst-swap.c: New. + 2015-09-22 Manuel López-Ibáñez * gcc.target/i386/inline_error.c (int bar): Use dg-message for note. diff --git a/gcc/testsuite/gcc.target/aarch64/atomic-inst-ops.inc b/gcc/testsuite/gcc.target/aarch64/atomic-inst-ops.inc index 72c7e5c3f0c..c2fdcba7930 100644 --- a/gcc/testsuite/gcc.target/aarch64/atomic-inst-ops.inc +++ b/gcc/testsuite/gcc.target/aarch64/atomic-inst-ops.inc @@ -32,6 +32,15 @@ typedef __uint128_t uint128; TEST_M##N (NAME, FN, int128, MODEL1, MODEL2) \ TEST_M##N (NAME, FN, uint128, MODEL1, MODEL2) +/* Models to test. */ +#define TEST_MODEL(NAME, FN, N) \ + TEST_TY (NAME##_relaxed, FN, N, __ATOMIC_RELAXED, DUMMY) \ + TEST_TY (NAME##_consume, FN, N, __ATOMIC_CONSUME, DUMMY) \ + TEST_TY (NAME##_acquire, FN, N, __ATOMIC_ACQUIRE, DUMMY) \ + TEST_TY (NAME##_release, FN, N, __ATOMIC_RELEASE, DUMMY) \ + TEST_TY (NAME##_acq_rel, FN, N, __ATOMIC_ACQ_REL, DUMMY) \ + TEST_TY (NAME##_seq_cst, FN, N, __ATOMIC_SEQ_CST, DUMMY) \ + /* Cross-product of models to test. */ #define TEST_MODEL_M1(NAME, FN, N, M) \ TEST_TY (NAME##_relaxed, FN, N, M, __ATOMIC_RELAXED) \ @@ -51,3 +60,7 @@ typedef __uint128_t uint128; /* Expand functions for a cross-product of memory models and types. */ #define TEST_TWO(NAME, FN) TEST_MODEL_M2 (NAME, FN) + +/* Expand functions for a set of memory models and types. */ +#define TEST_ONE(NAME, FN) TEST_MODEL (NAME, FN, 1) + diff --git a/gcc/testsuite/gcc.target/aarch64/atomic-inst-swp.c b/gcc/testsuite/gcc.target/aarch64/atomic-inst-swp.c new file mode 100644 index 00000000000..dabc9b9a08a --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/atomic-inst-swp.c @@ -0,0 +1,44 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -march=armv8-a+lse" } */ + +/* Test ARMv8.1-A SWP instruction. */ + +#include "atomic-inst-ops.inc" + +#define TEST TEST_ONE + +#define SWAP_ATOMIC(FN, TY, MODEL) \ + TY FNNAME (FN, TY) (TY* val, TY foo) \ + { \ + return __atomic_exchange_n (val, foo, MODEL); \ + } + +#define SWAP_ATOMIC_NORETURN(FN, TY, MODEL) \ + void FNNAME (FN, TY) (TY* val, TY* foo, TY* bar) \ + { \ + __atomic_exchange (val, foo, bar, MODEL); \ + } + + +TEST (swap_atomic, SWAP_ATOMIC) +TEST (swap_atomic_noreturn, SWAP_ATOMIC_NORETURN) + + +/* { dg-final { scan-assembler-times "swpb\t" 4} } */ +/* { dg-final { scan-assembler-times "swpab\t" 8} } */ +/* { dg-final { scan-assembler-times "swplb\t" 4} } */ +/* { dg-final { scan-assembler-times "swpalb\t" 8} } */ + +/* { dg-final { scan-assembler-times "swph\t" 4} } */ +/* { dg-final { scan-assembler-times "swpah\t" 8} } */ +/* { dg-final { scan-assembler-times "swplh\t" 4} } */ +/* { dg-final { scan-assembler-times "swpalh\t" 8} } */ + +/* { dg-final { scan-assembler-times "swp\t" 8} } */ +/* { dg-final { scan-assembler-times "swpa\t" 16} } */ +/* { dg-final { scan-assembler-times "swpl\t" 8} } */ +/* { dg-final { scan-assembler-times "swpal\t" 16} } */ + +/* { dg-final { scan-assembler-not "ldaxr\t" } } */ +/* { dg-final { scan-assembler-not "stlxr\t" } } */ +/* { dg-final { scan-assembler-not "dmb" } } */ -- 2.30.2