+2011-11-14 Richard Henderson <rth@redhat.com>
+
+ * config/rs6000/rs6000.c (emit_load_locked): Assert the mode is handled.
+ (emit_store_conditional): Likewise.
+ (rs6000_pre_atomic_barrier, rs6000_post_atomic_barrier): New.
+ (rs6000_adjust_atomic_subword): New.
+ (rs6000_mask_atomic_subword, rs6000_finish_atomic_subword): New.
+ (rs6000_expand_atomic_op): Rename from rs6000_emit_sync; rewrite
+ for pre-reload expansion.
+ (rs6000_split_compare_and_swap, rs6000_split_compare_and_swapqhi,
+ rs6000_expand_compare_and_swapqhi): Merge into ...
+ (rs6000_expand_atomic_compare_and_swap): ... here. New function.
+ rs6000_split_lock_test_and_set; expand immediately. Handle
+ QImode and HImode.
+ * config/rs6000/rs6000.md (UNSPEC_LWSYNC): Move and rename
+ from UNSPECV_LWSYNC.
+ * config/rs6000/sync.md (fetchopsi_constr, fetchopdi_constr): Remove.
+ (mem_thread_fence): New.
+ (hwsync): Rename from memory_barrier.
+ (*hwsync): Rename from *sync_internal.
+ (lwsync, *lwsync): Mirror hwsync implementation.
+ (isync): Don't reference memory.
+ (loadsync): New.
+ (atomic_load<INT>, atomic_store<INT>): New.
+ (ATOMIC): New mode iterator.
+ (load_locked<ATOMIC>): Rename from load_locked_<GPR>.
+ (store_conditional<ATOMIC>): Rename from store_conditional_<GPR>.
+ (sync_compare_and_swap<GPR>): Remove.
+ (sync_compare_and_swaphi, sync_compare_and_swapqi): Remove.
+ (sync_compare_and_swapqhi_internal): Remove.
+ (sync_lock_test_and_set<GPR>): Remove.
+ (sync_<FETCHOP><INT1>): Remove.
+ (*sync_<FETCHOP>si_internal, *sync_<FETCHOP>di_internal): Remove.
+ (sync_nand<INT1>, *sync_nand<GPR>_internal): Remove.
+ (sync_old_<FETCHOP><GPR>): Remove.
+ (*sync_old_<FETCHOP>si_internal): Remove.
+ (*sync_old_<FETCHOP>di_internal): Remove.
+ (sync_old_nand<INT1>): Remove.
+ (*sync_old_nand<GPR>_internal): Remove.
+ (sync_new_<FETCHOP><GPR>): Remove.
+ (*sync_new_<FETCHOP>si_internal): Remove.
+ (*sync_new_<FETCHOP>di_internal): Remove.
+ (sync_new_nand<INT1>): Remove.
+ (*sync_new_nand<GPR>_internal): Remove.
+ (*atomic_andsi, *atomic_anddi): Remove.
+ (*sync_addshort_internal, *sync_subshort_internal): Remove.
+ (*sync_andsi_internal, *sync_boolsi_internal): Remove.
+ (*sync_boolcshort_internal): Remove.
+ (sync_lock_release<INT1>): Remove.
+ (atomic_compare_and_swap<INT1>): New.
+ (atomic_exchange<INT1>): New.
+ (atomic_<FETCHOP><INT1>, atomic_nand<INT1>): New.
+ (atomic_fetch_<FETCHOP><INT1>, atomic_fetch_nand<INT1>): New.
+ (atomic_<FETCHOP>_fetch<INT1>, atomic_nand_fetch<INT1>): New.
+
2011-11-14 Uros Bizjak <ubizjak@gmail.com>
* config/i386/sse.md (round<mode>2): Use register_operand for
extern int rs6000_emit_cmove (rtx, rtx, rtx, rtx);
extern int rs6000_emit_vector_cond_expr (rtx, rtx, rtx, rtx, rtx, rtx);
extern void rs6000_emit_minmax (rtx, enum rtx_code, rtx, rtx);
-extern void rs6000_emit_sync (enum rtx_code, enum machine_mode,
- rtx, rtx, rtx, rtx, bool);
-extern void rs6000_split_atomic_op (enum rtx_code, rtx, rtx, rtx, rtx, rtx);
-extern void rs6000_split_compare_and_swap (rtx, rtx, rtx, rtx, rtx);
-extern void rs6000_expand_compare_and_swapqhi (rtx, rtx, rtx, rtx);
-extern void rs6000_split_compare_and_swapqhi (rtx, rtx, rtx, rtx, rtx, rtx);
-extern void rs6000_split_lock_test_and_set (rtx, rtx, rtx, rtx);
+extern void rs6000_expand_atomic_compare_and_swap (rtx op[]);
+extern void rs6000_expand_atomic_exchange (rtx op[]);
+extern void rs6000_expand_atomic_op (enum rtx_code, rtx, rtx, rtx, rtx, rtx);
extern void rs6000_emit_swdiv (rtx, rtx, rtx, bool);
extern void rs6000_emit_swrsqrt (rtx, rtx);
extern void output_toc (FILE *, rtx, int, enum machine_mode);
emit_move_insn (dest, target);
}
-/* Emit instructions to perform a load-reserved/store-conditional operation.
- The operation performed is an atomic
- (set M (CODE:MODE M OP))
- If not NULL, BEFORE is atomically set to M before the operation, and
- AFTER is set to M after the operation (that is, (CODE:MODE M OP)).
- If SYNC_P then a memory barrier is emitted before the operation.
- Either OP or M may be wrapped in a NOT operation. */
-
-void
-rs6000_emit_sync (enum rtx_code code, enum machine_mode mode,
- rtx m, rtx op, rtx before_param, rtx after_param,
- bool sync_p)
-{
- enum machine_mode used_mode;
- rtx the_op, set_before, set_after, set_atomic, cc_scratch, before, after;
- rtx used_m;
- rtvec vec;
- HOST_WIDE_INT imask = GET_MODE_MASK (mode);
- rtx shift = NULL_RTX;
-
- if (sync_p)
- emit_insn (gen_lwsync ());
-
- used_m = m;
-
- /* If this is smaller than SImode, we'll have to use SImode with
- adjustments. */
- if (mode == QImode || mode == HImode)
- {
- rtx newop, oldop;
-
- if (MEM_ALIGN (used_m) >= 32)
- {
- int ishift = 0;
- if (BYTES_BIG_ENDIAN)
- ishift = GET_MODE_BITSIZE (SImode) - GET_MODE_BITSIZE (mode);
-
- shift = GEN_INT (ishift);
- used_m = change_address (used_m, SImode, 0);
- }
- else
- {
- rtx addrSI, aligned_addr;
- int shift_mask = mode == QImode ? 0x18 : 0x10;
-
- addrSI = gen_lowpart_common (SImode,
- force_reg (Pmode, XEXP (used_m, 0)));
- addrSI = force_reg (SImode, addrSI);
- shift = gen_reg_rtx (SImode);
-
- emit_insn (gen_rlwinm (shift, addrSI, GEN_INT (3),
- GEN_INT (shift_mask)));
- emit_insn (gen_xorsi3 (shift, shift, GEN_INT (shift_mask)));
-
- aligned_addr = expand_binop (Pmode, and_optab,
- XEXP (used_m, 0),
- GEN_INT (-4), NULL_RTX,
- 1, OPTAB_LIB_WIDEN);
- used_m = change_address (used_m, SImode, aligned_addr);
- set_mem_align (used_m, 32);
- }
- /* It's safe to keep the old alias set of USED_M, because
- the operation is atomic and only affects the original
- USED_M. */
- m = used_m;
-
- if (GET_CODE (op) == NOT)
- {
- oldop = lowpart_subreg (SImode, XEXP (op, 0), mode);
- oldop = gen_rtx_NOT (SImode, oldop);
- }
- else
- oldop = lowpart_subreg (SImode, op, mode);
-
- switch (code)
- {
- case IOR:
- case XOR:
- newop = expand_binop (SImode, and_optab,
- oldop, GEN_INT (imask), NULL_RTX,
- 1, OPTAB_LIB_WIDEN);
- emit_insn (gen_ashlsi3 (newop, newop, shift));
- break;
-
- case NOT: /* NAND */
- newop = expand_binop (SImode, ior_optab,
- oldop, GEN_INT (~imask), NULL_RTX,
- 1, OPTAB_LIB_WIDEN);
- emit_insn (gen_rotlsi3 (newop, newop, shift));
- break;
-
- case AND:
- newop = expand_binop (SImode, ior_optab,
- oldop, GEN_INT (~imask), NULL_RTX,
- 1, OPTAB_LIB_WIDEN);
- emit_insn (gen_rotlsi3 (newop, newop, shift));
- break;
-
- case PLUS:
- case MINUS:
- {
- rtx mask;
-
- newop = expand_binop (SImode, and_optab,
- oldop, GEN_INT (imask), NULL_RTX,
- 1, OPTAB_LIB_WIDEN);
- emit_insn (gen_ashlsi3 (newop, newop, shift));
-
- mask = gen_reg_rtx (SImode);
- emit_move_insn (mask, GEN_INT (imask));
- emit_insn (gen_ashlsi3 (mask, mask, shift));
-
- if (code == PLUS)
- newop = gen_rtx_PLUS (SImode, m, newop);
- else
- newop = gen_rtx_MINUS (SImode, m, newop);
- newop = gen_rtx_AND (SImode, newop, mask);
- newop = gen_rtx_IOR (SImode, newop,
- gen_rtx_AND (SImode,
- gen_rtx_NOT (SImode, mask),
- m));
- break;
- }
-
- default:
- gcc_unreachable ();
- }
-
- op = newop;
- used_mode = SImode;
- before = gen_reg_rtx (used_mode);
- after = gen_reg_rtx (used_mode);
- }
- else
- {
- used_mode = mode;
- before = before_param;
- after = after_param;
-
- if (before == NULL_RTX)
- before = gen_reg_rtx (used_mode);
- if (after == NULL_RTX)
- after = gen_reg_rtx (used_mode);
- }
-
- if ((code == PLUS || code == MINUS)
- && used_mode != mode)
- the_op = op; /* Computed above. */
- else if (GET_CODE (op) == NOT && GET_CODE (m) != NOT)
- the_op = gen_rtx_fmt_ee (code, used_mode, op, m);
- else if (code == NOT)
- the_op = gen_rtx_fmt_ee (IOR, used_mode,
- gen_rtx_NOT (used_mode, m),
- gen_rtx_NOT (used_mode, op));
- else
- the_op = gen_rtx_fmt_ee (code, used_mode, m, op);
-
- set_after = gen_rtx_SET (VOIDmode, after, the_op);
- set_before = gen_rtx_SET (VOIDmode, before, used_m);
- set_atomic = gen_rtx_SET (VOIDmode, used_m,
- gen_rtx_UNSPEC (used_mode,
- gen_rtvec (1, the_op),
- UNSPEC_SYNC_OP));
- cc_scratch = gen_rtx_CLOBBER (VOIDmode, gen_rtx_SCRATCH (CCmode));
-
- if ((code == PLUS || code == MINUS) && used_mode != mode)
- vec = gen_rtvec (5, set_after, set_before, set_atomic, cc_scratch,
- gen_rtx_CLOBBER (VOIDmode, gen_rtx_SCRATCH (SImode)));
- else
- vec = gen_rtvec (4, set_after, set_before, set_atomic, cc_scratch);
- emit_insn (gen_rtx_PARALLEL (VOIDmode, vec));
-
- /* Shift and mask the return values properly. */
- if (used_mode != mode && before_param)
- {
- emit_insn (gen_lshrsi3 (before, before, shift));
- convert_move (before_param, before, 1);
- }
-
- if (used_mode != mode && after_param)
- {
- emit_insn (gen_lshrsi3 (after, after, shift));
- convert_move (after_param, after, 1);
- }
-
- /* The previous sequence will end with a branch that's dependent on
- the conditional store, so placing an isync will ensure that no
- other instructions (especially, no load or store instructions)
- can start before the atomic operation completes. */
- if (sync_p)
- emit_insn (gen_isync ());
-}
-
/* A subroutine of the atomic operation splitters. Jump to LABEL if
COND is true. Mark the jump as unlikely to be taken. */
emit_load_locked (enum machine_mode mode, rtx reg, rtx mem)
{
rtx (*fn) (rtx, rtx) = NULL;
- if (mode == SImode)
- fn = gen_load_locked_si;
- else if (mode == DImode)
- fn = gen_load_locked_di;
+
+ switch (mode)
+ {
+ case SImode:
+ fn = gen_load_lockedsi;
+ break;
+ case DImode:
+ fn = gen_load_lockeddi;
+ break;
+ default:
+ gcc_unreachable ();
+ }
emit_insn (fn (reg, mem));
}
emit_store_conditional (enum machine_mode mode, rtx res, rtx mem, rtx val)
{
rtx (*fn) (rtx, rtx, rtx) = NULL;
- if (mode == SImode)
- fn = gen_store_conditional_si;
- else if (mode == DImode)
- fn = gen_store_conditional_di;
+
+ switch (mode)
+ {
+ case SImode:
+ fn = gen_store_conditionalsi;
+ break;
+ case DImode:
+ fn = gen_store_conditionaldi;
+ break;
+ default:
+ gcc_unreachable ();
+ }
/* Emit sync before stwcx. to address PPC405 Erratum. */
if (PPC405_ERRATUM77)
- emit_insn (gen_memory_barrier ());
+ emit_insn (gen_hwsync ());
emit_insn (fn (res, mem, val));
}
-/* Expand an atomic fetch-and-operate pattern. CODE is the binary operation
- to perform. MEM is the memory on which to operate. VAL is the second
- operand of the binary operator. BEFORE and AFTER are optional locations to
- return the value of MEM either before of after the operation. SCRATCH is
- a scratch register. */
+/* Expand barriers before and after a load_locked/store_cond sequence. */
-void
-rs6000_split_atomic_op (enum rtx_code code, rtx mem, rtx val,
- rtx before, rtx after, rtx scratch)
+static void
+rs6000_pre_atomic_barrier (enum memmodel model)
+{
+ switch (model)
+ {
+ case MEMMODEL_RELAXED:
+ case MEMMODEL_CONSUME:
+ case MEMMODEL_ACQUIRE:
+ break;
+ case MEMMODEL_RELEASE:
+ case MEMMODEL_ACQ_REL:
+ emit_insn (gen_lwsync ());
+ break;
+ case MEMMODEL_SEQ_CST:
+ emit_insn (gen_hwsync ());
+ break;
+ default:
+ gcc_unreachable ();
+ }
+}
+
+static void
+rs6000_post_atomic_barrier (enum memmodel model)
+{
+ switch (model)
+ {
+ case MEMMODEL_RELAXED:
+ case MEMMODEL_CONSUME:
+ case MEMMODEL_RELEASE:
+ break;
+ case MEMMODEL_ACQUIRE:
+ case MEMMODEL_ACQ_REL:
+ case MEMMODEL_SEQ_CST:
+ emit_insn (gen_isync ());
+ break;
+ default:
+ gcc_unreachable ();
+ }
+}
+
+/* A subroutine of the various atomic expanders. For sub-word operations,
+ we must adjust things to operate on SImode. Given the original MEM,
+ return a new aligned memory. Also build and return the quantities by
+ which to shift and mask. */
+
+static rtx
+rs6000_adjust_atomic_subword (rtx mem, rtx *pshift, rtx *pmask)
{
+ rtx addr, align, shift, mask;
+ HOST_WIDE_INT shift_mask;
enum machine_mode mode = GET_MODE (mem);
- rtx label, x, cond = gen_rtx_REG (CCmode, CR0_REGNO);
- emit_insn (gen_lwsync ());
+ /* For smaller modes, we have to implement this via SImode. */
+ shift_mask = (mode == QImode ? 0x18 : 0x10);
- label = gen_label_rtx ();
- emit_label (label);
- label = gen_rtx_LABEL_REF (VOIDmode, label);
+ addr = XEXP (mem, 0);
+ addr = force_reg (GET_MODE (addr), addr);
+
+ /* Aligned memory containing subword. Generate a new memory. We
+ do not want any of the existing MEM_ATTR data, as we're now
+ accessing memory outside the original object. */
+ align = expand_simple_binop (Pmode, AND, addr, GEN_INT (-4),
+ NULL_RTX, 1, OPTAB_LIB_WIDEN);
+ mem = gen_rtx_MEM (SImode, align);
+ MEM_VOLATILE_P (mem) = 1;
- if (before == NULL_RTX)
- before = scratch;
- emit_load_locked (mode, before, mem);
+ /* Shift amount for subword relative to aligned word. */
+ shift = gen_reg_rtx (SImode);
+ addr = gen_lowpart (SImode, addr);
+ emit_insn (gen_rlwinm (shift, addr, GEN_INT (3), GEN_INT (shift_mask)));
+ shift = expand_simple_binop (SImode, XOR, shift, GEN_INT (shift_mask),
+ shift, 1, OPTAB_LIB_WIDEN);
+ *pshift = shift;
- if (code == NOT)
- x = gen_rtx_IOR (mode,
- gen_rtx_NOT (mode, before),
- gen_rtx_NOT (mode, val));
- else if (code == AND)
- x = gen_rtx_UNSPEC (mode, gen_rtvec (2, before, val), UNSPEC_AND);
- else
- x = gen_rtx_fmt_ee (code, mode, before, val);
+ /* Mask for insertion. */
+ mask = expand_simple_binop (SImode, ASHIFT, GEN_INT (GET_MODE_MASK (mode)),
+ shift, NULL_RTX, 1, OPTAB_LIB_WIDEN);
+ *pmask = mask;
- if (after != NULL_RTX)
- emit_insn (gen_rtx_SET (VOIDmode, after, copy_rtx (x)));
- emit_insn (gen_rtx_SET (VOIDmode, scratch, x));
+ return mem;
+}
- emit_store_conditional (mode, cond, mem, scratch);
+/* A subroutine of the various atomic expanders. For sub-word operands,
+ combine OLDVAL and NEWVAL via MASK. Returns a new pseduo. */
- x = gen_rtx_NE (VOIDmode, cond, const0_rtx);
- emit_unlikely_jump (x, label);
+static rtx
+rs6000_mask_atomic_subword (rtx oldval, rtx newval, rtx mask)
+{
+ rtx x;
- emit_insn (gen_isync ());
+ x = gen_reg_rtx (SImode);
+ emit_insn (gen_rtx_SET (VOIDmode, x,
+ gen_rtx_AND (SImode,
+ gen_rtx_NOT (SImode, mask),
+ oldval)));
+
+ x = expand_simple_binop (SImode, IOR, newval, x, x, 1, OPTAB_LIB_WIDEN);
+
+ return x;
+}
+
+/* A subroutine of the various atomic expanders. For sub-word operands,
+ extract WIDE to NARROW via SHIFT. */
+
+static void
+rs6000_finish_atomic_subword (rtx narrow, rtx wide, rtx shift)
+{
+ wide = expand_simple_binop (SImode, LSHIFTRT, wide, shift,
+ wide, 1, OPTAB_LIB_WIDEN);
+ emit_move_insn (narrow, gen_lowpart (GET_MODE (narrow), wide));
}
-/* Expand an atomic compare and swap operation. MEM is the memory on which
- to operate. OLDVAL is the old value to be compared. NEWVAL is the new
- value to be stored. SCRATCH is a scratch GPR. */
+/* Expand an atomic compare and swap operation. */
void
-rs6000_split_compare_and_swap (rtx retval, rtx mem, rtx oldval, rtx newval,
- rtx scratch)
+rs6000_expand_atomic_compare_and_swap (rtx operands[])
{
- enum machine_mode mode = GET_MODE (mem);
- rtx label1, label2, x, cond = gen_rtx_REG (CCmode, CR0_REGNO);
+ rtx boolval, retval, mem, oldval, newval, cond;
+ rtx label1, label2, x, mask, shift;
+ enum machine_mode mode;
+ enum memmodel mod_s, mod_f;
+ bool is_weak;
+
+ boolval = operands[0];
+ retval = operands[1];
+ mem = operands[2];
+ oldval = operands[3];
+ newval = operands[4];
+ is_weak = (INTVAL (operands[5]) != 0);
+ mod_s = (enum memmodel) INTVAL (operands[6]);
+ mod_f = (enum memmodel) INTVAL (operands[7]);
+ mode = GET_MODE (mem);
+
+ mask = shift = NULL_RTX;
+ if (mode == QImode || mode == HImode)
+ {
+ mem = rs6000_adjust_atomic_subword (mem, &shift, &mask);
+
+ /* Shift and mask OLDVAL into position with the word. */
+ oldval = convert_modes (SImode, mode, oldval, 1);
+ oldval = expand_simple_binop (SImode, ASHIFT, oldval, shift,
+ oldval, 1, OPTAB_LIB_WIDEN);
+
+ /* Shift and mask NEWVAL into position within the word. */
+ newval = convert_modes (SImode, mode, newval, 1);
+ newval = expand_simple_binop (SImode, ASHIFT, newval, shift,
+ newval, 1, OPTAB_LIB_WIDEN);
- emit_insn (gen_lwsync ());
+ /* Prepare to adjust the return value. */
+ retval = gen_reg_rtx (SImode);
+ mode = SImode;
+ }
+
+ rs6000_pre_atomic_barrier (mod_s);
- label1 = gen_rtx_LABEL_REF (VOIDmode, gen_label_rtx ());
+ emit_move_insn (boolval, const0_rtx);
+
+ label1 = NULL_RTX;
+ if (!is_weak)
+ {
+ label1 = gen_rtx_LABEL_REF (VOIDmode, gen_label_rtx ());
+ emit_label (XEXP (label1, 0));
+ }
label2 = gen_rtx_LABEL_REF (VOIDmode, gen_label_rtx ());
- emit_label (XEXP (label1, 0));
emit_load_locked (mode, retval, mem);
- x = gen_rtx_COMPARE (CCmode, retval, oldval);
- emit_insn (gen_rtx_SET (VOIDmode, cond, x));
+ x = retval;
+ if (mask)
+ {
+ x = expand_simple_binop (SImode, AND, retval, mask,
+ NULL_RTX, 1, OPTAB_LIB_WIDEN);
+ }
- x = gen_rtx_NE (VOIDmode, cond, const0_rtx);
+ x = gen_rtx_NE (VOIDmode, x, oldval);
+ x = rs6000_generate_compare (x, mode);
emit_unlikely_jump (x, label2);
- emit_move_insn (scratch, newval);
- emit_store_conditional (mode, cond, mem, scratch);
+ x = newval;
+ if (mask)
+ x = rs6000_mask_atomic_subword (retval, newval, mask);
- x = gen_rtx_NE (VOIDmode, cond, const0_rtx);
- emit_unlikely_jump (x, label1);
+ cond = gen_reg_rtx (CCmode);
+ emit_store_conditional (mode, cond, mem, x);
- emit_insn (gen_isync ());
- emit_label (XEXP (label2, 0));
+ if (is_weak)
+ {
+ /* ??? It's either this or an unlikely jump over (set bool 1). */
+ x = gen_rtx_EQ (SImode, cond, const0_rtx);
+ emit_insn (gen_rtx_SET (VOIDmode, boolval, x));
+ }
+ else
+ {
+ x = gen_rtx_NE (VOIDmode, cond, const0_rtx);
+ emit_unlikely_jump (x, label1);
+ emit_move_insn (boolval, const1_rtx);
+ }
+
+ if (mod_f != MEMMODEL_RELAXED)
+ emit_label (XEXP (label2, 0));
+
+ rs6000_post_atomic_barrier (mod_s);
+
+ if (mod_f == MEMMODEL_RELAXED)
+ emit_label (XEXP (label2, 0));
+
+ if (shift)
+ rs6000_finish_atomic_subword (operands[1], retval, shift);
}
-/* Expand an atomic test and set operation. MEM is the memory on which
- to operate. VAL is the value set. SCRATCH is a scratch GPR. */
+/* Expand an atomic exchange operation. */
void
-rs6000_split_lock_test_and_set (rtx retval, rtx mem, rtx val, rtx scratch)
+rs6000_expand_atomic_exchange (rtx operands[])
{
- enum machine_mode mode = GET_MODE (mem);
- rtx label, x, cond = gen_rtx_REG (CCmode, CR0_REGNO);
+ rtx retval, mem, val, cond;
+ enum machine_mode mode;
+ enum memmodel model;
+ rtx label, x, mask, shift;
+
+ retval = operands[0];
+ mem = operands[1];
+ val = operands[2];
+ model = (enum memmodel) INTVAL (operands[3]);
+ mode = GET_MODE (mem);
+
+ mask = shift = NULL_RTX;
+ if (mode == QImode || mode == HImode)
+ {
+ mem = rs6000_adjust_atomic_subword (mem, &shift, &mask);
+
+ /* Shift and mask VAL into position with the word. */
+ val = convert_modes (SImode, mode, val, 1);
+ val = expand_simple_binop (SImode, ASHIFT, val, shift,
+ val, 1, OPTAB_LIB_WIDEN);
+
+ /* Prepare to adjust the return value. */
+ retval = gen_reg_rtx (SImode);
+ mode = SImode;
+ }
+
+ rs6000_pre_atomic_barrier (model);
label = gen_rtx_LABEL_REF (VOIDmode, gen_label_rtx ());
emit_label (XEXP (label, 0));
emit_load_locked (mode, retval, mem);
- emit_move_insn (scratch, val);
- emit_store_conditional (mode, cond, mem, scratch);
+
+ x = val;
+ if (mask)
+ x = rs6000_mask_atomic_subword (retval, val, mask);
+
+ cond = gen_reg_rtx (CCmode);
+ emit_store_conditional (mode, cond, mem, x);
x = gen_rtx_NE (VOIDmode, cond, const0_rtx);
emit_unlikely_jump (x, label);
- emit_insn (gen_isync ());
+ rs6000_post_atomic_barrier (model);
+
+ if (shift)
+ rs6000_finish_atomic_subword (operands[0], retval, shift);
}
+/* Expand an atomic fetch-and-operate pattern. CODE is the binary operation
+ to perform. MEM is the memory on which to operate. VAL is the second
+ operand of the binary operator. BEFORE and AFTER are optional locations to
+ return the value of MEM either before of after the operation. MODEL_RTX
+ is a CONST_INT containing the memory model to use. */
+
void
-rs6000_expand_compare_and_swapqhi (rtx dst, rtx mem, rtx oldval, rtx newval)
+rs6000_expand_atomic_op (enum rtx_code code, rtx mem, rtx val,
+ rtx orig_before, rtx orig_after, rtx model_rtx)
{
+ enum memmodel model = (enum memmodel) INTVAL (model_rtx);
enum machine_mode mode = GET_MODE (mem);
- rtx addrSI, align, wdst, shift, mask;
- HOST_WIDE_INT shift_mask = mode == QImode ? 0x18 : 0x10;
- HOST_WIDE_INT imask = GET_MODE_MASK (mode);
+ rtx label, x, cond, mask, shift;
+ rtx before = orig_before, after = orig_after;
- /* Shift amount for subword relative to aligned word. */
- addrSI = force_reg (GET_MODE (XEXP (mem, 0)), XEXP (mem, 0));
- addrSI = force_reg (SImode, gen_lowpart_common (SImode, addrSI));
- shift = gen_reg_rtx (SImode);
- emit_insn (gen_rlwinm (shift, addrSI, GEN_INT (3),
- GEN_INT (shift_mask)));
- emit_insn (gen_xorsi3 (shift, shift, GEN_INT (shift_mask)));
-
- /* Shift and mask old value into position within word. */
- oldval = convert_modes (SImode, mode, oldval, 1);
- oldval = expand_binop (SImode, and_optab,
- oldval, GEN_INT (imask), NULL_RTX,
- 1, OPTAB_LIB_WIDEN);
- emit_insn (gen_ashlsi3 (oldval, oldval, shift));
-
- /* Shift and mask new value into position within word. */
- newval = convert_modes (SImode, mode, newval, 1);
- newval = expand_binop (SImode, and_optab,
- newval, GEN_INT (imask), NULL_RTX,
- 1, OPTAB_LIB_WIDEN);
- emit_insn (gen_ashlsi3 (newval, newval, shift));
+ mask = shift = NULL_RTX;
+ if (mode == QImode || mode == HImode)
+ {
+ mem = rs6000_adjust_atomic_subword (mem, &shift, &mask);
- /* Mask for insertion. */
- mask = gen_reg_rtx (SImode);
- emit_move_insn (mask, GEN_INT (imask));
- emit_insn (gen_ashlsi3 (mask, mask, shift));
-
- /* Address of aligned word containing subword. */
- align = expand_binop (Pmode, and_optab, XEXP (mem, 0), GEN_INT (-4),
- NULL_RTX, 1, OPTAB_LIB_WIDEN);
- mem = change_address (mem, SImode, align);
- set_mem_align (mem, 32);
- MEM_VOLATILE_P (mem) = 1;
+ /* Shift and mask VAL into position with the word. */
+ val = convert_modes (SImode, mode, val, 1);
+ val = expand_simple_binop (SImode, ASHIFT, val, shift,
+ val, 1, OPTAB_LIB_WIDEN);
- wdst = gen_reg_rtx (SImode);
- emit_insn (gen_sync_compare_and_swapqhi_internal (wdst, mask,
- oldval, newval, mem));
+ switch (code)
+ {
+ case IOR:
+ case XOR:
+ /* We've already zero-extended VAL. That is sufficient to
+ make certain that it does not affect other bits. */
+ mask = NULL;
+ break;
- /* Shift the result back. */
- emit_insn (gen_lshrsi3 (wdst, wdst, shift));
+ case AND:
+ /* If we make certain that all of the other bits in VAL are
+ set, that will be sufficient to not affect other bits. */
+ x = gen_rtx_NOT (SImode, mask);
+ x = gen_rtx_IOR (SImode, x, val);
+ emit_insn (gen_rtx_SET (VOIDmode, val, x));
+ mask = NULL;
+ break;
- emit_move_insn (dst, gen_lowpart (mode, wdst));
-}
+ case NOT:
+ case PLUS:
+ case MINUS:
+ /* These will all affect bits outside the field and need
+ adjustment via MASK within the loop. */
+ break;
-void
-rs6000_split_compare_and_swapqhi (rtx dest, rtx mask,
- rtx oldval, rtx newval, rtx mem,
- rtx scratch)
-{
- rtx label1, label2, x, cond = gen_rtx_REG (CCmode, CR0_REGNO);
+ default:
+ gcc_unreachable ();
+ }
- emit_insn (gen_lwsync ());
- label1 = gen_rtx_LABEL_REF (VOIDmode, gen_label_rtx ());
- label2 = gen_rtx_LABEL_REF (VOIDmode, gen_label_rtx ());
- emit_label (XEXP (label1, 0));
+ /* Prepare to adjust the return value. */
+ before = gen_reg_rtx (SImode);
+ if (after)
+ after = gen_reg_rtx (SImode);
+ mode = SImode;
+ }
- emit_load_locked (SImode, scratch, mem);
+ rs6000_pre_atomic_barrier (model);
- /* Mask subword within loaded value for comparison with oldval.
- Use UNSPEC_AND to avoid clobber.*/
- emit_insn (gen_rtx_SET (SImode, dest,
- gen_rtx_UNSPEC (SImode,
- gen_rtvec (2, scratch, mask),
- UNSPEC_AND)));
+ label = gen_label_rtx ();
+ emit_label (label);
+ label = gen_rtx_LABEL_REF (VOIDmode, label);
- x = gen_rtx_COMPARE (CCmode, dest, oldval);
- emit_insn (gen_rtx_SET (VOIDmode, cond, x));
+ if (before == NULL_RTX)
+ before = gen_reg_rtx (mode);
- x = gen_rtx_NE (VOIDmode, cond, const0_rtx);
- emit_unlikely_jump (x, label2);
+ emit_load_locked (mode, before, mem);
- /* Clear subword within loaded value for insertion of new value. */
- emit_insn (gen_rtx_SET (SImode, scratch,
- gen_rtx_AND (SImode,
- gen_rtx_NOT (SImode, mask), scratch)));
- emit_insn (gen_iorsi3 (scratch, scratch, newval));
- emit_store_conditional (SImode, cond, mem, scratch);
+ if (code == NOT)
+ {
+ x = expand_simple_binop (mode, AND, before, val,
+ NULL_RTX, 1, OPTAB_LIB_WIDEN);
+ after = expand_simple_unop (mode, NOT, x, after, 1);
+ }
+ else
+ {
+ after = expand_simple_binop (mode, code, before, val,
+ after, 1, OPTAB_LIB_WIDEN);
+ }
+
+ x = after;
+ if (mask)
+ {
+ x = expand_simple_binop (SImode, AND, after, mask,
+ NULL_RTX, 1, OPTAB_LIB_WIDEN);
+ x = rs6000_mask_atomic_subword (before, x, mask);
+ }
+
+ cond = gen_reg_rtx (CCmode);
+ emit_store_conditional (mode, cond, mem, x);
x = gen_rtx_NE (VOIDmode, cond, const0_rtx);
- emit_unlikely_jump (x, label1);
+ emit_unlikely_jump (x, label);
- emit_insn (gen_isync ());
- emit_label (XEXP (label2, 0));
-}
+ rs6000_post_atomic_barrier (model);
+ if (shift)
+ {
+ if (orig_before)
+ rs6000_finish_atomic_subword (orig_before, before, shift);
+ if (orig_after)
+ rs6000_finish_atomic_subword (orig_after, after, shift);
+ }
+ else if (orig_after && after != orig_after)
+ emit_move_insn (orig_after, after);
+}
- /* Emit instructions to move SRC to DST. Called by splitters for
+/* Emit instructions to move SRC to DST. Called by splitters for
multi-register moves. It will emit at most one instruction for
each register that is accessed; that is, it won't emit li/lis pairs
(or equivalent for 64-bit code). One of SRC or DST must be a hard
UNSPEC_SP_SET
UNSPEC_SP_TEST
UNSPEC_SYNC
+ UNSPEC_LWSYNC
UNSPEC_SYNC_OP
UNSPEC_ATOMIC
UNSPEC_CMPXCHG
UNSPECV_PROBE_STACK_RANGE ; probe range of stack addresses
UNSPECV_EH_RR ; eh_reg_restore
UNSPECV_ISYNC ; isync instruction
- UNSPECV_LWSYNC ; lwsync
])
\f
(define_code_attr fetchop_pred
[(plus "add_operand") (minus "gpc_reg_operand")
(ior "logical_operand") (xor "logical_operand") (and "and_operand")])
-(define_code_attr fetchopsi_constr
- [(plus "rIL") (minus "r") (ior "rKL") (xor "rKL") (and "rTKL")])
-(define_code_attr fetchopdi_constr
- [(plus "rIL") (minus "r") (ior "rKJF") (xor "rKJF") (and "rSTKJ")])
-(define_expand "memory_barrier"
+(define_expand "mem_thread_fence"
+ [(match_operand:SI 0 "const_int_operand" "")] ;; model
+ ""
+{
+ enum memmodel model = (enum memmodel) INTVAL (operands[0]);
+ switch (model)
+ {
+ case MEMMODEL_RELAXED:
+ break;
+ case MEMMODEL_CONSUME:
+ case MEMMODEL_ACQUIRE:
+ case MEMMODEL_RELEASE:
+ case MEMMODEL_ACQ_REL:
+ emit_insn (gen_lwsync ());
+ break;
+ case MEMMODEL_SEQ_CST:
+ emit_insn (gen_hwsync ());
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ DONE;
+})
+
+(define_expand "hwsync"
[(set (match_dup 0)
(unspec:BLK [(match_dup 0)] UNSPEC_SYNC))]
""
MEM_VOLATILE_P (operands[0]) = 1;
})
-(define_insn "*sync_internal"
+(define_insn "*hwsync"
[(set (match_operand:BLK 0 "" "")
(unspec:BLK [(match_dup 0)] UNSPEC_SYNC))]
""
"{dcs|sync}"
[(set_attr "type" "sync")])
-(define_insn "load_locked_<mode>"
- [(set (match_operand:GPR 0 "gpc_reg_operand" "=r")
- (unspec_volatile:GPR
- [(match_operand:GPR 1 "memory_operand" "Z")] UNSPECV_LL))]
- "TARGET_POWERPC"
- "<larx> %0,%y1"
- [(set_attr "type" "load_l")])
-
-(define_insn "store_conditional_<mode>"
- [(set (match_operand:CC 0 "cc_reg_operand" "=x")
- (unspec_volatile:CC [(const_int 0)] UNSPECV_SC))
- (set (match_operand:GPR 1 "memory_operand" "=Z")
- (match_operand:GPR 2 "gpc_reg_operand" "r"))]
- "TARGET_POWERPC"
- "<stcx> %2,%y1"
- [(set_attr "type" "store_c")])
-
-(define_insn_and_split "sync_compare_and_swap<mode>"
- [(set (match_operand:GPR 0 "gpc_reg_operand" "=&r")
- (match_operand:GPR 1 "memory_operand" "+Z"))
- (set (match_dup 1)
- (unspec:GPR
- [(match_operand:GPR 2 "reg_or_short_operand" "rI")
- (match_operand:GPR 3 "gpc_reg_operand" "r")]
- UNSPEC_CMPXCHG))
- (clobber (match_scratch:GPR 4 "=&r"))
- (clobber (match_scratch:CC 5 "=&x"))]
- "TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
+(define_expand "lwsync"
+ [(set (match_dup 0)
+ (unspec:BLK [(match_dup 0)] UNSPEC_LWSYNC))]
+ ""
{
- rs6000_split_compare_and_swap (operands[0], operands[1], operands[2],
- operands[3], operands[4]);
- DONE;
+ operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
+ MEM_VOLATILE_P (operands[0]) = 1;
})
-(define_expand "sync_compare_and_swaphi"
- [(match_operand:HI 0 "gpc_reg_operand" "")
- (match_operand:HI 1 "memory_operand" "")
- (match_operand:HI 2 "gpc_reg_operand" "")
- (match_operand:HI 3 "gpc_reg_operand" "")]
- "TARGET_POWERPC"
+(define_insn "*lwsync"
+ [(set (match_operand:BLK 0 "" "")
+ (unspec:BLK [(match_dup 0)] UNSPEC_LWSYNC))]
+ ""
{
- rs6000_expand_compare_and_swapqhi (operands[0], operands[1],
- operands[2], operands[3]);
- DONE;
-})
+ /* Some AIX assemblers don't accept lwsync, so we use a .long. */
+ if (TARGET_NO_LWSYNC)
+ return "sync";
+ else if (TARGET_LWSYNC_INSTRUCTION)
+ return "lwsync";
+ else
+ return ".long 0x7c2004ac";
+}
+ [(set_attr "type" "sync")])
-(define_expand "sync_compare_and_swapqi"
- [(match_operand:QI 0 "gpc_reg_operand" "")
- (match_operand:QI 1 "memory_operand" "")
- (match_operand:QI 2 "gpc_reg_operand" "")
- (match_operand:QI 3 "gpc_reg_operand" "")]
- "TARGET_POWERPC"
-{
- rs6000_expand_compare_and_swapqhi (operands[0], operands[1],
- operands[2], operands[3]);
- DONE;
-})
+(define_insn "isync"
+ [(unspec_volatile:BLK [(const_int 0)] UNSPECV_ISYNC)]
+ ""
+ "{ics|isync}"
+ [(set_attr "type" "isync")])
-(define_insn_and_split "sync_compare_and_swapqhi_internal"
- [(set (match_operand:SI 0 "gpc_reg_operand" "=&r")
- (match_operand:SI 4 "memory_operand" "+Z"))
- (set (match_dup 4)
- (unspec:SI
- [(match_operand:SI 1 "gpc_reg_operand" "r")
- (match_operand:SI 2 "gpc_reg_operand" "r")
- (match_operand:SI 3 "gpc_reg_operand" "r")]
- UNSPEC_CMPXCHG))
- (clobber (match_scratch:SI 5 "=&r"))
- (clobber (match_scratch:CC 6 "=&x"))]
- "TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
+;; The control dependency used for load dependency described
+;; in B.2.3 of the Power ISA 2.06B.
+(define_insn "loadsync"
+ [(unspec_volatile:BLK [(match_operand 0 "register_operand" "r")]
+ UNSPECV_ISYNC)
+ (clobber (match_scratch:CC 1 "=y"))]
+ ""
+ "cmpw %1,%0,%0\;bne- %1,$+4\;isync"
+ [(set_attr "type" "isync")
+ (set_attr "length" "12")])
+
+(define_expand "atomic_load<mode>"
+ [(set (match_operand:INT 0 "register_operand" "") ;; output
+ (match_operand:INT 1 "memory_operand" "")) ;; memory
+ (use (match_operand:SI 2 "const_int_operand" ""))] ;; model
+ ""
{
- rs6000_split_compare_and_swapqhi (operands[0], operands[1],
- operands[2], operands[3], operands[4],
- operands[5]);
- DONE;
-})
+ enum memmodel model = (enum memmodel) INTVAL (operands[2]);
-(define_insn_and_split "sync_lock_test_and_set<mode>"
- [(set (match_operand:GPR 0 "gpc_reg_operand" "=&r")
- (match_operand:GPR 1 "memory_operand" "+Z"))
- (set (match_dup 1)
- (unspec:GPR
- [(match_operand:GPR 2 "reg_or_short_operand" "rL")]
- UNSPEC_XCHG))
- (clobber (match_scratch:GPR 3 "=&r"))
- (clobber (match_scratch:CC 4 "=&x"))]
- "TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
-{
- rs6000_split_lock_test_and_set (operands[0], operands[1], operands[2],
- operands[3]);
- DONE;
-})
+ if (model == MEMMODEL_SEQ_CST)
+ emit_insn (gen_hwsync ());
-(define_expand "sync_<fetchop_name><mode>"
- [(parallel [(set (match_operand:INT1 0 "memory_operand" "")
- (unspec:INT1
- [(FETCHOP:INT1 (match_dup 0)
- (match_operand:INT1 1 "<fetchop_pred>" ""))]
- UNSPEC_ATOMIC))
- (clobber (scratch:INT1))
- (clobber (scratch:CC))])]
- "TARGET_POWERPC"
- "
-{
- if (<MODE>mode != SImode && <MODE>mode != DImode)
+ emit_move_insn (operands[0], operands[1]);
+
+ switch (model)
{
- if (PPC405_ERRATUM77)
- FAIL;
- rs6000_emit_sync (<CODE>, <MODE>mode, operands[0], operands[1],
- NULL_RTX, NULL_RTX, true);
- DONE;
+ case MEMMODEL_RELAXED:
+ break;
+ case MEMMODEL_CONSUME:
+ case MEMMODEL_ACQUIRE:
+ case MEMMODEL_SEQ_CST:
+ emit_insn (gen_loadsync (operands[0]));
+ break;
+ default:
+ gcc_unreachable ();
}
-}")
-
-(define_insn_and_split "*sync_<fetchop_name>si_internal"
- [(set (match_operand:SI 0 "memory_operand" "+Z")
- (unspec:SI
- [(FETCHOP:SI (match_dup 0)
- (match_operand:SI 1 "<fetchop_pred>" "<fetchopsi_constr>"))]
- UNSPEC_ATOMIC))
- (clobber (match_scratch:SI 2 "=&b"))
- (clobber (match_scratch:CC 3 "=&x"))]
- "TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
-{
- rs6000_split_atomic_op (<CODE>, operands[0], operands[1],
- NULL_RTX, NULL_RTX, operands[2]);
DONE;
})
-(define_insn_and_split "*sync_<fetchop_name>di_internal"
- [(set (match_operand:DI 0 "memory_operand" "+Z")
- (unspec:DI
- [(FETCHOP:DI (match_dup 0)
- (match_operand:DI 1 "<fetchop_pred>" "<fetchopdi_constr>"))]
- UNSPEC_ATOMIC))
- (clobber (match_scratch:DI 2 "=&b"))
- (clobber (match_scratch:CC 3 "=&x"))]
- "TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
+(define_expand "atomic_store<mode>"
+ [(set (match_operand:INT 0 "memory_operand" "") ;; memory
+ (match_operand:INT 1 "register_operand" "")) ;; input
+ (use (match_operand:SI 2 "const_int_operand" ""))] ;; model
+ ""
{
- rs6000_split_atomic_op (<CODE>, operands[0], operands[1],
- NULL_RTX, NULL_RTX, operands[2]);
+ enum memmodel model = (enum memmodel) INTVAL (operands[2]);
+ switch (model)
+ {
+ case MEMMODEL_RELAXED:
+ break;
+ case MEMMODEL_RELEASE:
+ emit_insn (gen_lwsync ());
+ break;
+ case MEMMODEL_SEQ_CST:
+ emit_insn (gen_hwsync ());
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ emit_move_insn (operands[0], operands[1]);
DONE;
})
-(define_expand "sync_nand<mode>"
- [(parallel [(set (match_operand:INT1 0 "memory_operand" "")
- (unspec:INT1
- [(ior:INT1 (not:INT1 (match_dup 0))
- (not:INT1 (match_operand:INT1 1 "gpc_reg_operand" "")))]
- UNSPEC_ATOMIC))
- (clobber (scratch:INT1))
- (clobber (scratch:CC))])]
- "TARGET_POWERPC"
- "
-{
- if (<MODE>mode != SImode && <MODE>mode != DImode)
- {
- FAIL;
- if (PPC405_ERRATUM77)
- FAIL;
- rs6000_emit_sync (NOT, <MODE>mode, operands[0], operands[1],
- NULL_RTX, NULL_RTX, true);
- DONE;
- }
-}")
+;; ??? Power ISA 2.06B says that there *is* a load-{byte,half}-and-reserve
+;; opcode that is "phased-in". Not implemented as of Power7, so not yet used,
+;; but let's prepare the macros anyway.
+
+(define_mode_iterator ATOMIC [SI (DI "TARGET_64BIT")])
-(define_insn_and_split "*sync_nand<mode>_internal"
- [(set (match_operand:GPR 0 "memory_operand" "+Z")
- (unspec:GPR
- [(ior:GPR (not:GPR (match_dup 0))
- (not:GPR (match_operand:GPR 1 "gpc_reg_operand" "r")))]
- UNSPEC_ATOMIC))
- (clobber (match_scratch:GPR 2 "=&r"))
- (clobber (match_scratch:CC 3 "=&x"))]
+(define_insn "load_locked<mode>"
+ [(set (match_operand:ATOMIC 0 "gpc_reg_operand" "=r")
+ (unspec_volatile:ATOMIC
+ [(match_operand:ATOMIC 1 "memory_operand" "Z")] UNSPECV_LL))]
"TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
-{
- rs6000_split_atomic_op (NOT, operands[0], operands[1],
- NULL_RTX, NULL_RTX, operands[2]);
- DONE;
-})
+ "<larx> %0,%y1"
+ [(set_attr "type" "load_l")])
-(define_expand "sync_old_<fetchop_name><mode>"
- [(parallel [(set (match_operand:INT1 0 "gpc_reg_operand" "")
- (match_operand:INT1 1 "memory_operand" ""))
- (set (match_dup 1)
- (unspec:INT1
- [(FETCHOP:INT1 (match_dup 1)
- (match_operand:INT1 2 "<fetchop_pred>" ""))]
- UNSPEC_ATOMIC))
- (clobber (scratch:INT1))
- (clobber (scratch:CC))])]
+(define_insn "store_conditional<mode>"
+ [(set (match_operand:CC 0 "cc_reg_operand" "=x")
+ (unspec_volatile:CC [(const_int 0)] UNSPECV_SC))
+ (set (match_operand:ATOMIC 1 "memory_operand" "=Z")
+ (match_operand:ATOMIC 2 "gpc_reg_operand" "r"))]
"TARGET_POWERPC"
- "
-{
- if (<MODE>mode != SImode && <MODE>mode != DImode)
- {
- if (PPC405_ERRATUM77)
- FAIL;
- rs6000_emit_sync (<CODE>, <MODE>mode, operands[1], operands[2],
- operands[0], NULL_RTX, true);
- DONE;
- }
-}")
+ "<stcx> %2,%y1"
+ [(set_attr "type" "store_c")])
-(define_insn_and_split "*sync_old_<fetchop_name>si_internal"
- [(set (match_operand:SI 0 "gpc_reg_operand" "=&r")
- (match_operand:SI 1 "memory_operand" "+Z"))
- (set (match_dup 1)
- (unspec:SI
- [(FETCHOP:SI (match_dup 1)
- (match_operand:SI 2 "<fetchop_pred>" "<fetchopsi_constr>"))]
- UNSPEC_ATOMIC))
- (clobber (match_scratch:SI 3 "=&b"))
- (clobber (match_scratch:CC 4 "=&x"))]
+(define_expand "atomic_compare_and_swap<mode>"
+ [(match_operand:SI 0 "gpc_reg_operand" "") ;; bool out
+ (match_operand:INT1 1 "gpc_reg_operand" "") ;; val out
+ (match_operand:INT1 2 "memory_operand" "") ;; memory
+ (match_operand:INT1 3 "reg_or_short_operand" "") ;; expected
+ (match_operand:INT1 4 "gpc_reg_operand" "") ;; desired
+ (match_operand:SI 5 "const_int_operand" "") ;; is_weak
+ (match_operand:SI 6 "const_int_operand" "") ;; model succ
+ (match_operand:SI 7 "const_int_operand" "")] ;; model fail
"TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
{
- rs6000_split_atomic_op (<CODE>, operands[1], operands[2],
- operands[0], NULL_RTX, operands[3]);
+ rs6000_expand_atomic_compare_and_swap (operands);
DONE;
})
-(define_insn_and_split "*sync_old_<fetchop_name>di_internal"
- [(set (match_operand:DI 0 "gpc_reg_operand" "=&r")
- (match_operand:DI 1 "memory_operand" "+Z"))
- (set (match_dup 1)
- (unspec:DI
- [(FETCHOP:DI (match_dup 1)
- (match_operand:DI 2 "<fetchop_pred>" "<fetchopdi_constr>"))]
- UNSPEC_ATOMIC))
- (clobber (match_scratch:DI 3 "=&b"))
- (clobber (match_scratch:CC 4 "=&x"))]
+(define_expand "atomic_exchange<mode>"
+ [(match_operand:INT1 0 "gpc_reg_operand" "") ;; output
+ (match_operand:INT1 1 "memory_operand" "") ;; memory
+ (match_operand:INT1 2 "gpc_reg_operand" "") ;; input
+ (match_operand:SI 3 "const_int_operand" "")] ;; model
"TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
{
- rs6000_split_atomic_op (<CODE>, operands[1], operands[2],
- operands[0], NULL_RTX, operands[3]);
+ rs6000_expand_atomic_exchange (operands);
DONE;
})
-(define_expand "sync_old_nand<mode>"
- [(parallel [(set (match_operand:INT1 0 "gpc_reg_operand" "")
- (match_operand:INT1 1 "memory_operand" ""))
- (set (match_dup 1)
- (unspec:INT1
- [(ior:INT1 (not:INT1 (match_dup 1))
- (not:INT1 (match_operand:INT1 2 "gpc_reg_operand" "")))]
- UNSPEC_ATOMIC))
- (clobber (scratch:INT1))
- (clobber (scratch:CC))])]
- "TARGET_POWERPC"
- "
-{
- if (<MODE>mode != SImode && <MODE>mode != DImode)
- {
- FAIL;
- if (PPC405_ERRATUM77)
- FAIL;
- rs6000_emit_sync (NOT, <MODE>mode, operands[1], operands[2],
- operands[0], NULL_RTX, true);
- DONE;
- }
-}")
-
-(define_insn_and_split "*sync_old_nand<mode>_internal"
- [(set (match_operand:GPR 0 "gpc_reg_operand" "=&r")
- (match_operand:GPR 1 "memory_operand" "+Z"))
- (set (match_dup 1)
- (unspec:GPR
- [(ior:GPR (not:GPR (match_dup 1))
- (not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r")))]
- UNSPEC_ATOMIC))
- (clobber (match_scratch:GPR 3 "=&r"))
- (clobber (match_scratch:CC 4 "=&x"))]
+(define_expand "atomic_<fetchop_name><mode>"
+ [(match_operand:INT1 0 "memory_operand" "") ;; memory
+ (FETCHOP:INT1 (match_dup 0)
+ (match_operand:INT1 1 "<fetchop_pred>" "")) ;; operand
+ (match_operand:SI 2 "const_int_operand" "")] ;; model
"TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
{
- rs6000_split_atomic_op (NOT, operands[1], operands[2],
- operands[0], NULL_RTX, operands[3]);
+ rs6000_expand_atomic_op (<CODE>, operands[0], operands[1],
+ NULL_RTX, NULL_RTX, operands[2]);
DONE;
})
-(define_expand "sync_new_<fetchop_name><mode>"
- [(parallel [(set (match_operand:INT1 0 "gpc_reg_operand" "")
- (FETCHOP:INT1
- (match_operand:INT1 1 "memory_operand" "")
- (match_operand:INT1 2 "<fetchop_pred>" "")))
- (set (match_dup 1)
- (unspec:INT1
- [(FETCHOP:INT1 (match_dup 1) (match_dup 2))]
- UNSPEC_ATOMIC))
- (clobber (scratch:INT1))
- (clobber (scratch:CC))])]
+(define_expand "atomic_nand<mode>"
+ [(match_operand:INT1 0 "memory_operand" "") ;; memory
+ (match_operand:INT1 1 "gpc_reg_operand" "") ;; operand
+ (match_operand:SI 2 "const_int_operand" "")] ;; model
"TARGET_POWERPC"
- "
{
- if (<MODE>mode != SImode && <MODE>mode != DImode)
- {
- if (PPC405_ERRATUM77)
- FAIL;
- rs6000_emit_sync (<CODE>, <MODE>mode, operands[1], operands[2],
- NULL_RTX, operands[0], true);
- DONE;
- }
-}")
-
-(define_insn_and_split "*sync_new_<fetchop_name>si_internal"
- [(set (match_operand:SI 0 "gpc_reg_operand" "=&r")
- (FETCHOP:SI
- (match_operand:SI 1 "memory_operand" "+Z")
- (match_operand:SI 2 "<fetchop_pred>" "<fetchopsi_constr>")))
- (set (match_dup 1)
- (unspec:SI
- [(FETCHOP:SI (match_dup 1) (match_dup 2))]
- UNSPEC_ATOMIC))
- (clobber (match_scratch:SI 3 "=&b"))
- (clobber (match_scratch:CC 4 "=&x"))]
- "TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
-{
- rs6000_split_atomic_op (<CODE>, operands[1], operands[2],
- NULL_RTX, operands[0], operands[3]);
+ rs6000_expand_atomic_op (NOT, operands[0], operands[1],
+ NULL_RTX, NULL_RTX, operands[2]);
DONE;
})
-(define_insn_and_split "*sync_new_<fetchop_name>di_internal"
- [(set (match_operand:DI 0 "gpc_reg_operand" "=&r")
- (FETCHOP:DI
- (match_operand:DI 1 "memory_operand" "+Z")
- (match_operand:DI 2 "<fetchop_pred>" "<fetchopdi_constr>")))
- (set (match_dup 1)
- (unspec:DI
- [(FETCHOP:DI (match_dup 1) (match_dup 2))]
- UNSPEC_ATOMIC))
- (clobber (match_scratch:DI 3 "=&b"))
- (clobber (match_scratch:CC 4 "=&x"))]
+(define_expand "atomic_fetch_<fetchop_name><mode>"
+ [(match_operand:INT1 0 "gpc_reg_operand" "") ;; output
+ (match_operand:INT1 1 "memory_operand" "") ;; memory
+ (FETCHOP:INT1 (match_dup 1)
+ (match_operand:INT1 2 "<fetchop_pred>" "")) ;; operand
+ (match_operand:SI 3 "const_int_operand" "")] ;; model
"TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
-{
- rs6000_split_atomic_op (<CODE>, operands[1], operands[2],
- NULL_RTX, operands[0], operands[3]);
+{
+ rs6000_expand_atomic_op (<CODE>, operands[1], operands[2],
+ operands[0], NULL_RTX, operands[3]);
DONE;
})
-(define_expand "sync_new_nand<mode>"
- [(parallel [(set (match_operand:INT1 0 "gpc_reg_operand" "")
- (ior:INT1
- (not:INT1 (match_operand:INT1 1 "memory_operand" ""))
- (not:INT1 (match_operand:INT1 2 "gpc_reg_operand" ""))))
- (set (match_dup 1)
- (unspec:INT1
- [(ior:INT1 (not:INT1 (match_dup 1))
- (not:INT1 (match_dup 2)))]
- UNSPEC_ATOMIC))
- (clobber (scratch:INT1))
- (clobber (scratch:CC))])]
+(define_expand "atomic_fetch_nand<mode>"
+ [(match_operand:INT1 0 "gpc_reg_operand" "") ;; output
+ (match_operand:INT1 1 "memory_operand" "") ;; memory
+ (match_operand:INT1 2 "gpc_reg_operand" "") ;; operand
+ (match_operand:SI 3 "const_int_operand" "")] ;; model
"TARGET_POWERPC"
- "
{
- if (<MODE>mode != SImode && <MODE>mode != DImode)
- {
- FAIL;
- if (PPC405_ERRATUM77)
- FAIL;
- rs6000_emit_sync (NOT, <MODE>mode, operands[1], operands[2],
- NULL_RTX, operands[0], true);
- DONE;
- }
-}")
+ rs6000_expand_atomic_op (NOT, operands[1], operands[2],
+ operands[0], NULL_RTX, operands[3]);
+ DONE;
+})
-(define_insn_and_split "*sync_new_nand<mode>_internal"
- [(set (match_operand:GPR 0 "gpc_reg_operand" "=&r")
- (ior:GPR
- (not:GPR (match_operand:GPR 1 "memory_operand" "+Z"))
- (not:GPR (match_operand:GPR 2 "gpc_reg_operand" "r"))))
- (set (match_dup 1)
- (unspec:GPR
- [(ior:GPR (not:GPR (match_dup 1)) (not:GPR (match_dup 2)))]
- UNSPEC_ATOMIC))
- (clobber (match_scratch:GPR 3 "=&r"))
- (clobber (match_scratch:CC 4 "=&x"))]
+(define_expand "atomic_<fetchop_name>_fetch<mode>"
+ [(match_operand:INT1 0 "gpc_reg_operand" "") ;; output
+ (match_operand:INT1 1 "memory_operand" "") ;; memory
+ (FETCHOP:INT1 (match_dup 1)
+ (match_operand:INT1 2 "<fetchop_pred>" "")) ;; operand
+ (match_operand:SI 3 "const_int_operand" "")] ;; model
"TARGET_POWERPC"
- "#"
- "&& reload_completed"
- [(const_int 0)]
{
- rs6000_split_atomic_op (NOT, operands[1], operands[2],
- NULL_RTX, operands[0], operands[3]);
+ rs6000_expand_atomic_op (<CODE>, operands[1], operands[2],
+ NULL_RTX, operands[0], operands[3]);
DONE;
})
-; and<mode> without cr0 clobber to avoid generation of additional clobber
-; in atomic splitters causing internal consistency failure.
-; cr0 already clobbered by larx/stcx.
-(define_insn "*atomic_andsi"
- [(set (match_operand:SI 0 "gpc_reg_operand" "=r,r,r,r")
- (unspec:SI [(match_operand:SI 1 "gpc_reg_operand" "%r,r,r,r")
- (match_operand:SI 2 "and_operand" "?r,T,K,L")]
- UNSPEC_AND))]
- ""
- "@
- and %0,%1,%2
- {rlinm|rlwinm} %0,%1,0,%m2,%M2
- {andil.|andi.} %0,%1,%b2
- {andiu.|andis.} %0,%1,%u2"
- [(set_attr "type" "*,*,compare,compare")])
-
-(define_insn "*atomic_anddi"
- [(set (match_operand:DI 0 "gpc_reg_operand" "=r,r,r,r,r")
- (unspec:DI [(match_operand:DI 1 "gpc_reg_operand" "%r,r,r,r,r")
- (match_operand:DI 2 "and_operand" "?r,S,T,K,J")]
- UNSPEC_AND))]
- "TARGET_POWERPC64"
- "@
- and %0,%1,%2
- rldic%B2 %0,%1,0,%S2
- rlwinm %0,%1,0,%m2,%M2
- andi. %0,%1,%b2
- andis. %0,%1,%u2"
- [(set_attr "type" "*,*,*,compare,compare")
- (set_attr "length" "4,4,4,4,4")])
-
-; the sync_*_internal patterns all have these operands:
-; 0 - memory location
-; 1 - operand
-; 2 - value in memory after operation
-; 3 - value in memory immediately before operation
-
-(define_insn "*sync_addshort_internal"
- [(set (match_operand:SI 2 "gpc_reg_operand" "=&r")
- (ior:SI (and:SI (plus:SI (match_operand:SI 0 "memory_operand" "+Z")
- (match_operand:SI 1 "add_operand" "rI"))
- (match_operand:SI 4 "gpc_reg_operand" "r"))
- (and:SI (not:SI (match_dup 4)) (match_dup 0))))
- (set (match_operand:SI 3 "gpc_reg_operand" "=&b") (match_dup 0))
- (set (match_dup 0)
- (unspec:SI [(ior:SI (and:SI (plus:SI (match_dup 0) (match_dup 1))
- (match_dup 4))
- (and:SI (not:SI (match_dup 4)) (match_dup 0)))]
- UNSPEC_SYNC_OP))
- (clobber (match_scratch:CC 5 "=&x"))
- (clobber (match_scratch:SI 6 "=&r"))]
- "TARGET_POWERPC && !PPC405_ERRATUM77"
- "lwarx %3,%y0\n\tadd%I1 %2,%3,%1\n\tandc %6,%3,%4\n\tand %2,%2,%4\n\tor %2,%2,%6\n\tstwcx. %2,%y0\n\tbne- $-24"
- [(set_attr "length" "28")])
-
-(define_insn "*sync_subshort_internal"
- [(set (match_operand:SI 2 "gpc_reg_operand" "=&r")
- (ior:SI (and:SI (minus:SI (match_operand:SI 0 "memory_operand" "+Z")
- (match_operand:SI 1 "add_operand" "rI"))
- (match_operand:SI 4 "gpc_reg_operand" "r"))
- (and:SI (not:SI (match_dup 4)) (match_dup 0))))
- (set (match_operand:SI 3 "gpc_reg_operand" "=&b") (match_dup 0))
- (set (match_dup 0)
- (unspec:SI [(ior:SI (and:SI (minus:SI (match_dup 0) (match_dup 1))
- (match_dup 4))
- (and:SI (not:SI (match_dup 4)) (match_dup 0)))]
- UNSPEC_SYNC_OP))
- (clobber (match_scratch:CC 5 "=&x"))
- (clobber (match_scratch:SI 6 "=&r"))]
- "TARGET_POWERPC && !PPC405_ERRATUM77"
- "lwarx %3,%y0\n\tsubf %2,%1,%3\n\tandc %6,%3,%4\n\tand %2,%2,%4\n\tor %2,%2,%6\n\tstwcx. %2,%y0\n\tbne- $-24"
- [(set_attr "length" "28")])
-
-(define_insn "*sync_andsi_internal"
- [(set (match_operand:SI 2 "gpc_reg_operand" "=&r,&r,&r,&r")
- (and:SI (match_operand:SI 0 "memory_operand" "+Z,Z,Z,Z")
- (match_operand:SI 1 "and_operand" "r,T,K,L")))
- (set (match_operand:SI 3 "gpc_reg_operand" "=&b,&b,&b,&b") (match_dup 0))
- (set (match_dup 0)
- (unspec:SI [(and:SI (match_dup 0) (match_dup 1))]
- UNSPEC_SYNC_OP))
- (clobber (match_scratch:CC 4 "=&x,&x,&x,&x"))]
- "TARGET_POWERPC && !PPC405_ERRATUM77"
- "@
- lwarx %3,%y0\n\tand %2,%3,%1\n\tstwcx. %2,%y0\n\tbne- $-12
- lwarx %3,%y0\n\trlwinm %2,%3,0,%m1,%M1\n\tstwcx. %2,%y0\n\tbne- $-12
- lwarx %3,%y0\n\tandi. %2,%3,%b1\n\tstwcx. %2,%y0\n\tbne- $-12
- lwarx %3,%y0\n\tandis. %2,%3,%u1\n\tstwcx. %2,%y0\n\tbne- $-12"
- [(set_attr "length" "16,16,16,16")])
-
-(define_insn "*sync_boolsi_internal"
- [(set (match_operand:SI 2 "gpc_reg_operand" "=&r,&r,&r")
- (match_operator:SI 4 "boolean_or_operator"
- [(match_operand:SI 0 "memory_operand" "+Z,Z,Z")
- (match_operand:SI 1 "logical_operand" "r,K,L")]))
- (set (match_operand:SI 3 "gpc_reg_operand" "=&b,&b,&b") (match_dup 0))
- (set (match_dup 0) (unspec:SI [(match_dup 4)] UNSPEC_SYNC_OP))
- (clobber (match_scratch:CC 5 "=&x,&x,&x"))]
- "TARGET_POWERPC && !PPC405_ERRATUM77"
- "@
- lwarx %3,%y0\n\t%q4 %2,%3,%1\n\tstwcx. %2,%y0\n\tbne- $-12
- lwarx %3,%y0\n\t%q4i %2,%3,%b1\n\tstwcx. %2,%y0\n\tbne- $-12
- lwarx %3,%y0\n\t%q4is %2,%3,%u1\n\tstwcx. %2,%y0\n\tbne- $-12"
- [(set_attr "length" "16,16,16")])
-
-; This pattern could also take immediate values of operand 1,
-; since the non-NOT version of the operator is used; but this is not
-; very useful, since in practice operand 1 is a full 32-bit value.
-; Likewise, operand 5 is in practice either <= 2^16 or it is a register.
-(define_insn "*sync_boolcshort_internal"
- [(set (match_operand:SI 2 "gpc_reg_operand" "=&r")
- (match_operator:SI 4 "boolean_or_operator"
- [(xor:SI (not:SI (match_operand:SI 0 "memory_operand" "+Z"))
- (not:SI (match_operand:SI 5 "logical_operand" "rK")))
- (match_operand:SI 1 "gpc_reg_operand" "r")]))
- (set (match_operand:SI 3 "gpc_reg_operand" "=&b") (match_dup 0))
- (set (match_dup 0) (unspec:SI [(match_dup 4)] UNSPEC_SYNC_OP))
- (clobber (match_scratch:CC 6 "=&x"))]
- "TARGET_POWERPC && !PPC405_ERRATUM77"
- "lwarx %3,%y0\n\txor%I2 %2,%3,%5\n\t%q4 %2,%2,%1\n\tstwcx. %2,%y0\n\tbne- $-16"
- [(set_attr "length" "20")])
-
-(define_insn "isync"
- [(set (mem:BLK (match_scratch 0 "X"))
- (unspec_volatile:BLK [(mem:BLK (match_scratch 1 "X"))] UNSPECV_ISYNC))]
- ""
- "{ics|isync}"
- [(set_attr "type" "isync")])
-
-(define_expand "sync_lock_release<mode>"
- [(set (match_operand:INT 0 "memory_operand")
- (match_operand:INT 1 "any_operand"))]
- ""
- "
+(define_expand "atomic_nand_fetch<mode>"
+ [(match_operand:INT1 0 "gpc_reg_operand" "") ;; output
+ (match_operand:INT1 1 "memory_operand" "") ;; memory
+ (match_operand:INT1 2 "gpc_reg_operand" "") ;; operand
+ (match_operand:SI 3 "const_int_operand" "")] ;; model
+ "TARGET_POWERPC"
{
- emit_insn (gen_lwsync ());
- emit_move_insn (operands[0], operands[1]);
+ rs6000_expand_atomic_op (NOT, operands[1], operands[2],
+ NULL_RTX, operands[0], operands[3]);
DONE;
-}")
-
-; Some AIX assemblers don't accept lwsync, so we use a .long.
-(define_insn "lwsync"
- [(set (mem:BLK (match_scratch 0 "X"))
- (unspec_volatile:BLK [(mem:BLK (match_scratch 1 "X"))] UNSPECV_LWSYNC))]
- ""
-{
- if (TARGET_NO_LWSYNC)
- return "sync";
- else
- return (TARGET_LWSYNC_INSTRUCTION) ? "lwsync" : ".long 0x7c2004ac";
-}
- [(set_attr "type" "sync")])
-
+})
projects / gcc.git / commitdiff