1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2020 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 is properly defined. */
40 #include "stor-layout.h"
45 #include "optabs-tree.h"
47 #include "internal-fn.h"
48 #include "langhooks.h"
50 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
52 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
53 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
55 /* Debug facility for use in GDB. */
56 void debug_optab_libfuncs (void);
58 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
59 the result of operation CODE applied to OP0 (and OP1 if it is a binary
60 operation). OP0_MODE is OP0's mode.
62 If the last insn does not set TARGET, don't do anything, but return 1.
64 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
65 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
66 try again, ensuring that TARGET is not one of the operands. */
69 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
,
70 rtx op1
, machine_mode op0_mode
)
76 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
78 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
79 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
80 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
81 && GET_RTX_CLASS (code
) != RTX_COMPARE
82 && GET_RTX_CLASS (code
) != RTX_UNARY
)
85 if (GET_CODE (target
) == ZERO_EXTRACT
)
88 for (last_insn
= insns
;
89 NEXT_INSN (last_insn
) != NULL_RTX
;
90 last_insn
= NEXT_INSN (last_insn
))
93 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
94 a value changing in the insn, so the note would be invalid for CSE. */
95 if (reg_overlap_mentioned_p (target
, op0
)
96 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
99 && (rtx_equal_p (target
, op0
)
100 || (op1
&& rtx_equal_p (target
, op1
))))
102 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
103 over expanding it as temp = MEM op X, MEM = temp. If the target
104 supports MEM = MEM op X instructions, it is sometimes too hard
105 to reconstruct that form later, especially if X is also a memory,
106 and due to multiple occurrences of addresses the address might
107 be forced into register unnecessarily.
108 Note that not emitting the REG_EQUIV note might inhibit
109 CSE in some cases. */
110 set
= single_set (last_insn
);
112 && GET_CODE (SET_SRC (set
)) == code
113 && MEM_P (SET_DEST (set
))
114 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
115 || (op1
&& rtx_equal_p (SET_DEST (set
),
116 XEXP (SET_SRC (set
), 1)))))
122 set
= set_for_reg_notes (last_insn
);
126 if (! rtx_equal_p (SET_DEST (set
), target
)
127 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
128 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
129 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
132 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
142 if (op0_mode
!= VOIDmode
&& GET_MODE (target
) != op0_mode
)
144 note
= gen_rtx_fmt_e (code
, op0_mode
, copy_rtx (op0
));
145 if (GET_MODE_UNIT_SIZE (op0_mode
)
146 > GET_MODE_UNIT_SIZE (GET_MODE (target
)))
147 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
150 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
156 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
160 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
162 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
167 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
168 for a widening operation would be. In most cases this would be OP0, but if
169 that's a constant it'll be VOIDmode, which isn't useful. */
172 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
174 machine_mode m0
= GET_MODE (op0
);
175 machine_mode m1
= GET_MODE (op1
);
178 if (m0
== VOIDmode
&& m1
== VOIDmode
)
180 else if (m0
== VOIDmode
|| GET_MODE_UNIT_SIZE (m0
) < GET_MODE_UNIT_SIZE (m1
))
185 if (GET_MODE_UNIT_SIZE (result
) > GET_MODE_UNIT_SIZE (to_mode
))
191 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
192 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
193 not actually do a sign-extend or zero-extend, but can leave the
194 higher-order bits of the result rtx undefined, for example, in the case
195 of logical operations, but not right shifts. */
198 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
199 int unsignedp
, int no_extend
)
202 scalar_int_mode int_mode
;
204 /* If we don't have to extend and this is a constant, return it. */
205 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
208 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
209 extend since it will be more efficient to do so unless the signedness of
210 a promoted object differs from our extension. */
212 || !is_a
<scalar_int_mode
> (mode
, &int_mode
)
213 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
214 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
215 return convert_modes (mode
, oldmode
, op
, unsignedp
);
217 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
219 if (GET_MODE_SIZE (int_mode
) <= UNITS_PER_WORD
)
220 return gen_lowpart (int_mode
, force_reg (GET_MODE (op
), op
));
222 /* Otherwise, get an object of MODE, clobber it, and set the low-order
225 result
= gen_reg_rtx (int_mode
);
226 emit_clobber (result
);
227 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
231 /* Expand vector widening operations.
233 There are two different classes of operations handled here:
234 1) Operations whose result is wider than all the arguments to the operation.
235 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
236 In this case OP0 and optionally OP1 would be initialized,
237 but WIDE_OP wouldn't (not relevant for this case).
238 2) Operations whose result is of the same size as the last argument to the
239 operation, but wider than all the other arguments to the operation.
240 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
241 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
243 E.g, when called to expand the following operations, this is how
244 the arguments will be initialized:
246 widening-sum 2 oprnd0 - oprnd1
247 widening-dot-product 3 oprnd0 oprnd1 oprnd2
248 widening-mult 2 oprnd0 oprnd1 -
249 type-promotion (vec-unpack) 1 oprnd0 - - */
252 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
253 rtx target
, int unsignedp
)
255 class expand_operand eops
[4];
256 tree oprnd0
, oprnd1
, oprnd2
;
257 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
258 optab widen_pattern_optab
;
259 enum insn_code icode
;
260 int nops
= TREE_CODE_LENGTH (ops
->code
);
265 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
266 if (ops
->code
== VEC_UNPACK_FIX_TRUNC_HI_EXPR
267 || ops
->code
== VEC_UNPACK_FIX_TRUNC_LO_EXPR
)
268 /* The sign is from the result type rather than operand's type
271 = optab_for_tree_code (ops
->code
, ops
->type
, optab_default
);
272 else if ((ops
->code
== VEC_UNPACK_HI_EXPR
273 || ops
->code
== VEC_UNPACK_LO_EXPR
)
274 && VECTOR_BOOLEAN_TYPE_P (ops
->type
)
275 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0
))
276 && TYPE_MODE (ops
->type
) == TYPE_MODE (TREE_TYPE (oprnd0
))
277 && SCALAR_INT_MODE_P (TYPE_MODE (ops
->type
)))
279 /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
280 the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
281 vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
282 the pattern number of elements in the wider vector. */
284 = (ops
->code
== VEC_UNPACK_HI_EXPR
285 ? vec_unpacks_sbool_hi_optab
: vec_unpacks_sbool_lo_optab
);
290 = optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
291 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
292 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
293 icode
= find_widening_optab_handler (widen_pattern_optab
,
294 TYPE_MODE (TREE_TYPE (ops
->op2
)),
297 icode
= optab_handler (widen_pattern_optab
, tmode0
);
298 gcc_assert (icode
!= CODE_FOR_nothing
);
303 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
308 op1
= GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0
)).to_constant ());
312 /* The last operand is of a wider mode than the rest of the operands. */
317 gcc_assert (tmode1
== tmode0
);
320 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
324 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
325 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
327 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
329 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
330 expand_insn (icode
, op
, eops
);
331 return eops
[0].value
;
334 /* Generate code to perform an operation specified by TERNARY_OPTAB
335 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
337 UNSIGNEDP is for the case where we have to widen the operands
338 to perform the operation. It says to use zero-extension.
340 If TARGET is nonzero, the value
341 is generated there, if it is convenient to do so.
342 In all cases an rtx is returned for the locus of the value;
343 this may or may not be TARGET. */
346 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
347 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
349 class expand_operand ops
[4];
350 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
352 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
354 create_output_operand (&ops
[0], target
, mode
);
355 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
356 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
357 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
358 expand_insn (icode
, 4, ops
);
363 /* Like expand_binop, but return a constant rtx if the result can be
364 calculated at compile time. The arguments and return value are
365 otherwise the same as for expand_binop. */
368 simplify_expand_binop (machine_mode mode
, optab binoptab
,
369 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
370 enum optab_methods methods
)
372 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
374 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
380 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
383 /* Like simplify_expand_binop, but always put the result in TARGET.
384 Return true if the expansion succeeded. */
387 force_expand_binop (machine_mode mode
, optab binoptab
,
388 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
389 enum optab_methods methods
)
391 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
392 target
, unsignedp
, methods
);
396 emit_move_insn (target
, x
);
400 /* Create a new vector value in VMODE with all elements set to OP. The
401 mode of OP must be the element mode of VMODE. If OP is a constant,
402 then the return value will be a constant. */
405 expand_vector_broadcast (machine_mode vmode
, rtx op
)
410 gcc_checking_assert (VECTOR_MODE_P (vmode
));
412 if (valid_for_const_vector_p (vmode
, op
))
413 return gen_const_vec_duplicate (vmode
, op
);
415 insn_code icode
= optab_handler (vec_duplicate_optab
, vmode
);
416 if (icode
!= CODE_FOR_nothing
)
418 class expand_operand ops
[2];
419 create_output_operand (&ops
[0], NULL_RTX
, vmode
);
420 create_input_operand (&ops
[1], op
, GET_MODE (op
));
421 expand_insn (icode
, 2, ops
);
425 if (!GET_MODE_NUNITS (vmode
).is_constant (&n
))
428 /* ??? If the target doesn't have a vec_init, then we have no easy way
429 of performing this operation. Most of this sort of generic support
430 is hidden away in the vector lowering support in gimple. */
431 icode
= convert_optab_handler (vec_init_optab
, vmode
,
432 GET_MODE_INNER (vmode
));
433 if (icode
== CODE_FOR_nothing
)
436 vec
= rtvec_alloc (n
);
437 for (int i
= 0; i
< n
; ++i
)
438 RTVEC_ELT (vec
, i
) = op
;
439 rtx ret
= gen_reg_rtx (vmode
);
440 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
445 /* This subroutine of expand_doubleword_shift handles the cases in which
446 the effective shift value is >= BITS_PER_WORD. The arguments and return
447 value are the same as for the parent routine, except that SUPERWORD_OP1
448 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
449 INTO_TARGET may be null if the caller has decided to calculate it. */
452 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
453 rtx outof_target
, rtx into_target
,
454 int unsignedp
, enum optab_methods methods
)
456 if (into_target
!= 0)
457 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
458 into_target
, unsignedp
, methods
))
461 if (outof_target
!= 0)
463 /* For a signed right shift, we must fill OUTOF_TARGET with copies
464 of the sign bit, otherwise we must fill it with zeros. */
465 if (binoptab
!= ashr_optab
)
466 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
468 if (!force_expand_binop (word_mode
, binoptab
, outof_input
,
469 gen_int_shift_amount (word_mode
,
471 outof_target
, unsignedp
, methods
))
477 /* This subroutine of expand_doubleword_shift handles the cases in which
478 the effective shift value is < BITS_PER_WORD. The arguments and return
479 value are the same as for the parent routine. */
482 expand_subword_shift (scalar_int_mode op1_mode
, optab binoptab
,
483 rtx outof_input
, rtx into_input
, rtx op1
,
484 rtx outof_target
, rtx into_target
,
485 int unsignedp
, enum optab_methods methods
,
486 unsigned HOST_WIDE_INT shift_mask
)
488 optab reverse_unsigned_shift
, unsigned_shift
;
491 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
492 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
494 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
495 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
496 the opposite direction to BINOPTAB. */
497 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
499 carries
= outof_input
;
500 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
501 op1_mode
), op1_mode
);
502 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
507 /* We must avoid shifting by BITS_PER_WORD bits since that is either
508 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
509 has unknown behavior. Do a single shift first, then shift by the
510 remainder. It's OK to use ~OP1 as the remainder if shift counts
511 are truncated to the mode size. */
512 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
513 outof_input
, const1_rtx
, 0, unsignedp
, methods
);
514 if (shift_mask
== BITS_PER_WORD
- 1)
516 tmp
= immed_wide_int_const
517 (wi::minus_one (GET_MODE_PRECISION (op1_mode
)), op1_mode
);
518 tmp
= simplify_expand_binop (op1_mode
, xor_optab
, op1
, tmp
,
523 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
524 op1_mode
), op1_mode
);
525 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
529 if (tmp
== 0 || carries
== 0)
531 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
532 carries
, tmp
, 0, unsignedp
, methods
);
536 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
537 so the result can go directly into INTO_TARGET if convenient. */
538 tmp
= expand_binop (word_mode
, unsigned_shift
, into_input
, op1
,
539 into_target
, unsignedp
, methods
);
543 /* Now OR in the bits carried over from OUTOF_INPUT. */
544 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
545 into_target
, unsignedp
, methods
))
548 /* Use a standard word_mode shift for the out-of half. */
549 if (outof_target
!= 0)
550 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
551 outof_target
, unsignedp
, methods
))
558 /* Try implementing expand_doubleword_shift using conditional moves.
559 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
560 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
561 are the shift counts to use in the former and latter case. All other
562 arguments are the same as the parent routine. */
565 expand_doubleword_shift_condmove (scalar_int_mode op1_mode
, optab binoptab
,
566 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
567 rtx outof_input
, rtx into_input
,
568 rtx subword_op1
, rtx superword_op1
,
569 rtx outof_target
, rtx into_target
,
570 int unsignedp
, enum optab_methods methods
,
571 unsigned HOST_WIDE_INT shift_mask
)
573 rtx outof_superword
, into_superword
;
575 /* Put the superword version of the output into OUTOF_SUPERWORD and
577 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
578 if (outof_target
!= 0 && subword_op1
== superword_op1
)
580 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
581 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
582 into_superword
= outof_target
;
583 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
584 outof_superword
, 0, unsignedp
, methods
))
589 into_superword
= gen_reg_rtx (word_mode
);
590 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
591 outof_superword
, into_superword
,
596 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
597 if (!expand_subword_shift (op1_mode
, binoptab
,
598 outof_input
, into_input
, subword_op1
,
599 outof_target
, into_target
,
600 unsignedp
, methods
, shift_mask
))
603 /* Select between them. Do the INTO half first because INTO_SUPERWORD
604 might be the current value of OUTOF_TARGET. */
605 if (!emit_conditional_move (into_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
606 into_target
, into_superword
, word_mode
, false))
609 if (outof_target
!= 0)
610 if (!emit_conditional_move (outof_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
611 outof_target
, outof_superword
,
618 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
619 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
620 input operand; the shift moves bits in the direction OUTOF_INPUT->
621 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
622 of the target. OP1 is the shift count and OP1_MODE is its mode.
623 If OP1 is constant, it will have been truncated as appropriate
624 and is known to be nonzero.
626 If SHIFT_MASK is zero, the result of word shifts is undefined when the
627 shift count is outside the range [0, BITS_PER_WORD). This routine must
628 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
630 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
631 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
632 fill with zeros or sign bits as appropriate.
634 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
635 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
636 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
637 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
640 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
641 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
642 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
643 function wants to calculate it itself.
645 Return true if the shift could be successfully synthesized. */
648 expand_doubleword_shift (scalar_int_mode op1_mode
, optab binoptab
,
649 rtx outof_input
, rtx into_input
, rtx op1
,
650 rtx outof_target
, rtx into_target
,
651 int unsignedp
, enum optab_methods methods
,
652 unsigned HOST_WIDE_INT shift_mask
)
654 rtx superword_op1
, tmp
, cmp1
, cmp2
;
655 enum rtx_code cmp_code
;
657 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
658 fill the result with sign or zero bits as appropriate. If so, the value
659 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
660 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
661 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
663 This isn't worthwhile for constant shifts since the optimizers will
664 cope better with in-range shift counts. */
665 if (shift_mask
>= BITS_PER_WORD
667 && !CONSTANT_P (op1
))
669 if (!expand_doubleword_shift (op1_mode
, binoptab
,
670 outof_input
, into_input
, op1
,
672 unsignedp
, methods
, shift_mask
))
674 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
675 outof_target
, unsignedp
, methods
))
680 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
681 is true when the effective shift value is less than BITS_PER_WORD.
682 Set SUPERWORD_OP1 to the shift count that should be used to shift
683 OUTOF_INPUT into INTO_TARGET when the condition is false. */
684 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
685 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
687 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
688 is a subword shift count. */
689 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
691 cmp2
= CONST0_RTX (op1_mode
);
697 /* Set CMP1 to OP1 - BITS_PER_WORD. */
698 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
700 cmp2
= CONST0_RTX (op1_mode
);
702 superword_op1
= cmp1
;
707 /* If we can compute the condition at compile time, pick the
708 appropriate subroutine. */
709 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
710 if (tmp
!= 0 && CONST_INT_P (tmp
))
712 if (tmp
== const0_rtx
)
713 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
714 outof_target
, into_target
,
717 return expand_subword_shift (op1_mode
, binoptab
,
718 outof_input
, into_input
, op1
,
719 outof_target
, into_target
,
720 unsignedp
, methods
, shift_mask
);
723 /* Try using conditional moves to generate straight-line code. */
724 if (HAVE_conditional_move
)
726 rtx_insn
*start
= get_last_insn ();
727 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
728 cmp_code
, cmp1
, cmp2
,
729 outof_input
, into_input
,
731 outof_target
, into_target
,
732 unsignedp
, methods
, shift_mask
))
734 delete_insns_since (start
);
737 /* As a last resort, use branches to select the correct alternative. */
738 rtx_code_label
*subword_label
= gen_label_rtx ();
739 rtx_code_label
*done_label
= gen_label_rtx ();
742 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
744 profile_probability::uninitialized ());
747 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
748 outof_target
, into_target
,
752 emit_jump_insn (targetm
.gen_jump (done_label
));
754 emit_label (subword_label
);
756 if (!expand_subword_shift (op1_mode
, binoptab
,
757 outof_input
, into_input
, op1
,
758 outof_target
, into_target
,
759 unsignedp
, methods
, shift_mask
))
762 emit_label (done_label
);
766 /* Subroutine of expand_binop. Perform a double word multiplication of
767 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
768 as the target's word_mode. This function return NULL_RTX if anything
769 goes wrong, in which case it may have already emitted instructions
770 which need to be deleted.
772 If we want to multiply two two-word values and have normal and widening
773 multiplies of single-word values, we can do this with three smaller
776 The multiplication proceeds as follows:
777 _______________________
778 [__op0_high_|__op0_low__]
779 _______________________
780 * [__op1_high_|__op1_low__]
781 _______________________________________________
782 _______________________
783 (1) [__op0_low__*__op1_low__]
784 _______________________
785 (2a) [__op0_low__*__op1_high_]
786 _______________________
787 (2b) [__op0_high_*__op1_low__]
788 _______________________
789 (3) [__op0_high_*__op1_high_]
792 This gives a 4-word result. Since we are only interested in the
793 lower 2 words, partial result (3) and the upper words of (2a) and
794 (2b) don't need to be calculated. Hence (2a) and (2b) can be
795 calculated using non-widening multiplication.
797 (1), however, needs to be calculated with an unsigned widening
798 multiplication. If this operation is not directly supported we
799 try using a signed widening multiplication and adjust the result.
800 This adjustment works as follows:
802 If both operands are positive then no adjustment is needed.
804 If the operands have different signs, for example op0_low < 0 and
805 op1_low >= 0, the instruction treats the most significant bit of
806 op0_low as a sign bit instead of a bit with significance
807 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
808 with 2**BITS_PER_WORD - op0_low, and two's complements the
809 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
812 Similarly, if both operands are negative, we need to add
813 (op0_low + op1_low) * 2**BITS_PER_WORD.
815 We use a trick to adjust quickly. We logically shift op0_low right
816 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
817 op0_high (op1_high) before it is used to calculate 2b (2a). If no
818 logical shift exists, we do an arithmetic right shift and subtract
822 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
823 bool umulp
, enum optab_methods methods
)
825 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
826 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
827 rtx wordm1
= (umulp
? NULL_RTX
828 : gen_int_shift_amount (word_mode
, BITS_PER_WORD
- 1));
829 rtx product
, adjust
, product_high
, temp
;
831 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
832 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
833 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
834 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
836 /* If we're using an unsigned multiply to directly compute the product
837 of the low-order words of the operands and perform any required
838 adjustments of the operands, we begin by trying two more multiplications
839 and then computing the appropriate sum.
841 We have checked above that the required addition is provided.
842 Full-word addition will normally always succeed, especially if
843 it is provided at all, so we don't worry about its failure. The
844 multiplication may well fail, however, so we do handle that. */
848 /* ??? This could be done with emit_store_flag where available. */
849 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
850 NULL_RTX
, 1, methods
);
852 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
853 NULL_RTX
, 0, OPTAB_DIRECT
);
856 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
857 NULL_RTX
, 0, methods
);
860 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
861 NULL_RTX
, 0, OPTAB_DIRECT
);
868 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
869 NULL_RTX
, 0, OPTAB_DIRECT
);
873 /* OP0_HIGH should now be dead. */
877 /* ??? This could be done with emit_store_flag where available. */
878 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
879 NULL_RTX
, 1, methods
);
881 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
882 NULL_RTX
, 0, OPTAB_DIRECT
);
885 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
886 NULL_RTX
, 0, methods
);
889 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
890 NULL_RTX
, 0, OPTAB_DIRECT
);
897 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
898 NULL_RTX
, 0, OPTAB_DIRECT
);
902 /* OP1_HIGH should now be dead. */
904 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
905 NULL_RTX
, 0, OPTAB_DIRECT
);
907 if (target
&& !REG_P (target
))
910 /* *_widen_optab needs to determine operand mode, make sure at least
911 one operand has non-VOID mode. */
912 if (GET_MODE (op0_low
) == VOIDmode
&& GET_MODE (op1_low
) == VOIDmode
)
913 op0_low
= force_reg (word_mode
, op0_low
);
916 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
917 target
, 1, OPTAB_DIRECT
);
919 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
920 target
, 1, OPTAB_DIRECT
);
925 product_high
= operand_subword (product
, high
, 1, mode
);
926 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
927 NULL_RTX
, 0, OPTAB_DIRECT
);
928 emit_move_insn (product_high
, adjust
);
932 /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
933 constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
934 (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
935 computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
936 + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
937 depends on the bit value, if 2, then carry from the addition needs to be
938 added too, i.e. like:
939 sum += __builtin_add_overflow (low, high, &sum)
941 Optimize signed double-word OP0 % OP1 similarly, just apply some correction
942 factor to the sum before doing unsigned remainder, in the form of
943 sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
944 then perform unsigned
945 remainder = sum % OP1;
947 remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
950 expand_doubleword_mod (machine_mode mode
, rtx op0
, rtx op1
, bool unsignedp
)
952 if (INTVAL (op1
) <= 1)
955 rtx_insn
*last
= get_last_insn ();
956 for (int bit
= BITS_PER_WORD
; bit
>= BITS_PER_WORD
/ 2; bit
--)
958 wide_int w
= wi::shifted_mask (bit
, 1, false, 2 * BITS_PER_WORD
);
959 if (wi::ne_p (wi::umod_trunc (w
, INTVAL (op1
)), 1))
961 rtx sum
= NULL_RTX
, mask
= NULL_RTX
;
962 if (bit
== BITS_PER_WORD
)
964 /* For signed modulo we need to add correction to the sum
965 and that might again overflow. */
968 if (optab_handler (uaddv4_optab
, word_mode
) == CODE_FOR_nothing
)
970 tree wtype
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
971 if (wtype
== NULL_TREE
)
973 tree ctype
= build_complex_type (wtype
);
974 if (TYPE_MODE (ctype
) != GET_MODE_COMPLEX_MODE (word_mode
))
976 machine_mode cmode
= TYPE_MODE (ctype
);
977 rtx op00
= operand_subword_force (op0
, 0, mode
);
978 rtx op01
= operand_subword_force (op0
, 1, mode
);
979 rtx cres
= gen_rtx_CONCAT (cmode
, gen_reg_rtx (word_mode
),
980 gen_reg_rtx (word_mode
));
981 tree lhs
= make_tree (ctype
, cres
);
982 tree arg0
= make_tree (wtype
, op00
);
983 tree arg1
= make_tree (wtype
, op01
);
984 expand_addsub_overflow (UNKNOWN_LOCATION
, PLUS_EXPR
, lhs
, arg0
,
985 arg1
, true, true, true, false, NULL
);
986 sum
= expand_simple_binop (word_mode
, PLUS
, XEXP (cres
, 0),
987 XEXP (cres
, 1), NULL_RTX
, 1,
994 /* Code below uses GEN_INT, so we need the masks to be representable
995 in HOST_WIDE_INTs. */
996 if (bit
>= HOST_BITS_PER_WIDE_INT
)
998 /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
999 overflow. Consider 64-bit -1ULL for word size 32, if we add
1000 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1001 if (bit
== BITS_PER_WORD
- 1)
1004 int count
= (2 * BITS_PER_WORD
+ bit
- 1) / bit
;
1005 rtx sum_corr
= NULL_RTX
;
1009 /* For signed modulo, compute it as unsigned modulo of
1010 sum with a correction added to it if OP0 is negative,
1011 such that the result can be computed as unsigned
1012 remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1013 w
= wi::min_value (2 * BITS_PER_WORD
, SIGNED
);
1014 wide_int wmod1
= wi::umod_trunc (w
, INTVAL (op1
));
1015 wide_int wmod2
= wi::smod_trunc (w
, INTVAL (op1
));
1016 /* wmod2 == -wmod1. */
1017 wmod2
= wmod2
+ (INTVAL (op1
) - 1);
1018 if (wi::ne_p (wmod1
, wmod2
))
1020 wide_int wcorr
= wmod2
- wmod1
;
1022 wcorr
= wcorr
+ INTVAL (op1
);
1023 /* Now verify if the count sums can't overflow, and punt
1025 w
= wi::mask (bit
, false, 2 * BITS_PER_WORD
);
1026 w
= w
* (count
- 1);
1027 w
= w
+ wi::mask (2 * BITS_PER_WORD
- (count
- 1) * bit
,
1028 false, 2 * BITS_PER_WORD
);
1030 w
= wi::lrshift (w
, BITS_PER_WORD
);
1031 if (wi::ne_p (w
, 0))
1034 mask
= operand_subword_force (op0
, WORDS_BIG_ENDIAN
? 0 : 1,
1036 mask
= expand_simple_binop (word_mode
, ASHIFTRT
, mask
,
1037 GEN_INT (BITS_PER_WORD
- 1),
1038 NULL_RTX
, 0, OPTAB_DIRECT
);
1039 if (mask
== NULL_RTX
)
1041 sum_corr
= immed_wide_int_const (wcorr
, word_mode
);
1042 sum_corr
= expand_simple_binop (word_mode
, AND
, mask
,
1043 sum_corr
, NULL_RTX
, 1,
1045 if (sum_corr
== NULL_RTX
)
1050 for (int i
= 0; i
< count
; i
++)
1054 v
= expand_simple_binop (mode
, LSHIFTRT
, v
, GEN_INT (i
* bit
),
1055 NULL_RTX
, 1, OPTAB_DIRECT
);
1058 v
= lowpart_subreg (word_mode
, v
, mode
);
1062 v
= expand_simple_binop (word_mode
, AND
, v
,
1063 GEN_INT ((HOST_WIDE_INT_1U
<< bit
)
1068 if (sum
== NULL_RTX
)
1071 sum
= expand_simple_binop (word_mode
, PLUS
, sum
, v
, NULL_RTX
,
1073 if (sum
== NULL_RTX
)
1078 sum
= expand_simple_binop (word_mode
, PLUS
, sum
, sum_corr
,
1079 NULL_RTX
, 1, OPTAB_DIRECT
);
1080 if (sum
== NULL_RTX
)
1084 rtx remainder
= expand_divmod (1, TRUNC_MOD_EXPR
, word_mode
, sum
, op1
,
1086 if (remainder
== NULL_RTX
)
1091 if (mask
== NULL_RTX
)
1093 mask
= operand_subword_force (op0
, WORDS_BIG_ENDIAN
? 0 : 1,
1095 mask
= expand_simple_binop (word_mode
, ASHIFTRT
, mask
,
1096 GEN_INT (BITS_PER_WORD
- 1),
1097 NULL_RTX
, 0, OPTAB_DIRECT
);
1098 if (mask
== NULL_RTX
)
1101 mask
= expand_simple_binop (word_mode
, AND
, mask
,
1102 GEN_INT (1 - INTVAL (op1
)),
1103 NULL_RTX
, 1, OPTAB_DIRECT
);
1104 if (mask
== NULL_RTX
)
1106 remainder
= expand_simple_binop (word_mode
, PLUS
, remainder
,
1107 mask
, NULL_RTX
, 1, OPTAB_DIRECT
);
1108 if (remainder
== NULL_RTX
)
1112 remainder
= convert_modes (mode
, word_mode
, remainder
, unsignedp
);
1113 /* Punt if we need any library calls. */
1114 for (; last
; last
= NEXT_INSN (last
))
1122 /* Wrapper around expand_binop which takes an rtx code to specify
1123 the operation to perform, not an optab pointer. All other
1124 arguments are the same. */
1126 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
1127 rtx op1
, rtx target
, int unsignedp
,
1128 enum optab_methods methods
)
1130 optab binop
= code_to_optab (code
);
1133 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
1136 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1137 binop. Order them according to commutative_operand_precedence and, if
1138 possible, try to put TARGET or a pseudo first. */
1140 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
1142 int op0_prec
= commutative_operand_precedence (op0
);
1143 int op1_prec
= commutative_operand_precedence (op1
);
1145 if (op0_prec
< op1_prec
)
1148 if (op0_prec
> op1_prec
)
1151 /* With equal precedence, both orders are ok, but it is better if the
1152 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1153 if (target
== 0 || REG_P (target
))
1154 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
1156 return rtx_equal_p (op1
, target
);
1159 /* Return true if BINOPTAB implements a shift operation. */
1162 shift_optab_p (optab binoptab
)
1164 switch (optab_to_code (binoptab
))
1180 /* Return true if BINOPTAB implements a commutative binary operation. */
1183 commutative_optab_p (optab binoptab
)
1185 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
1186 || binoptab
== smul_widen_optab
1187 || binoptab
== umul_widen_optab
1188 || binoptab
== smul_highpart_optab
1189 || binoptab
== umul_highpart_optab
);
1192 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1193 optimizing, and if the operand is a constant that costs more than
1194 1 instruction, force the constant into a register and return that
1195 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1198 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
1199 int opn
, rtx x
, bool unsignedp
)
1201 bool speed
= optimize_insn_for_speed_p ();
1203 if (mode
!= VOIDmode
1206 && (rtx_cost (x
, mode
, optab_to_code (binoptab
), opn
, speed
)
1207 > set_src_cost (x
, mode
, speed
)))
1209 if (CONST_INT_P (x
))
1211 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
1212 if (intval
!= INTVAL (x
))
1213 x
= GEN_INT (intval
);
1216 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
1217 x
= force_reg (mode
, x
);
1222 /* Helper function for expand_binop: handle the case where there
1223 is an insn ICODE that directly implements the indicated operation.
1224 Returns null if this is not possible. */
1226 expand_binop_directly (enum insn_code icode
, machine_mode mode
, optab binoptab
,
1228 rtx target
, int unsignedp
, enum optab_methods methods
,
1231 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
1232 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
1233 machine_mode mode0
, mode1
, tmp_mode
;
1234 class expand_operand ops
[3];
1237 rtx xop0
= op0
, xop1
= op1
;
1238 bool canonicalize_op1
= false;
1240 /* If it is a commutative operator and the modes would match
1241 if we would swap the operands, we can save the conversions. */
1242 commutative_p
= commutative_optab_p (binoptab
);
1244 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1245 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode0
)
1246 std::swap (xop0
, xop1
);
1248 /* If we are optimizing, force expensive constants into a register. */
1249 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1250 if (!shift_optab_p (binoptab
))
1251 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1253 /* Shifts and rotates often use a different mode for op1 from op0;
1254 for VOIDmode constants we don't know the mode, so force it
1255 to be canonicalized using convert_modes. */
1256 canonicalize_op1
= true;
1258 /* In case the insn wants input operands in modes different from
1259 those of the actual operands, convert the operands. It would
1260 seem that we don't need to convert CONST_INTs, but we do, so
1261 that they're properly zero-extended, sign-extended or truncated
1264 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1265 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1267 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1271 mode1
= ((GET_MODE (xop1
) != VOIDmode
|| canonicalize_op1
)
1272 ? GET_MODE (xop1
) : mode
);
1273 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1275 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1279 /* If operation is commutative,
1280 try to make the first operand a register.
1281 Even better, try to make it the same as the target.
1282 Also try to make the last operand a constant. */
1284 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1285 std::swap (xop0
, xop1
);
1287 /* Now, if insn's predicates don't allow our operands, put them into
1290 if (binoptab
== vec_pack_trunc_optab
1291 || binoptab
== vec_pack_usat_optab
1292 || binoptab
== vec_pack_ssat_optab
1293 || binoptab
== vec_pack_ufix_trunc_optab
1294 || binoptab
== vec_pack_sfix_trunc_optab
1295 || binoptab
== vec_packu_float_optab
1296 || binoptab
== vec_packs_float_optab
)
1298 /* The mode of the result is different then the mode of the
1300 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1301 if (VECTOR_MODE_P (mode
)
1302 && maybe_ne (GET_MODE_NUNITS (tmp_mode
), 2 * GET_MODE_NUNITS (mode
)))
1304 delete_insns_since (last
);
1311 create_output_operand (&ops
[0], target
, tmp_mode
);
1312 create_input_operand (&ops
[1], xop0
, mode0
);
1313 create_input_operand (&ops
[2], xop1
, mode1
);
1314 pat
= maybe_gen_insn (icode
, 3, ops
);
1317 /* If PAT is composed of more than one insn, try to add an appropriate
1318 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1319 operand, call expand_binop again, this time without a target. */
1320 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
1321 && ! add_equal_note (pat
, ops
[0].value
,
1322 optab_to_code (binoptab
),
1323 ops
[1].value
, ops
[2].value
, mode0
))
1325 delete_insns_since (last
);
1326 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1327 unsignedp
, methods
);
1331 return ops
[0].value
;
1333 delete_insns_since (last
);
1337 /* Generate code to perform an operation specified by BINOPTAB
1338 on operands OP0 and OP1, with result having machine-mode MODE.
1340 UNSIGNEDP is for the case where we have to widen the operands
1341 to perform the operation. It says to use zero-extension.
1343 If TARGET is nonzero, the value
1344 is generated there, if it is convenient to do so.
1345 In all cases an rtx is returned for the locus of the value;
1346 this may or may not be TARGET. */
1349 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1350 rtx target
, int unsignedp
, enum optab_methods methods
)
1352 enum optab_methods next_methods
1353 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1354 ? OPTAB_WIDEN
: methods
);
1355 enum mode_class mclass
;
1356 enum insn_code icode
;
1357 machine_mode wider_mode
;
1358 scalar_int_mode int_mode
;
1361 rtx_insn
*entry_last
= get_last_insn ();
1364 mclass
= GET_MODE_CLASS (mode
);
1366 /* If subtracting an integer constant, convert this into an addition of
1367 the negated constant. */
1369 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1371 op1
= negate_rtx (mode
, op1
);
1372 binoptab
= add_optab
;
1374 /* For shifts, constant invalid op1 might be expanded from different
1375 mode than MODE. As those are invalid, force them to a register
1376 to avoid further problems during expansion. */
1377 else if (CONST_INT_P (op1
)
1378 && shift_optab_p (binoptab
)
1379 && UINTVAL (op1
) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode
)))
1381 op1
= gen_int_mode (INTVAL (op1
), GET_MODE_INNER (mode
));
1382 op1
= force_reg (GET_MODE_INNER (mode
), op1
);
1385 /* Record where to delete back to if we backtrack. */
1386 last
= get_last_insn ();
1388 /* If we can do it with a three-operand insn, do so. */
1390 if (methods
!= OPTAB_MUST_WIDEN
)
1392 if (convert_optab_p (binoptab
))
1394 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
1395 icode
= find_widening_optab_handler (binoptab
, mode
, from_mode
);
1398 icode
= optab_handler (binoptab
, mode
);
1399 if (icode
!= CODE_FOR_nothing
)
1401 temp
= expand_binop_directly (icode
, mode
, binoptab
, op0
, op1
,
1402 target
, unsignedp
, methods
, last
);
1408 /* If we were trying to rotate, and that didn't work, try rotating
1409 the other direction before falling back to shifts and bitwise-or. */
1410 if (((binoptab
== rotl_optab
1411 && (icode
= optab_handler (rotr_optab
, mode
)) != CODE_FOR_nothing
)
1412 || (binoptab
== rotr_optab
1413 && (icode
= optab_handler (rotl_optab
, mode
)) != CODE_FOR_nothing
))
1414 && is_int_mode (mode
, &int_mode
))
1416 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1418 unsigned int bits
= GET_MODE_PRECISION (int_mode
);
1420 if (CONST_INT_P (op1
))
1421 newop1
= gen_int_shift_amount (int_mode
, bits
- INTVAL (op1
));
1422 else if (targetm
.shift_truncation_mask (int_mode
) == bits
- 1)
1423 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1425 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1426 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1427 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1429 temp
= expand_binop_directly (icode
, int_mode
, otheroptab
, op0
, newop1
,
1430 target
, unsignedp
, methods
, last
);
1435 /* If this is a multiply, see if we can do a widening operation that
1436 takes operands of this mode and makes a wider mode. */
1438 if (binoptab
== smul_optab
1439 && GET_MODE_2XWIDER_MODE (mode
).exists (&wider_mode
)
1440 && (convert_optab_handler ((unsignedp
1442 : smul_widen_optab
),
1443 wider_mode
, mode
) != CODE_FOR_nothing
))
1445 /* *_widen_optab needs to determine operand mode, make sure at least
1446 one operand has non-VOID mode. */
1447 if (GET_MODE (op0
) == VOIDmode
&& GET_MODE (op1
) == VOIDmode
)
1448 op0
= force_reg (mode
, op0
);
1449 temp
= expand_binop (wider_mode
,
1450 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1451 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1455 if (GET_MODE_CLASS (mode
) == MODE_INT
1456 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1457 return gen_lowpart (mode
, temp
);
1459 return convert_to_mode (mode
, temp
, unsignedp
);
1463 /* If this is a vector shift by a scalar, see if we can do a vector
1464 shift by a vector. If so, broadcast the scalar into a vector. */
1465 if (mclass
== MODE_VECTOR_INT
)
1467 optab otheroptab
= unknown_optab
;
1469 if (binoptab
== ashl_optab
)
1470 otheroptab
= vashl_optab
;
1471 else if (binoptab
== ashr_optab
)
1472 otheroptab
= vashr_optab
;
1473 else if (binoptab
== lshr_optab
)
1474 otheroptab
= vlshr_optab
;
1475 else if (binoptab
== rotl_optab
)
1476 otheroptab
= vrotl_optab
;
1477 else if (binoptab
== rotr_optab
)
1478 otheroptab
= vrotr_optab
;
1481 && (icode
= optab_handler (otheroptab
, mode
)) != CODE_FOR_nothing
)
1483 /* The scalar may have been extended to be too wide. Truncate
1484 it back to the proper size to fit in the broadcast vector. */
1485 scalar_mode inner_mode
= GET_MODE_INNER (mode
);
1486 if (!CONST_INT_P (op1
)
1487 && (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (op1
)))
1488 > GET_MODE_BITSIZE (inner_mode
)))
1489 op1
= force_reg (inner_mode
,
1490 simplify_gen_unary (TRUNCATE
, inner_mode
, op1
,
1492 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1495 temp
= expand_binop_directly (icode
, mode
, otheroptab
, op0
, vop1
,
1496 target
, unsignedp
, methods
, last
);
1503 /* Look for a wider mode of the same class for which we think we
1504 can open-code the operation. Check for a widening multiply at the
1505 wider mode as well. */
1507 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1508 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1509 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1511 machine_mode next_mode
;
1512 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1513 || (binoptab
== smul_optab
1514 && GET_MODE_WIDER_MODE (wider_mode
).exists (&next_mode
)
1515 && (find_widening_optab_handler ((unsignedp
1517 : smul_widen_optab
),
1519 != CODE_FOR_nothing
)))
1521 rtx xop0
= op0
, xop1
= op1
;
1524 /* For certain integer operations, we need not actually extend
1525 the narrow operands, as long as we will truncate
1526 the results to the same narrowness. */
1528 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1529 || binoptab
== xor_optab
1530 || binoptab
== add_optab
|| binoptab
== sub_optab
1531 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1532 && mclass
== MODE_INT
)
1535 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1537 if (binoptab
!= ashl_optab
)
1538 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1542 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1544 /* The second operand of a shift must always be extended. */
1545 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1546 no_extend
&& binoptab
!= ashl_optab
);
1548 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1549 unsignedp
, OPTAB_DIRECT
);
1552 if (mclass
!= MODE_INT
1553 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1556 target
= gen_reg_rtx (mode
);
1557 convert_move (target
, temp
, 0);
1561 return gen_lowpart (mode
, temp
);
1564 delete_insns_since (last
);
1568 /* If operation is commutative,
1569 try to make the first operand a register.
1570 Even better, try to make it the same as the target.
1571 Also try to make the last operand a constant. */
1572 if (commutative_optab_p (binoptab
)
1573 && swap_commutative_operands_with_target (target
, op0
, op1
))
1574 std::swap (op0
, op1
);
1576 /* These can be done a word at a time. */
1577 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1578 && is_int_mode (mode
, &int_mode
)
1579 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
1580 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1585 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1586 won't be accurate, so use a new target. */
1590 || reg_overlap_mentioned_p (target
, op0
)
1591 || reg_overlap_mentioned_p (target
, op1
)
1592 || !valid_multiword_target_p (target
))
1593 target
= gen_reg_rtx (int_mode
);
1597 /* Do the actual arithmetic. */
1598 machine_mode op0_mode
= GET_MODE (op0
);
1599 machine_mode op1_mode
= GET_MODE (op1
);
1600 if (op0_mode
== VOIDmode
)
1601 op0_mode
= int_mode
;
1602 if (op1_mode
== VOIDmode
)
1603 op1_mode
= int_mode
;
1604 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
1606 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
1607 rtx x
= expand_binop (word_mode
, binoptab
,
1608 operand_subword_force (op0
, i
, op0_mode
),
1609 operand_subword_force (op1
, i
, op1_mode
),
1610 target_piece
, unsignedp
, next_methods
);
1615 if (target_piece
!= x
)
1616 emit_move_insn (target_piece
, x
);
1619 insns
= get_insns ();
1622 if (i
== GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
)
1629 /* Synthesize double word shifts from single word shifts. */
1630 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1631 || binoptab
== ashr_optab
)
1632 && is_int_mode (mode
, &int_mode
)
1633 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1634 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1635 && GET_MODE_PRECISION (int_mode
) == GET_MODE_BITSIZE (int_mode
)
1636 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1637 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1638 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1640 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1641 scalar_int_mode op1_mode
;
1643 double_shift_mask
= targetm
.shift_truncation_mask (int_mode
);
1644 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1645 op1_mode
= (GET_MODE (op1
) != VOIDmode
1646 ? as_a
<scalar_int_mode
> (GET_MODE (op1
))
1649 /* Apply the truncation to constant shifts. */
1650 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1651 op1
= gen_int_mode (INTVAL (op1
) & double_shift_mask
, op1_mode
);
1653 if (op1
== CONST0_RTX (op1_mode
))
1656 /* Make sure that this is a combination that expand_doubleword_shift
1657 can handle. See the comments there for details. */
1658 if (double_shift_mask
== 0
1659 || (shift_mask
== BITS_PER_WORD
- 1
1660 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1663 rtx into_target
, outof_target
;
1664 rtx into_input
, outof_input
;
1665 int left_shift
, outof_word
;
1667 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1668 won't be accurate, so use a new target. */
1672 || reg_overlap_mentioned_p (target
, op0
)
1673 || reg_overlap_mentioned_p (target
, op1
)
1674 || !valid_multiword_target_p (target
))
1675 target
= gen_reg_rtx (int_mode
);
1679 /* OUTOF_* is the word we are shifting bits away from, and
1680 INTO_* is the word that we are shifting bits towards, thus
1681 they differ depending on the direction of the shift and
1682 WORDS_BIG_ENDIAN. */
1684 left_shift
= binoptab
== ashl_optab
;
1685 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1687 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1688 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1690 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1691 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1693 if (expand_doubleword_shift (op1_mode
, binoptab
,
1694 outof_input
, into_input
, op1
,
1695 outof_target
, into_target
,
1696 unsignedp
, next_methods
, shift_mask
))
1698 insns
= get_insns ();
1708 /* Synthesize double word rotates from single word shifts. */
1709 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1710 && is_int_mode (mode
, &int_mode
)
1711 && CONST_INT_P (op1
)
1712 && GET_MODE_PRECISION (int_mode
) == 2 * BITS_PER_WORD
1713 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1714 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1717 rtx into_target
, outof_target
;
1718 rtx into_input
, outof_input
;
1720 int shift_count
, left_shift
, outof_word
;
1722 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1723 won't be accurate, so use a new target. Do this also if target is not
1724 a REG, first because having a register instead may open optimization
1725 opportunities, and second because if target and op0 happen to be MEMs
1726 designating the same location, we would risk clobbering it too early
1727 in the code sequence we generate below. */
1732 || reg_overlap_mentioned_p (target
, op0
)
1733 || reg_overlap_mentioned_p (target
, op1
)
1734 || !valid_multiword_target_p (target
))
1735 target
= gen_reg_rtx (int_mode
);
1739 shift_count
= INTVAL (op1
);
1741 /* OUTOF_* is the word we are shifting bits away from, and
1742 INTO_* is the word that we are shifting bits towards, thus
1743 they differ depending on the direction of the shift and
1744 WORDS_BIG_ENDIAN. */
1746 left_shift
= (binoptab
== rotl_optab
);
1747 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1749 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1750 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1752 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1753 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1755 if (shift_count
== BITS_PER_WORD
)
1757 /* This is just a word swap. */
1758 emit_move_insn (outof_target
, into_input
);
1759 emit_move_insn (into_target
, outof_input
);
1764 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1765 HOST_WIDE_INT first_shift_count
, second_shift_count
;
1766 optab reverse_unsigned_shift
, unsigned_shift
;
1768 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1769 ? lshr_optab
: ashl_optab
);
1771 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1772 ? ashl_optab
: lshr_optab
);
1774 if (shift_count
> BITS_PER_WORD
)
1776 first_shift_count
= shift_count
- BITS_PER_WORD
;
1777 second_shift_count
= 2 * BITS_PER_WORD
- shift_count
;
1781 first_shift_count
= BITS_PER_WORD
- shift_count
;
1782 second_shift_count
= shift_count
;
1784 rtx first_shift_count_rtx
1785 = gen_int_shift_amount (word_mode
, first_shift_count
);
1786 rtx second_shift_count_rtx
1787 = gen_int_shift_amount (word_mode
, second_shift_count
);
1789 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1790 outof_input
, first_shift_count_rtx
,
1791 NULL_RTX
, unsignedp
, next_methods
);
1792 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1793 into_input
, second_shift_count_rtx
,
1794 NULL_RTX
, unsignedp
, next_methods
);
1796 if (into_temp1
!= 0 && into_temp2
!= 0)
1797 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1798 into_target
, unsignedp
, next_methods
);
1802 if (inter
!= 0 && inter
!= into_target
)
1803 emit_move_insn (into_target
, inter
);
1805 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1806 into_input
, first_shift_count_rtx
,
1807 NULL_RTX
, unsignedp
, next_methods
);
1808 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1809 outof_input
, second_shift_count_rtx
,
1810 NULL_RTX
, unsignedp
, next_methods
);
1812 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1813 inter
= expand_binop (word_mode
, ior_optab
,
1814 outof_temp1
, outof_temp2
,
1815 outof_target
, unsignedp
, next_methods
);
1817 if (inter
!= 0 && inter
!= outof_target
)
1818 emit_move_insn (outof_target
, inter
);
1821 insns
= get_insns ();
1831 /* These can be done a word at a time by propagating carries. */
1832 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1833 && is_int_mode (mode
, &int_mode
)
1834 && GET_MODE_SIZE (int_mode
) >= 2 * UNITS_PER_WORD
1835 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1838 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1839 const unsigned int nwords
= GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
;
1840 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1841 rtx xop0
, xop1
, xtarget
;
1843 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1844 value is one of those, use it. Otherwise, use 1 since it is the
1845 one easiest to get. */
1846 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1847 int normalizep
= STORE_FLAG_VALUE
;
1852 /* Prepare the operands. */
1853 xop0
= force_reg (int_mode
, op0
);
1854 xop1
= force_reg (int_mode
, op1
);
1856 xtarget
= gen_reg_rtx (int_mode
);
1858 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1861 /* Indicate for flow that the entire target reg is being set. */
1863 emit_clobber (xtarget
);
1865 /* Do the actual arithmetic. */
1866 for (i
= 0; i
< nwords
; i
++)
1868 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
1869 rtx target_piece
= operand_subword (xtarget
, index
, 1, int_mode
);
1870 rtx op0_piece
= operand_subword_force (xop0
, index
, int_mode
);
1871 rtx op1_piece
= operand_subword_force (xop1
, index
, int_mode
);
1874 /* Main add/subtract of the input operands. */
1875 x
= expand_binop (word_mode
, binoptab
,
1876 op0_piece
, op1_piece
,
1877 target_piece
, unsignedp
, next_methods
);
1883 /* Store carry from main add/subtract. */
1884 carry_out
= gen_reg_rtx (word_mode
);
1885 carry_out
= emit_store_flag_force (carry_out
,
1886 (binoptab
== add_optab
1889 word_mode
, 1, normalizep
);
1896 /* Add/subtract previous carry to main result. */
1897 newx
= expand_binop (word_mode
,
1898 normalizep
== 1 ? binoptab
: otheroptab
,
1900 NULL_RTX
, 1, next_methods
);
1904 /* Get out carry from adding/subtracting carry in. */
1905 rtx carry_tmp
= gen_reg_rtx (word_mode
);
1906 carry_tmp
= emit_store_flag_force (carry_tmp
,
1907 (binoptab
== add_optab
1910 word_mode
, 1, normalizep
);
1912 /* Logical-ior the two poss. carry together. */
1913 carry_out
= expand_binop (word_mode
, ior_optab
,
1914 carry_out
, carry_tmp
,
1915 carry_out
, 0, next_methods
);
1919 emit_move_insn (target_piece
, newx
);
1923 if (x
!= target_piece
)
1924 emit_move_insn (target_piece
, x
);
1927 carry_in
= carry_out
;
1930 if (i
== GET_MODE_BITSIZE (int_mode
) / (unsigned) BITS_PER_WORD
)
1932 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
1933 || ! rtx_equal_p (target
, xtarget
))
1935 rtx_insn
*temp
= emit_move_insn (target
, xtarget
);
1937 set_dst_reg_note (temp
, REG_EQUAL
,
1938 gen_rtx_fmt_ee (optab_to_code (binoptab
),
1939 int_mode
, copy_rtx (xop0
),
1950 delete_insns_since (last
);
1953 /* Attempt to synthesize double word multiplies using a sequence of word
1954 mode multiplications. We first attempt to generate a sequence using a
1955 more efficient unsigned widening multiply, and if that fails we then
1956 try using a signed widening multiply. */
1958 if (binoptab
== smul_optab
1959 && is_int_mode (mode
, &int_mode
)
1960 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1961 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
1962 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
1964 rtx product
= NULL_RTX
;
1965 if (convert_optab_handler (umul_widen_optab
, int_mode
, word_mode
)
1966 != CODE_FOR_nothing
)
1968 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
1971 delete_insns_since (last
);
1974 if (product
== NULL_RTX
1975 && (convert_optab_handler (smul_widen_optab
, int_mode
, word_mode
)
1976 != CODE_FOR_nothing
))
1978 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
1981 delete_insns_since (last
);
1984 if (product
!= NULL_RTX
)
1986 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
1988 rtx_insn
*move
= emit_move_insn (target
? target
: product
,
1990 set_dst_reg_note (move
,
1992 gen_rtx_fmt_ee (MULT
, int_mode
,
1995 target
? target
: product
);
2001 /* Attempt to synthetize double word modulo by constant divisor. */
2002 if ((binoptab
== umod_optab
|| binoptab
== smod_optab
)
2004 && CONST_INT_P (op1
)
2005 && is_int_mode (mode
, &int_mode
)
2006 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2007 && optab_handler (lshr_optab
, int_mode
) != CODE_FOR_nothing
2008 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
2009 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
2010 && optimize_insn_for_speed_p ())
2012 rtx remainder
= expand_doubleword_mod (int_mode
, op0
, op1
,
2013 binoptab
== umod_optab
);
2014 if (remainder
!= NULL_RTX
)
2016 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
2018 rtx_insn
*move
= emit_move_insn (target
? target
: remainder
,
2020 set_dst_reg_note (move
,
2022 gen_rtx_fmt_ee (UMOD
, int_mode
,
2023 copy_rtx (op0
), op1
),
2024 target
? target
: remainder
);
2029 delete_insns_since (last
);
2032 /* It can't be open-coded in this mode.
2033 Use a library call if one is available and caller says that's ok. */
2035 libfunc
= optab_libfunc (binoptab
, mode
);
2037 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
2041 machine_mode op1_mode
= mode
;
2046 if (shift_optab_p (binoptab
))
2048 op1_mode
= targetm
.libgcc_shift_count_mode ();
2049 /* Specify unsigned here,
2050 since negative shift counts are meaningless. */
2051 op1x
= convert_to_mode (op1_mode
, op1
, 1);
2054 if (GET_MODE (op0
) != VOIDmode
2055 && GET_MODE (op0
) != mode
)
2056 op0
= convert_to_mode (mode
, op0
, unsignedp
);
2058 /* Pass 1 for NO_QUEUE so we don't lose any increments
2059 if the libcall is cse'd or moved. */
2060 value
= emit_library_call_value (libfunc
,
2061 NULL_RTX
, LCT_CONST
, mode
,
2062 op0
, mode
, op1x
, op1_mode
);
2064 insns
= get_insns ();
2067 bool trapv
= trapv_binoptab_p (binoptab
);
2068 target
= gen_reg_rtx (mode
);
2069 emit_libcall_block_1 (insns
, target
, value
,
2071 : gen_rtx_fmt_ee (optab_to_code (binoptab
),
2072 mode
, op0
, op1
), trapv
);
2077 delete_insns_since (last
);
2079 /* It can't be done in this mode. Can we do it in a wider mode? */
2081 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
2082 || methods
== OPTAB_MUST_WIDEN
))
2084 /* Caller says, don't even try. */
2085 delete_insns_since (entry_last
);
2089 /* Compute the value of METHODS to pass to recursive calls.
2090 Don't allow widening to be tried recursively. */
2092 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
2094 /* Look for a wider mode of the same class for which it appears we can do
2097 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2099 /* This code doesn't make sense for conversion optabs, since we
2100 wouldn't then want to extend the operands to be the same size
2102 gcc_assert (!convert_optab_p (binoptab
));
2103 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2105 if (optab_handler (binoptab
, wider_mode
)
2106 || (methods
== OPTAB_LIB
2107 && optab_libfunc (binoptab
, wider_mode
)))
2109 rtx xop0
= op0
, xop1
= op1
;
2112 /* For certain integer operations, we need not actually extend
2113 the narrow operands, as long as we will truncate
2114 the results to the same narrowness. */
2116 if ((binoptab
== ior_optab
|| binoptab
== and_optab
2117 || binoptab
== xor_optab
2118 || binoptab
== add_optab
|| binoptab
== sub_optab
2119 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
2120 && mclass
== MODE_INT
)
2123 xop0
= widen_operand (xop0
, wider_mode
, mode
,
2124 unsignedp
, no_extend
);
2126 /* The second operand of a shift must always be extended. */
2127 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
2128 no_extend
&& binoptab
!= ashl_optab
);
2130 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
2131 unsignedp
, methods
);
2134 if (mclass
!= MODE_INT
2135 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2138 target
= gen_reg_rtx (mode
);
2139 convert_move (target
, temp
, 0);
2143 return gen_lowpart (mode
, temp
);
2146 delete_insns_since (last
);
2151 delete_insns_since (entry_last
);
2155 /* Expand a binary operator which has both signed and unsigned forms.
2156 UOPTAB is the optab for unsigned operations, and SOPTAB is for
2159 If we widen unsigned operands, we may use a signed wider operation instead
2160 of an unsigned wider operation, since the result would be the same. */
2163 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
2164 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
2165 enum optab_methods methods
)
2168 optab direct_optab
= unsignedp
? uoptab
: soptab
;
2171 /* Do it without widening, if possible. */
2172 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2173 unsignedp
, OPTAB_DIRECT
);
2174 if (temp
|| methods
== OPTAB_DIRECT
)
2177 /* Try widening to a signed int. Disable any direct use of any
2178 signed insn in the current mode. */
2179 save_enable
= swap_optab_enable (soptab
, mode
, false);
2181 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2182 unsignedp
, OPTAB_WIDEN
);
2184 /* For unsigned operands, try widening to an unsigned int. */
2185 if (!temp
&& unsignedp
)
2186 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2187 unsignedp
, OPTAB_WIDEN
);
2188 if (temp
|| methods
== OPTAB_WIDEN
)
2191 /* Use the right width libcall if that exists. */
2192 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2193 unsignedp
, OPTAB_LIB
);
2194 if (temp
|| methods
== OPTAB_LIB
)
2197 /* Must widen and use a libcall, use either signed or unsigned. */
2198 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2199 unsignedp
, methods
);
2200 if (!temp
&& unsignedp
)
2201 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2202 unsignedp
, methods
);
2205 /* Undo the fiddling above. */
2207 swap_optab_enable (soptab
, mode
, true);
2211 /* Generate code to perform an operation specified by UNOPPTAB
2212 on operand OP0, with two results to TARG0 and TARG1.
2213 We assume that the order of the operands for the instruction
2214 is TARG0, TARG1, OP0.
2216 Either TARG0 or TARG1 may be zero, but what that means is that
2217 the result is not actually wanted. We will generate it into
2218 a dummy pseudo-reg and discard it. They may not both be zero.
2220 Returns 1 if this operation can be performed; 0 if not. */
2223 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
2226 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2227 enum mode_class mclass
;
2228 machine_mode wider_mode
;
2229 rtx_insn
*entry_last
= get_last_insn ();
2232 mclass
= GET_MODE_CLASS (mode
);
2235 targ0
= gen_reg_rtx (mode
);
2237 targ1
= gen_reg_rtx (mode
);
2239 /* Record where to go back to if we fail. */
2240 last
= get_last_insn ();
2242 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2244 class expand_operand ops
[3];
2245 enum insn_code icode
= optab_handler (unoptab
, mode
);
2247 create_fixed_operand (&ops
[0], targ0
);
2248 create_fixed_operand (&ops
[1], targ1
);
2249 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
2250 if (maybe_expand_insn (icode
, 3, ops
))
2254 /* It can't be done in this mode. Can we do it in a wider mode? */
2256 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2258 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2260 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2262 rtx t0
= gen_reg_rtx (wider_mode
);
2263 rtx t1
= gen_reg_rtx (wider_mode
);
2264 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2266 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
2268 convert_move (targ0
, t0
, unsignedp
);
2269 convert_move (targ1
, t1
, unsignedp
);
2273 delete_insns_since (last
);
2278 delete_insns_since (entry_last
);
2282 /* Generate code to perform an operation specified by BINOPTAB
2283 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2284 We assume that the order of the operands for the instruction
2285 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2286 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2288 Either TARG0 or TARG1 may be zero, but what that means is that
2289 the result is not actually wanted. We will generate it into
2290 a dummy pseudo-reg and discard it. They may not both be zero.
2292 Returns 1 if this operation can be performed; 0 if not. */
2295 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
2298 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2299 enum mode_class mclass
;
2300 machine_mode wider_mode
;
2301 rtx_insn
*entry_last
= get_last_insn ();
2304 mclass
= GET_MODE_CLASS (mode
);
2307 targ0
= gen_reg_rtx (mode
);
2309 targ1
= gen_reg_rtx (mode
);
2311 /* Record where to go back to if we fail. */
2312 last
= get_last_insn ();
2314 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2316 class expand_operand ops
[4];
2317 enum insn_code icode
= optab_handler (binoptab
, mode
);
2318 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2319 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2320 rtx xop0
= op0
, xop1
= op1
;
2322 /* If we are optimizing, force expensive constants into a register. */
2323 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2324 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2326 create_fixed_operand (&ops
[0], targ0
);
2327 create_convert_operand_from (&ops
[1], xop0
, mode
, unsignedp
);
2328 create_convert_operand_from (&ops
[2], xop1
, mode
, unsignedp
);
2329 create_fixed_operand (&ops
[3], targ1
);
2330 if (maybe_expand_insn (icode
, 4, ops
))
2332 delete_insns_since (last
);
2335 /* It can't be done in this mode. Can we do it in a wider mode? */
2337 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2339 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2341 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2343 rtx t0
= gen_reg_rtx (wider_mode
);
2344 rtx t1
= gen_reg_rtx (wider_mode
);
2345 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2346 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2348 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2351 convert_move (targ0
, t0
, unsignedp
);
2352 convert_move (targ1
, t1
, unsignedp
);
2356 delete_insns_since (last
);
2361 delete_insns_since (entry_last
);
2365 /* Expand the two-valued library call indicated by BINOPTAB, but
2366 preserve only one of the values. If TARG0 is non-NULL, the first
2367 value is placed into TARG0; otherwise the second value is placed
2368 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2369 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2370 This routine assumes that the value returned by the library call is
2371 as if the return value was of an integral mode twice as wide as the
2372 mode of OP0. Returns 1 if the call was successful. */
2375 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2376 rtx targ0
, rtx targ1
, enum rtx_code code
)
2379 machine_mode libval_mode
;
2384 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2385 gcc_assert (!targ0
!= !targ1
);
2387 mode
= GET_MODE (op0
);
2388 libfunc
= optab_libfunc (binoptab
, mode
);
2392 /* The value returned by the library function will have twice as
2393 many bits as the nominal MODE. */
2394 libval_mode
= smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode
));
2396 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2400 /* Get the part of VAL containing the value that we want. */
2401 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2402 targ0
? 0 : GET_MODE_SIZE (mode
));
2403 insns
= get_insns ();
2405 /* Move the into the desired location. */
2406 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2407 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2413 /* Wrapper around expand_unop which takes an rtx code to specify
2414 the operation to perform, not an optab pointer. All other
2415 arguments are the same. */
2417 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2418 rtx target
, int unsignedp
)
2420 optab unop
= code_to_optab (code
);
2423 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2429 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2431 A similar operation can be used for clrsb. UNOPTAB says which operation
2432 we are trying to expand. */
2434 widen_leading (scalar_int_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2436 opt_scalar_int_mode wider_mode_iter
;
2437 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2439 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2440 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2445 last
= get_last_insn ();
2448 target
= gen_reg_rtx (mode
);
2449 xop0
= widen_operand (op0
, wider_mode
, mode
,
2450 unoptab
!= clrsb_optab
, false);
2451 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2452 unoptab
!= clrsb_optab
);
2455 (wider_mode
, sub_optab
, temp
,
2456 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2457 - GET_MODE_PRECISION (mode
),
2459 target
, true, OPTAB_DIRECT
);
2461 delete_insns_since (last
);
2469 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2470 quantities, choosing which based on whether the high word is nonzero. */
2472 expand_doubleword_clz (scalar_int_mode mode
, rtx op0
, rtx target
)
2474 rtx xop0
= force_reg (mode
, op0
);
2475 rtx subhi
= gen_highpart (word_mode
, xop0
);
2476 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2477 rtx_code_label
*hi0_label
= gen_label_rtx ();
2478 rtx_code_label
*after_label
= gen_label_rtx ();
2482 /* If we were not given a target, use a word_mode register, not a
2483 'mode' register. The result will fit, and nobody is expecting
2484 anything bigger (the return type of __builtin_clz* is int). */
2486 target
= gen_reg_rtx (word_mode
);
2488 /* In any case, write to a word_mode scratch in both branches of the
2489 conditional, so we can ensure there is a single move insn setting
2490 'target' to tag a REG_EQUAL note on. */
2491 result
= gen_reg_rtx (word_mode
);
2495 /* If the high word is not equal to zero,
2496 then clz of the full value is clz of the high word. */
2497 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2498 word_mode
, true, hi0_label
);
2500 temp
= expand_unop_direct (word_mode
, clz_optab
, subhi
, result
, true);
2505 convert_move (result
, temp
, true);
2507 emit_jump_insn (targetm
.gen_jump (after_label
));
2510 /* Else clz of the full value is clz of the low word plus the number
2511 of bits in the high word. */
2512 emit_label (hi0_label
);
2514 temp
= expand_unop_direct (word_mode
, clz_optab
, sublo
, 0, true);
2517 temp
= expand_binop (word_mode
, add_optab
, temp
,
2518 gen_int_mode (GET_MODE_BITSIZE (word_mode
), word_mode
),
2519 result
, true, OPTAB_DIRECT
);
2523 convert_move (result
, temp
, true);
2525 emit_label (after_label
);
2526 convert_move (target
, result
, true);
2531 add_equal_note (seq
, target
, CLZ
, xop0
, NULL_RTX
, mode
);
2540 /* Try calculating popcount of a double-word quantity as two popcount's of
2541 word-sized quantities and summing up the results. */
2543 expand_doubleword_popcount (scalar_int_mode mode
, rtx op0
, rtx target
)
2550 t0
= expand_unop_direct (word_mode
, popcount_optab
,
2551 operand_subword_force (op0
, 0, mode
), NULL_RTX
,
2553 t1
= expand_unop_direct (word_mode
, popcount_optab
,
2554 operand_subword_force (op0
, 1, mode
), NULL_RTX
,
2562 /* If we were not given a target, use a word_mode register, not a
2563 'mode' register. The result will fit, and nobody is expecting
2564 anything bigger (the return type of __builtin_popcount* is int). */
2566 target
= gen_reg_rtx (word_mode
);
2568 t
= expand_binop (word_mode
, add_optab
, t0
, t1
, target
, 0, OPTAB_DIRECT
);
2573 add_equal_note (seq
, t
, POPCOUNT
, op0
, NULL_RTX
, mode
);
2581 (parity:narrow (low (x) ^ high (x))) */
2583 expand_doubleword_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2585 rtx t
= expand_binop (word_mode
, xor_optab
,
2586 operand_subword_force (op0
, 0, mode
),
2587 operand_subword_force (op0
, 1, mode
),
2588 NULL_RTX
, 0, OPTAB_DIRECT
);
2589 return expand_unop (word_mode
, parity_optab
, t
, target
, true);
2595 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2597 widen_bswap (scalar_int_mode mode
, rtx op0
, rtx target
)
2601 opt_scalar_int_mode wider_mode_iter
;
2603 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2604 if (optab_handler (bswap_optab
, wider_mode_iter
.require ())
2605 != CODE_FOR_nothing
)
2608 if (!wider_mode_iter
.exists ())
2611 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2612 last
= get_last_insn ();
2614 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2615 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2617 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2618 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2620 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2621 GET_MODE_BITSIZE (wider_mode
)
2622 - GET_MODE_BITSIZE (mode
),
2628 target
= gen_reg_rtx (mode
);
2629 emit_move_insn (target
, gen_lowpart (mode
, x
));
2632 delete_insns_since (last
);
2637 /* Try calculating bswap as two bswaps of two word-sized operands. */
2640 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2644 t1
= expand_unop (word_mode
, bswap_optab
,
2645 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2646 t0
= expand_unop (word_mode
, bswap_optab
,
2647 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2649 if (target
== 0 || !valid_multiword_target_p (target
))
2650 target
= gen_reg_rtx (mode
);
2652 emit_clobber (target
);
2653 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2654 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2659 /* Try calculating (parity x) as (and (popcount x) 1), where
2660 popcount can also be done in a wider mode. */
2662 expand_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2664 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2665 opt_scalar_int_mode wider_mode_iter
;
2666 FOR_EACH_MODE_FROM (wider_mode_iter
, mode
)
2668 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2669 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2674 last
= get_last_insn ();
2676 if (target
== 0 || GET_MODE (target
) != wider_mode
)
2677 target
= gen_reg_rtx (wider_mode
);
2679 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2680 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2683 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2684 target
, true, OPTAB_DIRECT
);
2688 if (mclass
!= MODE_INT
2689 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2690 return convert_to_mode (mode
, temp
, 0);
2692 return gen_lowpart (mode
, temp
);
2695 delete_insns_since (last
);
2701 /* Try calculating ctz(x) as K - clz(x & -x) ,
2702 where K is GET_MODE_PRECISION(mode) - 1.
2704 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2705 don't have to worry about what the hardware does in that case. (If
2706 the clz instruction produces the usual value at 0, which is K, the
2707 result of this code sequence will be -1; expand_ffs, below, relies
2708 on this. It might be nice to have it be K instead, for consistency
2709 with the (very few) processors that provide a ctz with a defined
2710 value, but that would take one more instruction, and it would be
2711 less convenient for expand_ffs anyway. */
2714 expand_ctz (scalar_int_mode mode
, rtx op0
, rtx target
)
2719 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2724 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2726 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2727 true, OPTAB_DIRECT
);
2729 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2731 temp
= expand_binop (mode
, sub_optab
,
2732 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
2734 true, OPTAB_DIRECT
);
2744 add_equal_note (seq
, temp
, CTZ
, op0
, NULL_RTX
, mode
);
2750 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2751 else with the sequence used by expand_clz.
2753 The ffs builtin promises to return zero for a zero value and ctz/clz
2754 may have an undefined value in that case. If they do not give us a
2755 convenient value, we have to generate a test and branch. */
2757 expand_ffs (scalar_int_mode mode
, rtx op0
, rtx target
)
2759 HOST_WIDE_INT val
= 0;
2760 bool defined_at_zero
= false;
2764 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
2768 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
2772 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
2774 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
2777 temp
= expand_ctz (mode
, op0
, 0);
2781 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
2783 defined_at_zero
= true;
2784 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
2790 if (defined_at_zero
&& val
== -1)
2791 /* No correction needed at zero. */;
2794 /* We don't try to do anything clever with the situation found
2795 on some processors (eg Alpha) where ctz(0:mode) ==
2796 bitsize(mode). If someone can think of a way to send N to -1
2797 and leave alone all values in the range 0..N-1 (where N is a
2798 power of two), cheaper than this test-and-branch, please add it.
2800 The test-and-branch is done after the operation itself, in case
2801 the operation sets condition codes that can be recycled for this.
2802 (This is true on i386, for instance.) */
2804 rtx_code_label
*nonzero_label
= gen_label_rtx ();
2805 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
2806 mode
, true, nonzero_label
);
2808 convert_move (temp
, GEN_INT (-1), false);
2809 emit_label (nonzero_label
);
2812 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2813 to produce a value in the range 0..bitsize. */
2814 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
2815 target
, false, OPTAB_DIRECT
);
2822 add_equal_note (seq
, temp
, FFS
, op0
, NULL_RTX
, mode
);
2831 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2832 conditions, VAL may already be a SUBREG against which we cannot generate
2833 a further SUBREG. In this case, we expect forcing the value into a
2834 register will work around the situation. */
2837 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
2841 ret
= lowpart_subreg (omode
, val
, imode
);
2844 val
= force_reg (imode
, val
);
2845 ret
= lowpart_subreg (omode
, val
, imode
);
2846 gcc_assert (ret
!= NULL
);
2851 /* Expand a floating point absolute value or negation operation via a
2852 logical operation on the sign bit. */
2855 expand_absneg_bit (enum rtx_code code
, scalar_float_mode mode
,
2856 rtx op0
, rtx target
)
2858 const struct real_format
*fmt
;
2859 int bitpos
, word
, nwords
, i
;
2860 scalar_int_mode imode
;
2864 /* The format has to have a simple sign bit. */
2865 fmt
= REAL_MODE_FORMAT (mode
);
2869 bitpos
= fmt
->signbit_rw
;
2873 /* Don't create negative zeros if the format doesn't support them. */
2874 if (code
== NEG
&& !fmt
->has_signed_zero
)
2877 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
2879 if (!int_mode_for_mode (mode
).exists (&imode
))
2888 if (FLOAT_WORDS_BIG_ENDIAN
)
2889 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
2891 word
= bitpos
/ BITS_PER_WORD
;
2892 bitpos
= bitpos
% BITS_PER_WORD
;
2893 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
2896 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
2902 || reg_overlap_mentioned_p (target
, op0
)
2903 || (nwords
> 1 && !valid_multiword_target_p (target
)))
2904 target
= gen_reg_rtx (mode
);
2910 for (i
= 0; i
< nwords
; ++i
)
2912 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
2913 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
2917 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2919 immed_wide_int_const (mask
, imode
),
2920 targ_piece
, 1, OPTAB_LIB_WIDEN
);
2921 if (temp
!= targ_piece
)
2922 emit_move_insn (targ_piece
, temp
);
2925 emit_move_insn (targ_piece
, op0_piece
);
2928 insns
= get_insns ();
2935 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2936 gen_lowpart (imode
, op0
),
2937 immed_wide_int_const (mask
, imode
),
2938 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
2939 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
2941 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
2942 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
2949 /* As expand_unop, but will fail rather than attempt the operation in a
2950 different mode or with a libcall. */
2952 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2955 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2957 class expand_operand ops
[2];
2958 enum insn_code icode
= optab_handler (unoptab
, mode
);
2959 rtx_insn
*last
= get_last_insn ();
2962 create_output_operand (&ops
[0], target
, mode
);
2963 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2964 pat
= maybe_gen_insn (icode
, 2, ops
);
2967 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
2968 && ! add_equal_note (pat
, ops
[0].value
,
2969 optab_to_code (unoptab
),
2970 ops
[1].value
, NULL_RTX
, mode
))
2972 delete_insns_since (last
);
2973 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
2978 return ops
[0].value
;
2984 /* Generate code to perform an operation specified by UNOPTAB
2985 on operand OP0, with result having machine-mode MODE.
2987 UNSIGNEDP is for the case where we have to widen the operands
2988 to perform the operation. It says to use zero-extension.
2990 If TARGET is nonzero, the value
2991 is generated there, if it is convenient to do so.
2992 In all cases an rtx is returned for the locus of the value;
2993 this may or may not be TARGET. */
2996 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2999 enum mode_class mclass
= GET_MODE_CLASS (mode
);
3000 machine_mode wider_mode
;
3001 scalar_int_mode int_mode
;
3002 scalar_float_mode float_mode
;
3006 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
3010 /* It can't be done in this mode. Can we open-code it in a wider mode? */
3012 /* Widening (or narrowing) clz needs special treatment. */
3013 if (unoptab
== clz_optab
)
3015 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3017 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
3021 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3022 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3024 temp
= expand_doubleword_clz (int_mode
, op0
, target
);
3033 if (unoptab
== clrsb_optab
)
3035 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3037 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
3044 if (unoptab
== popcount_optab
3045 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3046 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3047 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3048 && optimize_insn_for_speed_p ())
3050 temp
= expand_doubleword_popcount (int_mode
, op0
, target
);
3055 if (unoptab
== parity_optab
3056 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3057 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3058 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3059 || optab_handler (popcount_optab
, word_mode
) != CODE_FOR_nothing
)
3060 && optimize_insn_for_speed_p ())
3062 temp
= expand_doubleword_parity (int_mode
, op0
, target
);
3067 /* Widening (or narrowing) bswap needs special treatment. */
3068 if (unoptab
== bswap_optab
)
3070 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3071 or ROTATERT. First try these directly; if this fails, then try the
3072 obvious pair of shifts with allowed widening, as this will probably
3073 be always more efficient than the other fallback methods. */
3079 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
3081 temp
= expand_binop (mode
, rotl_optab
, op0
,
3082 gen_int_shift_amount (mode
, 8),
3083 target
, unsignedp
, OPTAB_DIRECT
);
3088 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
3090 temp
= expand_binop (mode
, rotr_optab
, op0
,
3091 gen_int_shift_amount (mode
, 8),
3092 target
, unsignedp
, OPTAB_DIRECT
);
3097 last
= get_last_insn ();
3099 temp1
= expand_binop (mode
, ashl_optab
, op0
,
3100 gen_int_shift_amount (mode
, 8), NULL_RTX
,
3101 unsignedp
, OPTAB_WIDEN
);
3102 temp2
= expand_binop (mode
, lshr_optab
, op0
,
3103 gen_int_shift_amount (mode
, 8), NULL_RTX
,
3104 unsignedp
, OPTAB_WIDEN
);
3107 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
3108 unsignedp
, OPTAB_WIDEN
);
3113 delete_insns_since (last
);
3116 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3118 temp
= widen_bswap (int_mode
, op0
, target
);
3122 /* We do not provide a 128-bit bswap in libgcc so force the use of
3123 a double bswap for 64-bit targets. */
3124 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3125 && (UNITS_PER_WORD
== 8
3126 || optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
))
3128 temp
= expand_doubleword_bswap (mode
, op0
, target
);
3137 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3138 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3140 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
3143 rtx_insn
*last
= get_last_insn ();
3145 /* For certain operations, we need not actually extend
3146 the narrow operand, as long as we will truncate the
3147 results to the same narrowness. */
3149 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3150 (unoptab
== neg_optab
3151 || unoptab
== one_cmpl_optab
)
3152 && mclass
== MODE_INT
);
3154 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3159 if (mclass
!= MODE_INT
3160 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
3163 target
= gen_reg_rtx (mode
);
3164 convert_move (target
, temp
, 0);
3168 return gen_lowpart (mode
, temp
);
3171 delete_insns_since (last
);
3175 /* These can be done a word at a time. */
3176 if (unoptab
== one_cmpl_optab
3177 && is_int_mode (mode
, &int_mode
)
3178 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
3179 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3186 || reg_overlap_mentioned_p (target
, op0
)
3187 || !valid_multiword_target_p (target
))
3188 target
= gen_reg_rtx (int_mode
);
3192 /* Do the actual arithmetic. */
3193 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
3195 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
3196 rtx x
= expand_unop (word_mode
, unoptab
,
3197 operand_subword_force (op0
, i
, int_mode
),
3198 target_piece
, unsignedp
);
3200 if (target_piece
!= x
)
3201 emit_move_insn (target_piece
, x
);
3204 insns
= get_insns ();
3211 /* Emit ~op0 as op0 ^ -1. */
3212 if (unoptab
== one_cmpl_optab
3213 && (SCALAR_INT_MODE_P (mode
) || GET_MODE_CLASS (mode
) == MODE_VECTOR_INT
)
3214 && optab_handler (xor_optab
, mode
) != CODE_FOR_nothing
)
3216 temp
= expand_binop (mode
, xor_optab
, op0
, CONSTM1_RTX (mode
),
3217 target
, unsignedp
, OPTAB_DIRECT
);
3222 if (optab_to_code (unoptab
) == NEG
)
3224 /* Try negating floating point values by flipping the sign bit. */
3225 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3227 temp
= expand_absneg_bit (NEG
, float_mode
, op0
, target
);
3232 /* If there is no negation pattern, and we have no negative zero,
3233 try subtracting from zero. */
3234 if (!HONOR_SIGNED_ZEROS (mode
))
3236 temp
= expand_binop (mode
, (unoptab
== negv_optab
3237 ? subv_optab
: sub_optab
),
3238 CONST0_RTX (mode
), op0
, target
,
3239 unsignedp
, OPTAB_DIRECT
);
3245 /* Try calculating parity (x) as popcount (x) % 2. */
3246 if (unoptab
== parity_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3248 temp
= expand_parity (int_mode
, op0
, target
);
3253 /* Try implementing ffs (x) in terms of clz (x). */
3254 if (unoptab
== ffs_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3256 temp
= expand_ffs (int_mode
, op0
, target
);
3261 /* Try implementing ctz (x) in terms of clz (x). */
3262 if (unoptab
== ctz_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3264 temp
= expand_ctz (int_mode
, op0
, target
);
3270 /* Now try a library call in this mode. */
3271 libfunc
= optab_libfunc (unoptab
, mode
);
3277 machine_mode outmode
= mode
;
3279 /* All of these functions return small values. Thus we choose to
3280 have them return something that isn't a double-word. */
3281 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
3282 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
3283 || unoptab
== parity_optab
)
3285 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
3286 optab_libfunc (unoptab
, mode
)));
3290 /* Pass 1 for NO_QUEUE so we don't lose any increments
3291 if the libcall is cse'd or moved. */
3292 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
3294 insns
= get_insns ();
3297 target
= gen_reg_rtx (outmode
);
3298 bool trapv
= trapv_unoptab_p (unoptab
);
3300 eq_value
= NULL_RTX
;
3303 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
3304 if (GET_MODE_UNIT_SIZE (outmode
) < GET_MODE_UNIT_SIZE (mode
))
3305 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
3306 else if (GET_MODE_UNIT_SIZE (outmode
) > GET_MODE_UNIT_SIZE (mode
))
3307 eq_value
= simplify_gen_unary (ZERO_EXTEND
,
3308 outmode
, eq_value
, mode
);
3310 emit_libcall_block_1 (insns
, target
, value
, eq_value
, trapv
);
3315 /* It can't be done in this mode. Can we do it in a wider mode? */
3317 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3319 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3321 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
3322 || optab_libfunc (unoptab
, wider_mode
))
3325 rtx_insn
*last
= get_last_insn ();
3327 /* For certain operations, we need not actually extend
3328 the narrow operand, as long as we will truncate the
3329 results to the same narrowness. */
3330 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3331 (unoptab
== neg_optab
3332 || unoptab
== one_cmpl_optab
3333 || unoptab
== bswap_optab
)
3334 && mclass
== MODE_INT
);
3336 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3339 /* If we are generating clz using wider mode, adjust the
3340 result. Similarly for clrsb. */
3341 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3344 scalar_int_mode wider_int_mode
3345 = as_a
<scalar_int_mode
> (wider_mode
);
3346 int_mode
= as_a
<scalar_int_mode
> (mode
);
3348 (wider_mode
, sub_optab
, temp
,
3349 gen_int_mode (GET_MODE_PRECISION (wider_int_mode
)
3350 - GET_MODE_PRECISION (int_mode
),
3352 target
, true, OPTAB_DIRECT
);
3355 /* Likewise for bswap. */
3356 if (unoptab
== bswap_optab
&& temp
!= 0)
3358 scalar_int_mode wider_int_mode
3359 = as_a
<scalar_int_mode
> (wider_mode
);
3360 int_mode
= as_a
<scalar_int_mode
> (mode
);
3361 gcc_assert (GET_MODE_PRECISION (wider_int_mode
)
3362 == GET_MODE_BITSIZE (wider_int_mode
)
3363 && GET_MODE_PRECISION (int_mode
)
3364 == GET_MODE_BITSIZE (int_mode
));
3366 temp
= expand_shift (RSHIFT_EXPR
, wider_int_mode
, temp
,
3367 GET_MODE_BITSIZE (wider_int_mode
)
3368 - GET_MODE_BITSIZE (int_mode
),
3374 if (mclass
!= MODE_INT
)
3377 target
= gen_reg_rtx (mode
);
3378 convert_move (target
, temp
, 0);
3382 return gen_lowpart (mode
, temp
);
3385 delete_insns_since (last
);
3390 /* One final attempt at implementing negation via subtraction,
3391 this time allowing widening of the operand. */
3392 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3395 temp
= expand_binop (mode
,
3396 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3397 CONST0_RTX (mode
), op0
,
3398 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3406 /* Emit code to compute the absolute value of OP0, with result to
3407 TARGET if convenient. (TARGET may be 0.) The return value says
3408 where the result actually is to be found.
3410 MODE is the mode of the operand; the mode of the result is
3411 different but can be deduced from MODE.
3416 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3417 int result_unsignedp
)
3421 if (GET_MODE_CLASS (mode
) != MODE_INT
3423 result_unsignedp
= 1;
3425 /* First try to do it with a special abs instruction. */
3426 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3431 /* For floating point modes, try clearing the sign bit. */
3432 scalar_float_mode float_mode
;
3433 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3435 temp
= expand_absneg_bit (ABS
, float_mode
, op0
, target
);
3440 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3441 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3442 && !HONOR_SIGNED_ZEROS (mode
))
3444 rtx_insn
*last
= get_last_insn ();
3446 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3449 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3455 delete_insns_since (last
);
3458 /* If this machine has expensive jumps, we can do integer absolute
3459 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3460 where W is the width of MODE. */
3462 scalar_int_mode int_mode
;
3463 if (is_int_mode (mode
, &int_mode
)
3464 && BRANCH_COST (optimize_insn_for_speed_p (),
3467 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3468 GET_MODE_PRECISION (int_mode
) - 1,
3471 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3474 temp
= expand_binop (int_mode
,
3475 result_unsignedp
? sub_optab
: subv_optab
,
3476 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3486 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3487 int result_unsignedp
, int safe
)
3490 rtx_code_label
*op1
;
3492 if (GET_MODE_CLASS (mode
) != MODE_INT
3494 result_unsignedp
= 1;
3496 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3500 /* If that does not win, use conditional jump and negate. */
3502 /* It is safe to use the target if it is the same
3503 as the source if this is also a pseudo register */
3504 if (op0
== target
&& REG_P (op0
)
3505 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3508 op1
= gen_label_rtx ();
3509 if (target
== 0 || ! safe
3510 || GET_MODE (target
) != mode
3511 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3513 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3514 target
= gen_reg_rtx (mode
);
3516 emit_move_insn (target
, op0
);
3519 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3520 NULL_RTX
, NULL
, op1
,
3521 profile_probability::uninitialized ());
3523 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3526 emit_move_insn (target
, op0
);
3532 /* Emit code to compute the one's complement absolute value of OP0
3533 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3534 (TARGET may be NULL_RTX.) The return value says where the result
3535 actually is to be found.
3537 MODE is the mode of the operand; the mode of the result is
3538 different but can be deduced from MODE. */
3541 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3545 /* Not applicable for floating point modes. */
3546 if (FLOAT_MODE_P (mode
))
3549 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3550 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3552 rtx_insn
*last
= get_last_insn ();
3554 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3556 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3562 delete_insns_since (last
);
3565 /* If this machine has expensive jumps, we can do one's complement
3566 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3568 scalar_int_mode int_mode
;
3569 if (is_int_mode (mode
, &int_mode
)
3570 && BRANCH_COST (optimize_insn_for_speed_p (),
3573 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3574 GET_MODE_PRECISION (int_mode
) - 1,
3577 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3587 /* A subroutine of expand_copysign, perform the copysign operation using the
3588 abs and neg primitives advertised to exist on the target. The assumption
3589 is that we have a split register file, and leaving op0 in fp registers,
3590 and not playing with subregs so much, will help the register allocator. */
3593 expand_copysign_absneg (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3594 int bitpos
, bool op0_is_abs
)
3596 scalar_int_mode imode
;
3597 enum insn_code icode
;
3599 rtx_code_label
*label
;
3604 /* Check if the back end provides an insn that handles signbit for the
3606 icode
= optab_handler (signbit_optab
, mode
);
3607 if (icode
!= CODE_FOR_nothing
)
3609 imode
= as_a
<scalar_int_mode
> (insn_data
[(int) icode
].operand
[0].mode
);
3610 sign
= gen_reg_rtx (imode
);
3611 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3615 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3617 if (!int_mode_for_mode (mode
).exists (&imode
))
3619 op1
= gen_lowpart (imode
, op1
);
3626 if (FLOAT_WORDS_BIG_ENDIAN
)
3627 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3629 word
= bitpos
/ BITS_PER_WORD
;
3630 bitpos
= bitpos
% BITS_PER_WORD
;
3631 op1
= operand_subword_force (op1
, word
, mode
);
3634 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3635 sign
= expand_binop (imode
, and_optab
, op1
,
3636 immed_wide_int_const (mask
, imode
),
3637 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3642 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3649 if (target
== NULL_RTX
)
3650 target
= copy_to_reg (op0
);
3652 emit_move_insn (target
, op0
);
3655 label
= gen_label_rtx ();
3656 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3658 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3659 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3661 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3663 emit_move_insn (target
, op0
);
3671 /* A subroutine of expand_copysign, perform the entire copysign operation
3672 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3673 is true if op0 is known to have its sign bit clear. */
3676 expand_copysign_bit (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3677 int bitpos
, bool op0_is_abs
)
3679 scalar_int_mode imode
;
3680 int word
, nwords
, i
;
3684 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3686 if (!int_mode_for_mode (mode
).exists (&imode
))
3695 if (FLOAT_WORDS_BIG_ENDIAN
)
3696 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3698 word
= bitpos
/ BITS_PER_WORD
;
3699 bitpos
= bitpos
% BITS_PER_WORD
;
3700 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3703 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3708 || reg_overlap_mentioned_p (target
, op0
)
3709 || reg_overlap_mentioned_p (target
, op1
)
3710 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3711 target
= gen_reg_rtx (mode
);
3717 for (i
= 0; i
< nwords
; ++i
)
3719 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3720 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3726 = expand_binop (imode
, and_optab
, op0_piece
,
3727 immed_wide_int_const (~mask
, imode
),
3728 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3729 op1
= expand_binop (imode
, and_optab
,
3730 operand_subword_force (op1
, i
, mode
),
3731 immed_wide_int_const (mask
, imode
),
3732 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3734 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
3735 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3736 if (temp
!= targ_piece
)
3737 emit_move_insn (targ_piece
, temp
);
3740 emit_move_insn (targ_piece
, op0_piece
);
3743 insns
= get_insns ();
3750 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
3751 immed_wide_int_const (mask
, imode
),
3752 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3754 op0
= gen_lowpart (imode
, op0
);
3756 op0
= expand_binop (imode
, and_optab
, op0
,
3757 immed_wide_int_const (~mask
, imode
),
3758 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3760 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
3761 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3762 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3768 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3769 scalar floating point mode. Return NULL if we do not know how to
3770 expand the operation inline. */
3773 expand_copysign (rtx op0
, rtx op1
, rtx target
)
3775 scalar_float_mode mode
;
3776 const struct real_format
*fmt
;
3780 mode
= as_a
<scalar_float_mode
> (GET_MODE (op0
));
3781 gcc_assert (GET_MODE (op1
) == mode
);
3783 /* First try to do it with a special instruction. */
3784 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
3785 target
, 0, OPTAB_DIRECT
);
3789 fmt
= REAL_MODE_FORMAT (mode
);
3790 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
3794 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3796 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
3797 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
3801 if (fmt
->signbit_ro
>= 0
3802 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
3803 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
3804 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
3806 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
3807 fmt
->signbit_ro
, op0_is_abs
);
3812 if (fmt
->signbit_rw
< 0)
3814 return expand_copysign_bit (mode
, op0
, op1
, target
,
3815 fmt
->signbit_rw
, op0_is_abs
);
3818 /* Generate an instruction whose insn-code is INSN_CODE,
3819 with two operands: an output TARGET and an input OP0.
3820 TARGET *must* be nonzero, and the output is always stored there.
3821 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3822 the value that is stored into TARGET.
3824 Return false if expansion failed. */
3827 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
3830 class expand_operand ops
[2];
3833 create_output_operand (&ops
[0], target
, GET_MODE (target
));
3834 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
3835 pat
= maybe_gen_insn (icode
, 2, ops
);
3839 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
3841 add_equal_note (pat
, ops
[0].value
, code
, ops
[1].value
, NULL_RTX
,
3846 if (ops
[0].value
!= target
)
3847 emit_move_insn (target
, ops
[0].value
);
3850 /* Generate an instruction whose insn-code is INSN_CODE,
3851 with two operands: an output TARGET and an input OP0.
3852 TARGET *must* be nonzero, and the output is always stored there.
3853 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3854 the value that is stored into TARGET. */
3857 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
3859 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
3863 struct no_conflict_data
3866 rtx_insn
*first
, *insn
;
3870 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3871 the currently examined clobber / store has to stay in the list of
3872 insns that constitute the actual libcall block. */
3874 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
3876 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
3878 /* If this inns directly contributes to setting the target, it must stay. */
3879 if (reg_overlap_mentioned_p (p
->target
, dest
))
3880 p
->must_stay
= true;
3881 /* If we haven't committed to keeping any other insns in the list yet,
3882 there is nothing more to check. */
3883 else if (p
->insn
== p
->first
)
3885 /* If this insn sets / clobbers a register that feeds one of the insns
3886 already in the list, this insn has to stay too. */
3887 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
3888 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
3889 || reg_used_between_p (dest
, p
->first
, p
->insn
)
3890 /* Likewise if this insn depends on a register set by a previous
3891 insn in the list, or if it sets a result (presumably a hard
3892 register) that is set or clobbered by a previous insn.
3893 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3894 SET_DEST perform the former check on the address, and the latter
3895 check on the MEM. */
3896 || (GET_CODE (set
) == SET
3897 && (modified_in_p (SET_SRC (set
), p
->first
)
3898 || modified_in_p (SET_DEST (set
), p
->first
)
3899 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
3900 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
3901 p
->must_stay
= true;
3905 /* Emit code to make a call to a constant function or a library call.
3907 INSNS is a list containing all insns emitted in the call.
3908 These insns leave the result in RESULT. Our block is to copy RESULT
3909 to TARGET, which is logically equivalent to EQUIV.
3911 We first emit any insns that set a pseudo on the assumption that these are
3912 loading constants into registers; doing so allows them to be safely cse'ed
3913 between blocks. Then we emit all the other insns in the block, followed by
3914 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3915 note with an operand of EQUIV. */
3918 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
3919 bool equiv_may_trap
)
3921 rtx final_dest
= target
;
3922 rtx_insn
*next
, *last
, *insn
;
3924 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3925 into a MEM later. Protect the libcall block from this change. */
3926 if (! REG_P (target
) || REG_USERVAR_P (target
))
3927 target
= gen_reg_rtx (GET_MODE (target
));
3929 /* If we're using non-call exceptions, a libcall corresponding to an
3930 operation that may trap may also trap. */
3931 /* ??? See the comment in front of make_reg_eh_region_note. */
3932 if (cfun
->can_throw_non_call_exceptions
3933 && (equiv_may_trap
|| may_trap_p (equiv
)))
3935 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3938 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
3941 int lp_nr
= INTVAL (XEXP (note
, 0));
3942 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
3943 remove_note (insn
, note
);
3949 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3950 reg note to indicate that this call cannot throw or execute a nonlocal
3951 goto (unless there is already a REG_EH_REGION note, in which case
3953 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3955 make_reg_eh_region_note_nothrow_nononlocal (insn
);
3958 /* First emit all insns that set pseudos. Remove them from the list as
3959 we go. Avoid insns that set pseudos which were referenced in previous
3960 insns. These can be generated by move_by_pieces, for example,
3961 to update an address. Similarly, avoid insns that reference things
3962 set in previous insns. */
3964 for (insn
= insns
; insn
; insn
= next
)
3966 rtx set
= single_set (insn
);
3968 next
= NEXT_INSN (insn
);
3970 if (set
!= 0 && REG_P (SET_DEST (set
))
3971 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
3973 struct no_conflict_data data
;
3975 data
.target
= const0_rtx
;
3979 note_stores (insn
, no_conflict_move_test
, &data
);
3980 if (! data
.must_stay
)
3982 if (PREV_INSN (insn
))
3983 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
3988 SET_PREV_INSN (next
) = PREV_INSN (insn
);
3994 /* Some ports use a loop to copy large arguments onto the stack.
3995 Don't move anything outside such a loop. */
4000 /* Write the remaining insns followed by the final copy. */
4001 for (insn
= insns
; insn
; insn
= next
)
4003 next
= NEXT_INSN (insn
);
4008 last
= emit_move_insn (target
, result
);
4010 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
4012 if (final_dest
!= target
)
4013 emit_move_insn (final_dest
, target
);
4017 emit_libcall_block (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
)
4019 emit_libcall_block_1 (insns
, target
, result
, equiv
, false);
4022 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
4023 PURPOSE describes how this comparison will be used. CODE is the rtx
4024 comparison code we will be using.
4026 ??? Actually, CODE is slightly weaker than that. A target is still
4027 required to implement all of the normal bcc operations, but not
4028 required to implement all (or any) of the unordered bcc operations. */
4031 can_compare_p (enum rtx_code code
, machine_mode mode
,
4032 enum can_compare_purpose purpose
)
4035 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
4038 enum insn_code icode
;
4040 if (purpose
== ccp_jump
4041 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
4042 && insn_operand_matches (icode
, 0, test
))
4044 if (purpose
== ccp_store_flag
4045 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
4046 && insn_operand_matches (icode
, 1, test
))
4048 if (purpose
== ccp_cmov
4049 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
4052 mode
= GET_MODE_WIDER_MODE (mode
).else_void ();
4053 PUT_MODE (test
, mode
);
4055 while (mode
!= VOIDmode
);
4060 /* Return whether RTL code CODE corresponds to an unsigned optab. */
4063 unsigned_optab_p (enum rtx_code code
)
4065 return code
== LTU
|| code
== LEU
|| code
== GTU
|| code
== GEU
;
4068 /* Return whether the backend-emitted comparison for code CODE, comparing
4069 operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4070 operand OPNO of pattern ICODE. */
4073 insn_predicate_matches_p (enum insn_code icode
, unsigned int opno
,
4074 enum rtx_code code
, machine_mode mask_mode
,
4075 machine_mode value_mode
)
4077 rtx reg1
= alloca_raw_REG (value_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4078 rtx reg2
= alloca_raw_REG (value_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4079 rtx test
= alloca_rtx_fmt_ee (code
, mask_mode
, reg1
, reg2
);
4080 return insn_operand_matches (icode
, opno
, test
);
4083 /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4084 for code CODE, comparing operands of mode VALUE_MODE and producing a result
4088 can_vec_cmp_compare_p (enum rtx_code code
, machine_mode value_mode
,
4089 machine_mode mask_mode
)
4091 enum insn_code icode
4092 = get_vec_cmp_icode (value_mode
, mask_mode
, unsigned_optab_p (code
));
4093 if (icode
== CODE_FOR_nothing
)
4096 return insn_predicate_matches_p (icode
, 1, code
, mask_mode
, value_mode
);
4099 /* Return whether the backend can emit a vector comparison (vcond/vcondu) for
4100 code CODE, comparing operands of mode CMP_OP_MODE and producing a result
4104 can_vcond_compare_p (enum rtx_code code
, machine_mode value_mode
,
4105 machine_mode cmp_op_mode
)
4107 enum insn_code icode
4108 = get_vcond_icode (value_mode
, cmp_op_mode
, unsigned_optab_p (code
));
4109 if (icode
== CODE_FOR_nothing
)
4112 return insn_predicate_matches_p (icode
, 3, code
, value_mode
, cmp_op_mode
);
4115 /* Return whether the backend can emit vector set instructions for inserting
4116 element into vector at variable index position. */
4119 can_vec_set_var_idx_p (machine_mode vec_mode
)
4121 if (!VECTOR_MODE_P (vec_mode
))
4124 machine_mode inner_mode
= GET_MODE_INNER (vec_mode
);
4125 rtx reg1
= alloca_raw_REG (vec_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4126 rtx reg2
= alloca_raw_REG (inner_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4127 rtx reg3
= alloca_raw_REG (VOIDmode
, LAST_VIRTUAL_REGISTER
+ 3);
4129 enum insn_code icode
= optab_handler (vec_set_optab
, vec_mode
);
4131 return icode
!= CODE_FOR_nothing
&& insn_operand_matches (icode
, 0, reg1
)
4132 && insn_operand_matches (icode
, 1, reg2
)
4133 && insn_operand_matches (icode
, 2, reg3
);
4136 /* This function is called when we are going to emit a compare instruction that
4137 compares the values found in X and Y, using the rtl operator COMPARISON.
4139 If they have mode BLKmode, then SIZE specifies the size of both operands.
4141 UNSIGNEDP nonzero says that the operands are unsigned;
4142 this matters if they need to be widened (as given by METHODS).
4144 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
4145 if we failed to produce one.
4147 *PMODE is the mode of the inputs (in case they are const_int).
4149 This function performs all the setup necessary so that the caller only has
4150 to emit a single comparison insn. This setup can involve doing a BLKmode
4151 comparison or emitting a library call to perform the comparison if no insn
4152 is available to handle it.
4153 The values which are passed in through pointers can be modified; the caller
4154 should perform the comparison on the modified values. Constant
4155 comparisons must have already been folded. */
4158 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4159 int unsignedp
, enum optab_methods methods
,
4160 rtx
*ptest
, machine_mode
*pmode
)
4162 machine_mode mode
= *pmode
;
4164 machine_mode cmp_mode
;
4165 enum mode_class mclass
;
4167 /* The other methods are not needed. */
4168 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
4169 || methods
== OPTAB_LIB_WIDEN
);
4171 if (CONST_SCALAR_INT_P (y
))
4172 canonicalize_comparison (mode
, &comparison
, &y
);
4174 /* If we are optimizing, force expensive constants into a register. */
4175 if (CONSTANT_P (x
) && optimize
4176 && (rtx_cost (x
, mode
, COMPARE
, 0, optimize_insn_for_speed_p ())
4177 > COSTS_N_INSNS (1)))
4178 x
= force_reg (mode
, x
);
4180 if (CONSTANT_P (y
) && optimize
4181 && (rtx_cost (y
, mode
, COMPARE
, 1, optimize_insn_for_speed_p ())
4182 > COSTS_N_INSNS (1)))
4183 y
= force_reg (mode
, y
);
4186 /* Make sure if we have a canonical comparison. The RTL
4187 documentation states that canonical comparisons are required only
4188 for targets which have cc0. */
4189 gcc_assert (!CONSTANT_P (x
) || CONSTANT_P (y
));
4192 /* Don't let both operands fail to indicate the mode. */
4193 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
4194 x
= force_reg (mode
, x
);
4195 if (mode
== VOIDmode
)
4196 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
4198 /* Handle all BLKmode compares. */
4200 if (mode
== BLKmode
)
4202 machine_mode result_mode
;
4203 enum insn_code cmp_code
;
4206 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
4210 /* Try to use a memory block compare insn - either cmpstr
4211 or cmpmem will do. */
4212 opt_scalar_int_mode cmp_mode_iter
;
4213 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
4215 scalar_int_mode cmp_mode
= cmp_mode_iter
.require ();
4216 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
4217 if (cmp_code
== CODE_FOR_nothing
)
4218 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
4219 if (cmp_code
== CODE_FOR_nothing
)
4220 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
4221 if (cmp_code
== CODE_FOR_nothing
)
4224 /* Must make sure the size fits the insn's mode. */
4225 if (CONST_INT_P (size
)
4226 ? UINTVAL (size
) > GET_MODE_MASK (cmp_mode
)
4227 : (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (size
)))
4228 > GET_MODE_BITSIZE (cmp_mode
)))
4231 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
4232 result
= gen_reg_rtx (result_mode
);
4233 size
= convert_to_mode (cmp_mode
, size
, 1);
4234 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
4236 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
4237 *pmode
= result_mode
;
4241 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
4244 /* Otherwise call a library function. */
4245 result
= emit_block_comp_via_libcall (x
, y
, size
);
4249 mode
= TYPE_MODE (integer_type_node
);
4250 methods
= OPTAB_LIB_WIDEN
;
4254 /* Don't allow operands to the compare to trap, as that can put the
4255 compare and branch in different basic blocks. */
4256 if (cfun
->can_throw_non_call_exceptions
)
4259 x
= copy_to_reg (x
);
4261 y
= copy_to_reg (y
);
4264 if (GET_MODE_CLASS (mode
) == MODE_CC
)
4266 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
4267 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4268 gcc_assert (icode
!= CODE_FOR_nothing
4269 && insn_operand_matches (icode
, 0, test
));
4274 mclass
= GET_MODE_CLASS (mode
);
4275 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4276 FOR_EACH_MODE_FROM (cmp_mode
, mode
)
4278 enum insn_code icode
;
4279 icode
= optab_handler (cbranch_optab
, cmp_mode
);
4280 if (icode
!= CODE_FOR_nothing
4281 && insn_operand_matches (icode
, 0, test
))
4283 rtx_insn
*last
= get_last_insn ();
4284 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
4285 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
4287 && insn_operand_matches (icode
, 1, op0
)
4288 && insn_operand_matches (icode
, 2, op1
))
4290 XEXP (test
, 0) = op0
;
4291 XEXP (test
, 1) = op1
;
4296 delete_insns_since (last
);
4299 if (methods
== OPTAB_DIRECT
|| !CLASS_HAS_WIDER_MODES_P (mclass
))
4303 if (methods
!= OPTAB_LIB_WIDEN
)
4306 if (SCALAR_FLOAT_MODE_P (mode
))
4308 /* Small trick if UNORDERED isn't implemented by the hardware. */
4309 if (comparison
== UNORDERED
&& rtx_equal_p (x
, y
))
4311 prepare_cmp_insn (x
, y
, UNLT
, NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4317 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
4322 machine_mode ret_mode
;
4324 /* Handle a libcall just for the mode we are using. */
4325 libfunc
= optab_libfunc (cmp_optab
, mode
);
4326 gcc_assert (libfunc
);
4328 /* If we want unsigned, and this mode has a distinct unsigned
4329 comparison routine, use that. */
4332 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
4337 ret_mode
= targetm
.libgcc_cmp_return_mode ();
4338 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4339 ret_mode
, x
, mode
, y
, mode
);
4341 /* There are two kinds of comparison routines. Biased routines
4342 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4343 of gcc expect that the comparison operation is equivalent
4344 to the modified comparison. For signed comparisons compare the
4345 result against 1 in the biased case, and zero in the unbiased
4346 case. For unsigned comparisons always compare against 1 after
4347 biasing the unbiased result by adding 1. This gives us a way to
4349 The comparisons in the fixed-point helper library are always
4354 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
4357 x
= plus_constant (ret_mode
, result
, 1);
4363 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
4373 /* Before emitting an insn with code ICODE, make sure that X, which is going
4374 to be used for operand OPNUM of the insn, is converted from mode MODE to
4375 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4376 that it is accepted by the operand predicate. Return the new value. */
4379 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
4380 machine_mode wider_mode
, int unsignedp
)
4382 if (mode
!= wider_mode
)
4383 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
4385 if (!insn_operand_matches (icode
, opnum
, x
))
4387 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
4388 if (reload_completed
)
4390 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
4392 x
= copy_to_mode_reg (op_mode
, x
);
4398 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4399 we can do the branch. */
4402 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
,
4403 profile_probability prob
)
4405 machine_mode optab_mode
;
4406 enum mode_class mclass
;
4407 enum insn_code icode
;
4410 mclass
= GET_MODE_CLASS (mode
);
4411 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
4412 icode
= optab_handler (cbranch_optab
, optab_mode
);
4414 gcc_assert (icode
!= CODE_FOR_nothing
);
4415 gcc_assert (insn_operand_matches (icode
, 0, test
));
4416 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
4417 XEXP (test
, 1), label
));
4418 if (prob
.initialized_p ()
4419 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
4422 && any_condjump_p (insn
)
4423 && !find_reg_note (insn
, REG_BR_PROB
, 0))
4424 add_reg_br_prob_note (insn
, prob
);
4427 /* Generate code to compare X with Y so that the condition codes are
4428 set and to jump to LABEL if the condition is true. If X is a
4429 constant and Y is not a constant, then the comparison is swapped to
4430 ensure that the comparison RTL has the canonical form.
4432 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4433 need to be widened. UNSIGNEDP is also used to select the proper
4434 branch condition code.
4436 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4438 MODE is the mode of the inputs (in case they are const_int).
4440 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4441 It will be potentially converted into an unsigned variant based on
4442 UNSIGNEDP to select a proper jump instruction.
4444 PROB is the probability of jumping to LABEL. */
4447 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4448 machine_mode mode
, int unsignedp
, rtx label
,
4449 profile_probability prob
)
4451 rtx op0
= x
, op1
= y
;
4454 /* Swap operands and condition to ensure canonical RTL. */
4455 if (swap_commutative_operands_p (x
, y
)
4456 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4459 comparison
= swap_condition (comparison
);
4462 /* If OP0 is still a constant, then both X and Y must be constants
4463 or the opposite comparison is not supported. Force X into a register
4464 to create canonical RTL. */
4465 if (CONSTANT_P (op0
))
4466 op0
= force_reg (mode
, op0
);
4469 comparison
= unsigned_condition (comparison
);
4471 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4473 emit_cmp_and_jump_insn_1 (test
, mode
, label
, prob
);
4477 /* Emit a library call comparison between floating point X and Y.
4478 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4481 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4482 rtx
*ptest
, machine_mode
*pmode
)
4484 enum rtx_code swapped
= swap_condition (comparison
);
4485 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4486 machine_mode orig_mode
= GET_MODE (x
);
4488 rtx true_rtx
, false_rtx
;
4489 rtx value
, target
, equiv
;
4492 bool reversed_p
= false;
4493 scalar_int_mode cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4495 FOR_EACH_MODE_FROM (mode
, orig_mode
)
4497 if (code_to_optab (comparison
)
4498 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4501 if (code_to_optab (swapped
)
4502 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4505 comparison
= swapped
;
4509 if (code_to_optab (reversed
)
4510 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4512 comparison
= reversed
;
4518 gcc_assert (mode
!= VOIDmode
);
4520 if (mode
!= orig_mode
)
4522 x
= convert_to_mode (mode
, x
, 0);
4523 y
= convert_to_mode (mode
, y
, 0);
4526 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4527 the RTL. The allows the RTL optimizers to delete the libcall if the
4528 condition can be determined at compile-time. */
4529 if (comparison
== UNORDERED
4530 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4532 true_rtx
= const_true_rtx
;
4533 false_rtx
= const0_rtx
;
4540 true_rtx
= const0_rtx
;
4541 false_rtx
= const_true_rtx
;
4545 true_rtx
= const_true_rtx
;
4546 false_rtx
= const0_rtx
;
4550 true_rtx
= const1_rtx
;
4551 false_rtx
= const0_rtx
;
4555 true_rtx
= const0_rtx
;
4556 false_rtx
= constm1_rtx
;
4560 true_rtx
= constm1_rtx
;
4561 false_rtx
= const0_rtx
;
4565 true_rtx
= const0_rtx
;
4566 false_rtx
= const1_rtx
;
4574 if (comparison
== UNORDERED
)
4576 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4577 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4578 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4579 temp
, const_true_rtx
, equiv
);
4583 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4584 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4585 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4586 equiv
, true_rtx
, false_rtx
);
4590 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4591 cmp_mode
, x
, mode
, y
, mode
);
4592 insns
= get_insns ();
4595 target
= gen_reg_rtx (cmp_mode
);
4596 emit_libcall_block (insns
, target
, value
, equiv
);
4598 if (comparison
== UNORDERED
4599 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
4601 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
4603 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
4608 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4611 emit_indirect_jump (rtx loc
)
4613 if (!targetm
.have_indirect_jump ())
4614 sorry ("indirect jumps are not available on this target");
4617 class expand_operand ops
[1];
4618 create_address_operand (&ops
[0], loc
);
4619 expand_jump_insn (targetm
.code_for_indirect_jump
, 1, ops
);
4625 /* Emit a conditional move instruction if the machine supports one for that
4626 condition and machine mode.
4628 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4629 the mode to use should they be constants. If it is VOIDmode, they cannot
4632 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4633 should be stored there. MODE is the mode to use should they be constants.
4634 If it is VOIDmode, they cannot both be constants.
4636 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4637 is not supported. */
4640 emit_conditional_move (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4641 machine_mode cmode
, rtx op2
, rtx op3
,
4642 machine_mode mode
, int unsignedp
)
4646 enum insn_code icode
;
4647 enum rtx_code reversed
;
4649 /* If the two source operands are identical, that's just a move. */
4651 if (rtx_equal_p (op2
, op3
))
4654 target
= gen_reg_rtx (mode
);
4656 emit_move_insn (target
, op3
);
4660 /* If one operand is constant, make it the second one. Only do this
4661 if the other operand is not constant as well. */
4663 if (swap_commutative_operands_p (op0
, op1
))
4665 std::swap (op0
, op1
);
4666 code
= swap_condition (code
);
4669 /* get_condition will prefer to generate LT and GT even if the old
4670 comparison was against zero, so undo that canonicalization here since
4671 comparisons against zero are cheaper. */
4672 if (code
== LT
&& op1
== const1_rtx
)
4673 code
= LE
, op1
= const0_rtx
;
4674 else if (code
== GT
&& op1
== constm1_rtx
)
4675 code
= GE
, op1
= const0_rtx
;
4677 if (cmode
== VOIDmode
)
4678 cmode
= GET_MODE (op0
);
4680 enum rtx_code orig_code
= code
;
4681 bool swapped
= false;
4682 if (swap_commutative_operands_p (op2
, op3
)
4683 && ((reversed
= reversed_comparison_code_parts (code
, op0
, op1
, NULL
))
4686 std::swap (op2
, op3
);
4691 if (mode
== VOIDmode
)
4692 mode
= GET_MODE (op2
);
4694 icode
= direct_optab_handler (movcc_optab
, mode
);
4696 if (icode
== CODE_FOR_nothing
)
4700 target
= gen_reg_rtx (mode
);
4702 for (int pass
= 0; ; pass
++)
4704 code
= unsignedp
? unsigned_condition (code
) : code
;
4705 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4707 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4708 punt and let the caller figure out how best to deal with this
4710 if (COMPARISON_P (comparison
))
4712 saved_pending_stack_adjust save
;
4713 save_pending_stack_adjust (&save
);
4714 last
= get_last_insn ();
4715 do_pending_stack_adjust ();
4716 machine_mode cmpmode
= cmode
;
4717 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4718 GET_CODE (comparison
), NULL_RTX
, unsignedp
,
4719 OPTAB_WIDEN
, &comparison
, &cmpmode
);
4722 class expand_operand ops
[4];
4724 create_output_operand (&ops
[0], target
, mode
);
4725 create_fixed_operand (&ops
[1], comparison
);
4726 create_input_operand (&ops
[2], op2
, mode
);
4727 create_input_operand (&ops
[3], op3
, mode
);
4728 if (maybe_expand_insn (icode
, 4, ops
))
4730 if (ops
[0].value
!= target
)
4731 convert_move (target
, ops
[0].value
, false);
4735 delete_insns_since (last
);
4736 restore_pending_stack_adjust (&save
);
4742 /* If the preferred op2/op3 order is not usable, retry with other
4743 operand order, perhaps it will expand successfully. */
4746 else if ((reversed
= reversed_comparison_code_parts (orig_code
, op0
, op1
,
4752 std::swap (op2
, op3
);
4757 /* Emit a conditional negate or bitwise complement using the
4758 negcc or notcc optabs if available. Return NULL_RTX if such operations
4759 are not available. Otherwise return the RTX holding the result.
4760 TARGET is the desired destination of the result. COMP is the comparison
4761 on which to negate. If COND is true move into TARGET the negation
4762 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
4763 CODE is either NEG or NOT. MODE is the machine mode in which the
4764 operation is performed. */
4767 emit_conditional_neg_or_complement (rtx target
, rtx_code code
,
4768 machine_mode mode
, rtx cond
, rtx op1
,
4771 optab op
= unknown_optab
;
4774 else if (code
== NOT
)
4779 insn_code icode
= direct_optab_handler (op
, mode
);
4781 if (icode
== CODE_FOR_nothing
)
4785 target
= gen_reg_rtx (mode
);
4787 rtx_insn
*last
= get_last_insn ();
4788 class expand_operand ops
[4];
4790 create_output_operand (&ops
[0], target
, mode
);
4791 create_fixed_operand (&ops
[1], cond
);
4792 create_input_operand (&ops
[2], op1
, mode
);
4793 create_input_operand (&ops
[3], op2
, mode
);
4795 if (maybe_expand_insn (icode
, 4, ops
))
4797 if (ops
[0].value
!= target
)
4798 convert_move (target
, ops
[0].value
, false);
4802 delete_insns_since (last
);
4806 /* Emit a conditional addition instruction if the machine supports one for that
4807 condition and machine mode.
4809 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4810 the mode to use should they be constants. If it is VOIDmode, they cannot
4813 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4814 should be stored there. MODE is the mode to use should they be constants.
4815 If it is VOIDmode, they cannot both be constants.
4817 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4818 is not supported. */
4821 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4822 machine_mode cmode
, rtx op2
, rtx op3
,
4823 machine_mode mode
, int unsignedp
)
4827 enum insn_code icode
;
4829 /* If one operand is constant, make it the second one. Only do this
4830 if the other operand is not constant as well. */
4832 if (swap_commutative_operands_p (op0
, op1
))
4834 std::swap (op0
, op1
);
4835 code
= swap_condition (code
);
4838 /* get_condition will prefer to generate LT and GT even if the old
4839 comparison was against zero, so undo that canonicalization here since
4840 comparisons against zero are cheaper. */
4841 if (code
== LT
&& op1
== const1_rtx
)
4842 code
= LE
, op1
= const0_rtx
;
4843 else if (code
== GT
&& op1
== constm1_rtx
)
4844 code
= GE
, op1
= const0_rtx
;
4846 if (cmode
== VOIDmode
)
4847 cmode
= GET_MODE (op0
);
4849 if (mode
== VOIDmode
)
4850 mode
= GET_MODE (op2
);
4852 icode
= optab_handler (addcc_optab
, mode
);
4854 if (icode
== CODE_FOR_nothing
)
4858 target
= gen_reg_rtx (mode
);
4860 code
= unsignedp
? unsigned_condition (code
) : code
;
4861 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4863 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4864 return NULL and let the caller figure out how best to deal with this
4866 if (!COMPARISON_P (comparison
))
4869 do_pending_stack_adjust ();
4870 last
= get_last_insn ();
4871 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4872 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4873 &comparison
, &cmode
);
4876 class expand_operand ops
[4];
4878 create_output_operand (&ops
[0], target
, mode
);
4879 create_fixed_operand (&ops
[1], comparison
);
4880 create_input_operand (&ops
[2], op2
, mode
);
4881 create_input_operand (&ops
[3], op3
, mode
);
4882 if (maybe_expand_insn (icode
, 4, ops
))
4884 if (ops
[0].value
!= target
)
4885 convert_move (target
, ops
[0].value
, false);
4889 delete_insns_since (last
);
4893 /* These functions attempt to generate an insn body, rather than
4894 emitting the insn, but if the gen function already emits them, we
4895 make no attempt to turn them back into naked patterns. */
4897 /* Generate and return an insn body to add Y to X. */
4900 gen_add2_insn (rtx x
, rtx y
)
4902 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
4904 gcc_assert (insn_operand_matches (icode
, 0, x
));
4905 gcc_assert (insn_operand_matches (icode
, 1, x
));
4906 gcc_assert (insn_operand_matches (icode
, 2, y
));
4908 return GEN_FCN (icode
) (x
, x
, y
);
4911 /* Generate and return an insn body to add r1 and c,
4912 storing the result in r0. */
4915 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
4917 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
4919 if (icode
== CODE_FOR_nothing
4920 || !insn_operand_matches (icode
, 0, r0
)
4921 || !insn_operand_matches (icode
, 1, r1
)
4922 || !insn_operand_matches (icode
, 2, c
))
4925 return GEN_FCN (icode
) (r0
, r1
, c
);
4929 have_add2_insn (rtx x
, rtx y
)
4931 enum insn_code icode
;
4933 gcc_assert (GET_MODE (x
) != VOIDmode
);
4935 icode
= optab_handler (add_optab
, GET_MODE (x
));
4937 if (icode
== CODE_FOR_nothing
)
4940 if (!insn_operand_matches (icode
, 0, x
)
4941 || !insn_operand_matches (icode
, 1, x
)
4942 || !insn_operand_matches (icode
, 2, y
))
4948 /* Generate and return an insn body to add Y to X. */
4951 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
4953 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4955 gcc_assert (insn_operand_matches (icode
, 0, x
));
4956 gcc_assert (insn_operand_matches (icode
, 1, y
));
4957 gcc_assert (insn_operand_matches (icode
, 2, z
));
4959 return GEN_FCN (icode
) (x
, y
, z
);
4962 /* Return true if the target implements an addptr pattern and X, Y,
4963 and Z are valid for the pattern predicates. */
4966 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
4968 enum insn_code icode
;
4970 gcc_assert (GET_MODE (x
) != VOIDmode
);
4972 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4974 if (icode
== CODE_FOR_nothing
)
4977 if (!insn_operand_matches (icode
, 0, x
)
4978 || !insn_operand_matches (icode
, 1, y
)
4979 || !insn_operand_matches (icode
, 2, z
))
4985 /* Generate and return an insn body to subtract Y from X. */
4988 gen_sub2_insn (rtx x
, rtx y
)
4990 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
4992 gcc_assert (insn_operand_matches (icode
, 0, x
));
4993 gcc_assert (insn_operand_matches (icode
, 1, x
));
4994 gcc_assert (insn_operand_matches (icode
, 2, y
));
4996 return GEN_FCN (icode
) (x
, x
, y
);
4999 /* Generate and return an insn body to subtract r1 and c,
5000 storing the result in r0. */
5003 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
5005 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
5007 if (icode
== CODE_FOR_nothing
5008 || !insn_operand_matches (icode
, 0, r0
)
5009 || !insn_operand_matches (icode
, 1, r1
)
5010 || !insn_operand_matches (icode
, 2, c
))
5013 return GEN_FCN (icode
) (r0
, r1
, c
);
5017 have_sub2_insn (rtx x
, rtx y
)
5019 enum insn_code icode
;
5021 gcc_assert (GET_MODE (x
) != VOIDmode
);
5023 icode
= optab_handler (sub_optab
, GET_MODE (x
));
5025 if (icode
== CODE_FOR_nothing
)
5028 if (!insn_operand_matches (icode
, 0, x
)
5029 || !insn_operand_matches (icode
, 1, x
)
5030 || !insn_operand_matches (icode
, 2, y
))
5036 /* Generate the body of an insn to extend Y (with mode MFROM)
5037 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5040 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
5041 machine_mode mfrom
, int unsignedp
)
5043 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
5044 return GEN_FCN (icode
) (x
, y
);
5047 /* Generate code to convert FROM to floating point
5048 and store in TO. FROM must be fixed point and not VOIDmode.
5049 UNSIGNEDP nonzero means regard FROM as unsigned.
5050 Normally this is done by correcting the final value
5051 if it is negative. */
5054 expand_float (rtx to
, rtx from
, int unsignedp
)
5056 enum insn_code icode
;
5058 scalar_mode from_mode
, to_mode
;
5059 machine_mode fmode
, imode
;
5060 bool can_do_signed
= false;
5062 /* Crash now, because we won't be able to decide which mode to use. */
5063 gcc_assert (GET_MODE (from
) != VOIDmode
);
5065 /* Look for an insn to do the conversion. Do it in the specified
5066 modes if possible; otherwise convert either input, output or both to
5067 wider mode. If the integer mode is wider than the mode of FROM,
5068 we can do the conversion signed even if the input is unsigned. */
5070 FOR_EACH_MODE_FROM (fmode
, GET_MODE (to
))
5071 FOR_EACH_MODE_FROM (imode
, GET_MODE (from
))
5073 int doing_unsigned
= unsignedp
;
5075 if (fmode
!= GET_MODE (to
)
5076 && (significand_size (fmode
)
5077 < GET_MODE_UNIT_PRECISION (GET_MODE (from
))))
5080 icode
= can_float_p (fmode
, imode
, unsignedp
);
5081 if (icode
== CODE_FOR_nothing
&& unsignedp
)
5083 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
5084 if (scode
!= CODE_FOR_nothing
)
5085 can_do_signed
= true;
5086 if (imode
!= GET_MODE (from
))
5087 icode
= scode
, doing_unsigned
= 0;
5090 if (icode
!= CODE_FOR_nothing
)
5092 if (imode
!= GET_MODE (from
))
5093 from
= convert_to_mode (imode
, from
, unsignedp
);
5095 if (fmode
!= GET_MODE (to
))
5096 target
= gen_reg_rtx (fmode
);
5098 emit_unop_insn (icode
, target
, from
,
5099 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
5102 convert_move (to
, target
, 0);
5107 /* Unsigned integer, and no way to convert directly. Convert as signed,
5108 then unconditionally adjust the result. */
5111 && is_a
<scalar_mode
> (GET_MODE (to
), &to_mode
)
5112 && is_a
<scalar_mode
> (GET_MODE (from
), &from_mode
))
5114 opt_scalar_mode fmode_iter
;
5115 rtx_code_label
*label
= gen_label_rtx ();
5117 REAL_VALUE_TYPE offset
;
5119 /* Look for a usable floating mode FMODE wider than the source and at
5120 least as wide as the target. Using FMODE will avoid rounding woes
5121 with unsigned values greater than the signed maximum value. */
5123 FOR_EACH_MODE_FROM (fmode_iter
, to_mode
)
5125 scalar_mode fmode
= fmode_iter
.require ();
5126 if (GET_MODE_PRECISION (from_mode
) < GET_MODE_BITSIZE (fmode
)
5127 && can_float_p (fmode
, from_mode
, 0) != CODE_FOR_nothing
)
5131 if (!fmode_iter
.exists (&fmode
))
5133 /* There is no such mode. Pretend the target is wide enough. */
5136 /* Avoid double-rounding when TO is narrower than FROM. */
5137 if ((significand_size (fmode
) + 1)
5138 < GET_MODE_PRECISION (from_mode
))
5141 rtx_code_label
*neglabel
= gen_label_rtx ();
5143 /* Don't use TARGET if it isn't a register, is a hard register,
5144 or is the wrong mode. */
5146 || REGNO (target
) < FIRST_PSEUDO_REGISTER
5147 || GET_MODE (target
) != fmode
)
5148 target
= gen_reg_rtx (fmode
);
5151 do_pending_stack_adjust ();
5153 /* Test whether the sign bit is set. */
5154 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
5157 /* The sign bit is not set. Convert as signed. */
5158 expand_float (target
, from
, 0);
5159 emit_jump_insn (targetm
.gen_jump (label
));
5162 /* The sign bit is set.
5163 Convert to a usable (positive signed) value by shifting right
5164 one bit, while remembering if a nonzero bit was shifted
5165 out; i.e., compute (from & 1) | (from >> 1). */
5167 emit_label (neglabel
);
5168 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
5169 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
5170 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
5171 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
5173 expand_float (target
, temp
, 0);
5175 /* Multiply by 2 to undo the shift above. */
5176 temp
= expand_binop (fmode
, add_optab
, target
, target
,
5177 target
, 0, OPTAB_LIB_WIDEN
);
5179 emit_move_insn (target
, temp
);
5181 do_pending_stack_adjust ();
5187 /* If we are about to do some arithmetic to correct for an
5188 unsigned operand, do it in a pseudo-register. */
5190 if (to_mode
!= fmode
5191 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
5192 target
= gen_reg_rtx (fmode
);
5194 /* Convert as signed integer to floating. */
5195 expand_float (target
, from
, 0);
5197 /* If FROM is negative (and therefore TO is negative),
5198 correct its value by 2**bitwidth. */
5200 do_pending_stack_adjust ();
5201 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, from_mode
,
5205 real_2expN (&offset
, GET_MODE_PRECISION (from_mode
), fmode
);
5206 temp
= expand_binop (fmode
, add_optab
, target
,
5207 const_double_from_real_value (offset
, fmode
),
5208 target
, 0, OPTAB_LIB_WIDEN
);
5210 emit_move_insn (target
, temp
);
5212 do_pending_stack_adjust ();
5217 /* No hardware instruction available; call a library routine. */
5222 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
5224 if (is_narrower_int_mode (GET_MODE (from
), SImode
))
5225 from
= convert_to_mode (SImode
, from
, unsignedp
);
5227 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5228 gcc_assert (libfunc
);
5232 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5233 GET_MODE (to
), from
, GET_MODE (from
));
5234 insns
= get_insns ();
5237 emit_libcall_block (insns
, target
, value
,
5238 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
5239 GET_MODE (to
), from
));
5244 /* Copy result to requested destination
5245 if we have been computing in a temp location. */
5249 if (GET_MODE (target
) == GET_MODE (to
))
5250 emit_move_insn (to
, target
);
5252 convert_move (to
, target
, 0);
5256 /* Generate code to convert FROM to fixed point and store in TO. FROM
5257 must be floating point. */
5260 expand_fix (rtx to
, rtx from
, int unsignedp
)
5262 enum insn_code icode
;
5264 machine_mode fmode
, imode
;
5265 opt_scalar_mode fmode_iter
;
5266 bool must_trunc
= false;
5268 /* We first try to find a pair of modes, one real and one integer, at
5269 least as wide as FROM and TO, respectively, in which we can open-code
5270 this conversion. If the integer mode is wider than the mode of TO,
5271 we can do the conversion either signed or unsigned. */
5273 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5274 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5276 int doing_unsigned
= unsignedp
;
5278 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
5279 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
5280 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
5282 if (icode
!= CODE_FOR_nothing
)
5284 rtx_insn
*last
= get_last_insn ();
5285 if (fmode
!= GET_MODE (from
))
5286 from
= convert_to_mode (fmode
, from
, 0);
5290 rtx temp
= gen_reg_rtx (GET_MODE (from
));
5291 from
= expand_unop (GET_MODE (from
), ftrunc_optab
, from
,
5295 if (imode
!= GET_MODE (to
))
5296 target
= gen_reg_rtx (imode
);
5298 if (maybe_emit_unop_insn (icode
, target
, from
,
5299 doing_unsigned
? UNSIGNED_FIX
: FIX
))
5302 convert_move (to
, target
, unsignedp
);
5305 delete_insns_since (last
);
5309 /* For an unsigned conversion, there is one more way to do it.
5310 If we have a signed conversion, we generate code that compares
5311 the real value to the largest representable positive number. If if
5312 is smaller, the conversion is done normally. Otherwise, subtract
5313 one plus the highest signed number, convert, and add it back.
5315 We only need to check all real modes, since we know we didn't find
5316 anything with a wider integer mode.
5318 This code used to extend FP value into mode wider than the destination.
5319 This is needed for decimal float modes which cannot accurately
5320 represent one plus the highest signed number of the same size, but
5321 not for binary modes. Consider, for instance conversion from SFmode
5324 The hot path through the code is dealing with inputs smaller than 2^63
5325 and doing just the conversion, so there is no bits to lose.
5327 In the other path we know the value is positive in the range 2^63..2^64-1
5328 inclusive. (as for other input overflow happens and result is undefined)
5329 So we know that the most important bit set in mantissa corresponds to
5330 2^63. The subtraction of 2^63 should not generate any rounding as it
5331 simply clears out that bit. The rest is trivial. */
5333 scalar_int_mode to_mode
;
5335 && is_a
<scalar_int_mode
> (GET_MODE (to
), &to_mode
)
5336 && HWI_COMPUTABLE_MODE_P (to_mode
))
5337 FOR_EACH_MODE_FROM (fmode_iter
, as_a
<scalar_mode
> (GET_MODE (from
)))
5339 scalar_mode fmode
= fmode_iter
.require ();
5340 if (CODE_FOR_nothing
!= can_fix_p (to_mode
, fmode
,
5342 && (!DECIMAL_FLOAT_MODE_P (fmode
)
5343 || (GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (to_mode
))))
5346 REAL_VALUE_TYPE offset
;
5348 rtx_code_label
*lab1
, *lab2
;
5351 bitsize
= GET_MODE_PRECISION (to_mode
);
5352 real_2expN (&offset
, bitsize
- 1, fmode
);
5353 limit
= const_double_from_real_value (offset
, fmode
);
5354 lab1
= gen_label_rtx ();
5355 lab2
= gen_label_rtx ();
5357 if (fmode
!= GET_MODE (from
))
5358 from
= convert_to_mode (fmode
, from
, 0);
5360 /* See if we need to do the subtraction. */
5361 do_pending_stack_adjust ();
5362 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
,
5363 GET_MODE (from
), 0, lab1
);
5365 /* If not, do the signed "fix" and branch around fixup code. */
5366 expand_fix (to
, from
, 0);
5367 emit_jump_insn (targetm
.gen_jump (lab2
));
5370 /* Otherwise, subtract 2**(N-1), convert to signed number,
5371 then add 2**(N-1). Do the addition using XOR since this
5372 will often generate better code. */
5374 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
5375 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
5376 expand_fix (to
, target
, 0);
5377 target
= expand_binop (to_mode
, xor_optab
, to
,
5379 (HOST_WIDE_INT_1
<< (bitsize
- 1),
5381 to
, 1, OPTAB_LIB_WIDEN
);
5384 emit_move_insn (to
, target
);
5388 if (optab_handler (mov_optab
, to_mode
) != CODE_FOR_nothing
)
5390 /* Make a place for a REG_NOTE and add it. */
5391 insn
= emit_move_insn (to
, to
);
5392 set_dst_reg_note (insn
, REG_EQUAL
,
5393 gen_rtx_fmt_e (UNSIGNED_FIX
, to_mode
,
5402 /* We can't do it with an insn, so use a library call. But first ensure
5403 that the mode of TO is at least as wide as SImode, since those are the
5404 only library calls we know about. */
5406 if (is_narrower_int_mode (GET_MODE (to
), SImode
))
5408 target
= gen_reg_rtx (SImode
);
5410 expand_fix (target
, from
, unsignedp
);
5418 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
5419 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5420 gcc_assert (libfunc
);
5424 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5425 GET_MODE (to
), from
, GET_MODE (from
));
5426 insns
= get_insns ();
5429 emit_libcall_block (insns
, target
, value
,
5430 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
5431 GET_MODE (to
), from
));
5436 if (GET_MODE (to
) == GET_MODE (target
))
5437 emit_move_insn (to
, target
);
5439 convert_move (to
, target
, 0);
5444 /* Promote integer arguments for a libcall if necessary.
5445 emit_library_call_value cannot do the promotion because it does not
5446 know if it should do a signed or unsigned promotion. This is because
5447 there are no tree types defined for libcalls. */
5450 prepare_libcall_arg (rtx arg
, int uintp
)
5452 scalar_int_mode mode
;
5453 machine_mode arg_mode
;
5454 if (is_a
<scalar_int_mode
> (GET_MODE (arg
), &mode
))
5456 /* If we need to promote the integer function argument we need to do
5457 it here instead of inside emit_library_call_value because in
5458 emit_library_call_value we don't know if we should do a signed or
5459 unsigned promotion. */
5462 arg_mode
= promote_function_mode (NULL_TREE
, mode
,
5463 &unsigned_p
, NULL_TREE
, 0);
5464 if (arg_mode
!= mode
)
5465 return convert_to_mode (arg_mode
, arg
, uintp
);
5470 /* Generate code to convert FROM or TO a fixed-point.
5471 If UINTP is true, either TO or FROM is an unsigned integer.
5472 If SATP is true, we need to saturate the result. */
5475 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
5477 machine_mode to_mode
= GET_MODE (to
);
5478 machine_mode from_mode
= GET_MODE (from
);
5480 enum rtx_code this_code
;
5481 enum insn_code code
;
5486 if (to_mode
== from_mode
)
5488 emit_move_insn (to
, from
);
5494 tab
= satp
? satfractuns_optab
: fractuns_optab
;
5495 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
5499 tab
= satp
? satfract_optab
: fract_optab
;
5500 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
5502 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
5503 if (code
!= CODE_FOR_nothing
)
5505 emit_unop_insn (code
, to
, from
, this_code
);
5509 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
5510 gcc_assert (libfunc
);
5512 from
= prepare_libcall_arg (from
, uintp
);
5513 from_mode
= GET_MODE (from
);
5516 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
5518 insns
= get_insns ();
5521 emit_libcall_block (insns
, to
, value
,
5522 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
5525 /* Generate code to convert FROM to fixed point and store in TO. FROM
5526 must be floating point, TO must be signed. Use the conversion optab
5527 TAB to do the conversion. */
5530 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
5532 enum insn_code icode
;
5534 machine_mode fmode
, imode
;
5536 /* We first try to find a pair of modes, one real and one integer, at
5537 least as wide as FROM and TO, respectively, in which we can open-code
5538 this conversion. If the integer mode is wider than the mode of TO,
5539 we can do the conversion either signed or unsigned. */
5541 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5542 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5544 icode
= convert_optab_handler (tab
, imode
, fmode
);
5545 if (icode
!= CODE_FOR_nothing
)
5547 rtx_insn
*last
= get_last_insn ();
5548 if (fmode
!= GET_MODE (from
))
5549 from
= convert_to_mode (fmode
, from
, 0);
5551 if (imode
!= GET_MODE (to
))
5552 target
= gen_reg_rtx (imode
);
5554 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
5556 delete_insns_since (last
);
5560 convert_move (to
, target
, 0);
5568 /* Report whether we have an instruction to perform the operation
5569 specified by CODE on operands of mode MODE. */
5571 have_insn_for (enum rtx_code code
, machine_mode mode
)
5573 return (code_to_optab (code
)
5574 && (optab_handler (code_to_optab (code
), mode
)
5575 != CODE_FOR_nothing
));
5578 /* Print information about the current contents of the optabs on
5582 debug_optab_libfuncs (void)
5586 /* Dump the arithmetic optabs. */
5587 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
5588 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5590 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
5593 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5594 fprintf (stderr
, "%s\t%s:\t%s\n",
5595 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5601 /* Dump the conversion optabs. */
5602 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
5603 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5604 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
5606 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
5610 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5611 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
5612 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5620 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5621 CODE. Return 0 on failure. */
5624 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
5626 machine_mode mode
= GET_MODE (op1
);
5627 enum insn_code icode
;
5631 if (mode
== VOIDmode
)
5634 icode
= optab_handler (ctrap_optab
, mode
);
5635 if (icode
== CODE_FOR_nothing
)
5638 /* Some targets only accept a zero trap code. */
5639 if (!insn_operand_matches (icode
, 3, tcode
))
5642 do_pending_stack_adjust ();
5644 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
5649 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
5652 /* If that failed, then give up. */
5660 insn
= get_insns ();
5665 /* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
5666 or unsigned operation code. */
5669 get_rtx_code_1 (enum tree_code tcode
, bool unsignedp
)
5681 code
= unsignedp
? LTU
: LT
;
5684 code
= unsignedp
? LEU
: LE
;
5687 code
= unsignedp
? GTU
: GT
;
5690 code
= unsignedp
? GEU
: GE
;
5693 case UNORDERED_EXPR
:
5733 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5734 or unsigned operation code. */
5737 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
5739 enum rtx_code code
= get_rtx_code_1 (tcode
, unsignedp
);
5740 gcc_assert (code
!= UNKNOWN
);
5744 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
5745 select signed or unsigned operators. OPNO holds the index of the
5746 first comparison operand for insn ICODE. Do not generate the
5747 compare instruction itself. */
5750 vector_compare_rtx (machine_mode cmp_mode
, enum tree_code tcode
,
5751 tree t_op0
, tree t_op1
, bool unsignedp
,
5752 enum insn_code icode
, unsigned int opno
)
5754 class expand_operand ops
[2];
5755 rtx rtx_op0
, rtx_op1
;
5756 machine_mode m0
, m1
;
5757 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
5759 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
5761 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
5762 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5763 cases, use the original mode. */
5764 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
5766 m0
= GET_MODE (rtx_op0
);
5768 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
5770 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
5772 m1
= GET_MODE (rtx_op1
);
5774 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
5776 create_input_operand (&ops
[0], rtx_op0
, m0
);
5777 create_input_operand (&ops
[1], rtx_op1
, m1
);
5778 if (!maybe_legitimize_operands (icode
, opno
, 2, ops
))
5780 return gen_rtx_fmt_ee (rcode
, cmp_mode
, ops
[0].value
, ops
[1].value
);
5783 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
5784 the first vec_perm operand, assuming the second operand (for left shift
5785 first operand) is a constant vector of zeros. Return the shift distance
5786 in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
5787 mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
5788 shift or vec_shl_optab for left shift. */
5790 shift_amt_for_vec_perm_mask (machine_mode mode
, const vec_perm_indices
&sel
,
5793 unsigned int bitsize
= GET_MODE_UNIT_BITSIZE (mode
);
5794 poly_int64 first
= sel
[0];
5795 if (maybe_ge (sel
[0], GET_MODE_NUNITS (mode
)))
5798 if (shift_optab
== vec_shl_optab
)
5801 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
5803 unsigned firstidx
= 0;
5804 for (unsigned int i
= 0; i
< nelt
; i
++)
5806 if (known_eq (sel
[i
], nelt
))
5808 if (i
== 0 || firstidx
)
5813 ? maybe_ne (sel
[i
], nelt
+ i
- firstidx
)
5814 : maybe_ge (sel
[i
], nelt
))
5822 else if (!sel
.series_p (0, 1, first
, 1))
5825 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
5827 for (unsigned int i
= 1; i
< nelt
; i
++)
5829 poly_int64 expected
= i
+ first
;
5830 /* Indices into the second vector are all equivalent. */
5831 if (maybe_lt (sel
[i
], nelt
)
5832 ? maybe_ne (sel
[i
], expected
)
5833 : maybe_lt (expected
, nelt
))
5838 return gen_int_shift_amount (mode
, first
* bitsize
);
5841 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
5844 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
5845 rtx v0
, rtx v1
, rtx sel
)
5847 machine_mode tmode
= GET_MODE (target
);
5848 machine_mode smode
= GET_MODE (sel
);
5849 class expand_operand ops
[4];
5851 gcc_assert (GET_MODE_CLASS (smode
) == MODE_VECTOR_INT
5852 || related_int_vector_mode (tmode
).require () == smode
);
5853 create_output_operand (&ops
[0], target
, tmode
);
5854 create_input_operand (&ops
[3], sel
, smode
);
5856 /* Make an effort to preserve v0 == v1. The target expander is able to
5857 rely on this to determine if we're permuting a single input operand. */
5858 if (rtx_equal_p (v0
, v1
))
5860 if (!insn_operand_matches (icode
, 1, v0
))
5861 v0
= force_reg (tmode
, v0
);
5862 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
5863 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
5865 create_fixed_operand (&ops
[1], v0
);
5866 create_fixed_operand (&ops
[2], v0
);
5870 create_input_operand (&ops
[1], v0
, tmode
);
5871 create_input_operand (&ops
[2], v1
, tmode
);
5874 if (maybe_expand_insn (icode
, 4, ops
))
5875 return ops
[0].value
;
5879 /* Implement a permutation of vectors v0 and v1 using the permutation
5880 vector in SEL and return the result. Use TARGET to hold the result
5881 if nonnull and convenient.
5883 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
5884 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
5885 to have a particular mode. */
5888 expand_vec_perm_const (machine_mode mode
, rtx v0
, rtx v1
,
5889 const vec_perm_builder
&sel
, machine_mode sel_mode
,
5892 if (!target
|| !register_operand (target
, mode
))
5893 target
= gen_reg_rtx (mode
);
5895 /* Set QIMODE to a different vector mode with byte elements.
5896 If no such mode, or if MODE already has byte elements, use VOIDmode. */
5897 machine_mode qimode
;
5898 if (!qimode_for_vec_perm (mode
).exists (&qimode
))
5901 rtx_insn
*last
= get_last_insn ();
5903 bool single_arg_p
= rtx_equal_p (v0
, v1
);
5904 /* Always specify two input vectors here and leave the target to handle
5905 cases in which the inputs are equal. Not all backends can cope with
5906 the single-input representation when testing for a double-input
5907 target instruction. */
5908 vec_perm_indices
indices (sel
, 2, GET_MODE_NUNITS (mode
));
5910 /* See if this can be handled with a vec_shr or vec_shl. We only do this
5911 if the second (for vec_shr) or first (for vec_shl) vector is all
5913 insn_code shift_code
= CODE_FOR_nothing
;
5914 insn_code shift_code_qi
= CODE_FOR_nothing
;
5915 optab shift_optab
= unknown_optab
;
5917 if (v1
== CONST0_RTX (GET_MODE (v1
)))
5918 shift_optab
= vec_shr_optab
;
5919 else if (v0
== CONST0_RTX (GET_MODE (v0
)))
5921 shift_optab
= vec_shl_optab
;
5924 if (shift_optab
!= unknown_optab
)
5926 shift_code
= optab_handler (shift_optab
, mode
);
5927 shift_code_qi
= ((qimode
!= VOIDmode
&& qimode
!= mode
)
5928 ? optab_handler (shift_optab
, qimode
)
5929 : CODE_FOR_nothing
);
5931 if (shift_code
!= CODE_FOR_nothing
|| shift_code_qi
!= CODE_FOR_nothing
)
5933 rtx shift_amt
= shift_amt_for_vec_perm_mask (mode
, indices
, shift_optab
);
5936 class expand_operand ops
[3];
5937 if (shift_amt
== const0_rtx
)
5939 if (shift_code
!= CODE_FOR_nothing
)
5941 create_output_operand (&ops
[0], target
, mode
);
5942 create_input_operand (&ops
[1], v2
, mode
);
5943 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
5944 if (maybe_expand_insn (shift_code
, 3, ops
))
5945 return ops
[0].value
;
5947 if (shift_code_qi
!= CODE_FOR_nothing
)
5949 rtx tmp
= gen_reg_rtx (qimode
);
5950 create_output_operand (&ops
[0], tmp
, qimode
);
5951 create_input_operand (&ops
[1], gen_lowpart (qimode
, v2
), qimode
);
5952 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
5953 if (maybe_expand_insn (shift_code_qi
, 3, ops
))
5954 return gen_lowpart (mode
, ops
[0].value
);
5959 if (targetm
.vectorize
.vec_perm_const
!= NULL
)
5961 v0
= force_reg (mode
, v0
);
5965 v1
= force_reg (mode
, v1
);
5967 if (targetm
.vectorize
.vec_perm_const (mode
, target
, v0
, v1
, indices
))
5971 /* Fall back to a constant byte-based permutation. */
5972 vec_perm_indices qimode_indices
;
5973 rtx target_qi
= NULL_RTX
, v0_qi
= NULL_RTX
, v1_qi
= NULL_RTX
;
5974 if (qimode
!= VOIDmode
)
5976 qimode_indices
.new_expanded_vector (indices
, GET_MODE_UNIT_SIZE (mode
));
5977 target_qi
= gen_reg_rtx (qimode
);
5978 v0_qi
= gen_lowpart (qimode
, v0
);
5979 v1_qi
= gen_lowpart (qimode
, v1
);
5980 if (targetm
.vectorize
.vec_perm_const
!= NULL
5981 && targetm
.vectorize
.vec_perm_const (qimode
, target_qi
, v0_qi
,
5982 v1_qi
, qimode_indices
))
5983 return gen_lowpart (mode
, target_qi
);
5986 /* Otherwise expand as a fully variable permuation. */
5988 /* The optabs are only defined for selectors with the same width
5989 as the values being permuted. */
5990 machine_mode required_sel_mode
;
5991 if (!related_int_vector_mode (mode
).exists (&required_sel_mode
))
5993 delete_insns_since (last
);
5997 /* We know that it is semantically valid to treat SEL as having SEL_MODE.
5998 If that isn't the mode we want then we need to prove that using
5999 REQUIRED_SEL_MODE is OK. */
6000 if (sel_mode
!= required_sel_mode
)
6002 if (!selector_fits_mode_p (required_sel_mode
, indices
))
6004 delete_insns_since (last
);
6007 sel_mode
= required_sel_mode
;
6010 insn_code icode
= direct_optab_handler (vec_perm_optab
, mode
);
6011 if (icode
!= CODE_FOR_nothing
)
6013 rtx sel_rtx
= vec_perm_indices_to_rtx (sel_mode
, indices
);
6014 rtx tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel_rtx
);
6019 if (qimode
!= VOIDmode
6020 && selector_fits_mode_p (qimode
, qimode_indices
))
6022 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
6023 if (icode
!= CODE_FOR_nothing
)
6025 rtx sel_qi
= vec_perm_indices_to_rtx (qimode
, qimode_indices
);
6026 rtx tmp
= expand_vec_perm_1 (icode
, target_qi
, v0_qi
, v1_qi
, sel_qi
);
6028 return gen_lowpart (mode
, tmp
);
6032 delete_insns_since (last
);
6036 /* Implement a permutation of vectors v0 and v1 using the permutation
6037 vector in SEL and return the result. Use TARGET to hold the result
6038 if nonnull and convenient.
6040 MODE is the mode of the vectors being permuted (V0 and V1).
6041 SEL must have the integer equivalent of MODE and is known to be
6042 unsuitable for permutes with a constant permutation vector. */
6045 expand_vec_perm_var (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
6047 enum insn_code icode
;
6051 u
= GET_MODE_UNIT_SIZE (mode
);
6053 if (!target
|| GET_MODE (target
) != mode
)
6054 target
= gen_reg_rtx (mode
);
6056 icode
= direct_optab_handler (vec_perm_optab
, mode
);
6057 if (icode
!= CODE_FOR_nothing
)
6059 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
6064 /* As a special case to aid several targets, lower the element-based
6065 permutation to a byte-based permutation and try again. */
6066 machine_mode qimode
;
6067 if (!qimode_for_vec_perm (mode
).exists (&qimode
)
6068 || maybe_gt (GET_MODE_NUNITS (qimode
), GET_MODE_MASK (QImode
) + 1))
6070 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
6071 if (icode
== CODE_FOR_nothing
)
6074 /* Multiply each element by its byte size. */
6075 machine_mode selmode
= GET_MODE (sel
);
6077 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
6078 NULL
, 0, OPTAB_DIRECT
);
6080 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
6081 gen_int_shift_amount (selmode
, exact_log2 (u
)),
6082 NULL
, 0, OPTAB_DIRECT
);
6083 gcc_assert (sel
!= NULL
);
6085 /* Broadcast the low byte each element into each of its bytes.
6086 The encoding has U interleaved stepped patterns, one for each
6087 byte of an element. */
6088 vec_perm_builder
const_sel (GET_MODE_SIZE (mode
), u
, 3);
6089 unsigned int low_byte_in_u
= BYTES_BIG_ENDIAN
? u
- 1 : 0;
6090 for (i
= 0; i
< 3; ++i
)
6091 for (unsigned int j
= 0; j
< u
; ++j
)
6092 const_sel
.quick_push (i
* u
+ low_byte_in_u
);
6093 sel
= gen_lowpart (qimode
, sel
);
6094 sel
= expand_vec_perm_const (qimode
, sel
, sel
, const_sel
, qimode
, NULL
);
6095 gcc_assert (sel
!= NULL
);
6097 /* Add the byte offset to each byte element. */
6098 /* Note that the definition of the indicies here is memory ordering,
6099 so there should be no difference between big and little endian. */
6100 rtx_vector_builder
byte_indices (qimode
, u
, 1);
6101 for (i
= 0; i
< u
; ++i
)
6102 byte_indices
.quick_push (GEN_INT (i
));
6103 tmp
= byte_indices
.build ();
6104 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
6105 sel
, 0, OPTAB_DIRECT
);
6106 gcc_assert (sel_qi
!= NULL
);
6108 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
6109 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
6110 gen_lowpart (qimode
, v1
), sel_qi
);
6112 tmp
= gen_lowpart (mode
, tmp
);
6116 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
6117 Use TARGET for the result if nonnull and convenient. */
6120 expand_vec_series_expr (machine_mode vmode
, rtx op0
, rtx op1
, rtx target
)
6122 class expand_operand ops
[3];
6123 enum insn_code icode
;
6124 machine_mode emode
= GET_MODE_INNER (vmode
);
6126 icode
= direct_optab_handler (vec_series_optab
, vmode
);
6127 gcc_assert (icode
!= CODE_FOR_nothing
);
6129 create_output_operand (&ops
[0], target
, vmode
);
6130 create_input_operand (&ops
[1], op0
, emode
);
6131 create_input_operand (&ops
[2], op1
, emode
);
6133 expand_insn (icode
, 3, ops
);
6134 return ops
[0].value
;
6137 /* Generate insns for a vector comparison into a mask. */
6140 expand_vec_cmp_expr (tree type
, tree exp
, rtx target
)
6142 class expand_operand ops
[4];
6143 enum insn_code icode
;
6145 machine_mode mask_mode
= TYPE_MODE (type
);
6149 enum tree_code tcode
;
6151 op0a
= TREE_OPERAND (exp
, 0);
6152 op0b
= TREE_OPERAND (exp
, 1);
6153 tcode
= TREE_CODE (exp
);
6155 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
6156 vmode
= TYPE_MODE (TREE_TYPE (op0a
));
6158 icode
= get_vec_cmp_icode (vmode
, mask_mode
, unsignedp
);
6159 if (icode
== CODE_FOR_nothing
)
6161 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
6162 icode
= get_vec_cmp_eq_icode (vmode
, mask_mode
);
6163 if (icode
== CODE_FOR_nothing
)
6167 comparison
= vector_compare_rtx (mask_mode
, tcode
, op0a
, op0b
,
6168 unsignedp
, icode
, 2);
6169 create_output_operand (&ops
[0], target
, mask_mode
);
6170 create_fixed_operand (&ops
[1], comparison
);
6171 create_fixed_operand (&ops
[2], XEXP (comparison
, 0));
6172 create_fixed_operand (&ops
[3], XEXP (comparison
, 1));
6173 expand_insn (icode
, 4, ops
);
6174 return ops
[0].value
;
6177 /* Expand a highpart multiply. */
6180 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
6181 rtx target
, bool uns_p
)
6183 class expand_operand eops
[3];
6184 enum insn_code icode
;
6190 method
= can_mult_highpart_p (mode
, uns_p
);
6196 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
6197 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
6200 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
6201 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
6204 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
6205 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
6206 if (BYTES_BIG_ENDIAN
)
6207 std::swap (tab1
, tab2
);
6213 icode
= optab_handler (tab1
, mode
);
6214 wmode
= insn_data
[icode
].operand
[0].mode
;
6215 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode
),
6216 GET_MODE_NUNITS (mode
)));
6217 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode
), GET_MODE_SIZE (mode
)));
6219 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
6220 create_input_operand (&eops
[1], op0
, mode
);
6221 create_input_operand (&eops
[2], op1
, mode
);
6222 expand_insn (icode
, 3, eops
);
6223 m1
= gen_lowpart (mode
, eops
[0].value
);
6225 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
6226 create_input_operand (&eops
[1], op0
, mode
);
6227 create_input_operand (&eops
[2], op1
, mode
);
6228 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
6229 m2
= gen_lowpart (mode
, eops
[0].value
);
6231 vec_perm_builder sel
;
6234 /* The encoding has 2 interleaved stepped patterns. */
6235 sel
.new_vector (GET_MODE_NUNITS (mode
), 2, 3);
6236 for (i
= 0; i
< 6; ++i
)
6237 sel
.quick_push (!BYTES_BIG_ENDIAN
+ (i
& ~1)
6238 + ((i
& 1) ? GET_MODE_NUNITS (mode
) : 0));
6242 /* The encoding has a single interleaved stepped pattern. */
6243 sel
.new_vector (GET_MODE_NUNITS (mode
), 1, 3);
6244 for (i
= 0; i
< 3; ++i
)
6245 sel
.quick_push (2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1));
6248 return expand_vec_perm_const (mode
, m1
, m2
, sel
, BLKmode
, target
);
6251 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
6255 find_cc_set (rtx x
, const_rtx pat
, void *data
)
6257 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
6258 && GET_CODE (pat
) == SET
)
6260 rtx
*p_cc_reg
= (rtx
*) data
;
6261 gcc_assert (!*p_cc_reg
);
6266 /* This is a helper function for the other atomic operations. This function
6267 emits a loop that contains SEQ that iterates until a compare-and-swap
6268 operation at the end succeeds. MEM is the memory to be modified. SEQ is
6269 a set of instructions that takes a value from OLD_REG as an input and
6270 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6271 set to the current contents of MEM. After SEQ, a compare-and-swap will
6272 attempt to update MEM with NEW_REG. The function returns true when the
6273 loop was generated successfully. */
6276 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
6278 machine_mode mode
= GET_MODE (mem
);
6279 rtx_code_label
*label
;
6280 rtx cmp_reg
, success
, oldval
;
6282 /* The loop we want to generate looks like
6288 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6292 Note that we only do the plain load from memory once. Subsequent
6293 iterations use the value loaded by the compare-and-swap pattern. */
6295 label
= gen_label_rtx ();
6296 cmp_reg
= gen_reg_rtx (mode
);
6298 emit_move_insn (cmp_reg
, mem
);
6300 emit_move_insn (old_reg
, cmp_reg
);
6306 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
6307 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
6311 if (oldval
!= cmp_reg
)
6312 emit_move_insn (cmp_reg
, oldval
);
6314 /* Mark this jump predicted not taken. */
6315 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
6316 GET_MODE (success
), 1, label
,
6317 profile_probability::guessed_never ());
6322 /* This function tries to emit an atomic_exchange intruction. VAL is written
6323 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6324 using TARGET if possible. */
6327 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6329 machine_mode mode
= GET_MODE (mem
);
6330 enum insn_code icode
;
6332 /* If the target supports the exchange directly, great. */
6333 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
6334 if (icode
!= CODE_FOR_nothing
)
6336 class expand_operand ops
[4];
6338 create_output_operand (&ops
[0], target
, mode
);
6339 create_fixed_operand (&ops
[1], mem
);
6340 create_input_operand (&ops
[2], val
, mode
);
6341 create_integer_operand (&ops
[3], model
);
6342 if (maybe_expand_insn (icode
, 4, ops
))
6343 return ops
[0].value
;
6349 /* This function tries to implement an atomic exchange operation using
6350 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6351 The previous contents of *MEM are returned, using TARGET if possible.
6352 Since this instructionn is an acquire barrier only, stronger memory
6353 models may require additional barriers to be emitted. */
6356 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
6357 enum memmodel model
)
6359 machine_mode mode
= GET_MODE (mem
);
6360 enum insn_code icode
;
6361 rtx_insn
*last_insn
= get_last_insn ();
6363 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
6365 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6366 exists, and the memory model is stronger than acquire, add a release
6367 barrier before the instruction. */
6369 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
6370 expand_mem_thread_fence (model
);
6372 if (icode
!= CODE_FOR_nothing
)
6374 class expand_operand ops
[3];
6375 create_output_operand (&ops
[0], target
, mode
);
6376 create_fixed_operand (&ops
[1], mem
);
6377 create_input_operand (&ops
[2], val
, mode
);
6378 if (maybe_expand_insn (icode
, 3, ops
))
6379 return ops
[0].value
;
6382 /* If an external test-and-set libcall is provided, use that instead of
6383 any external compare-and-swap that we might get from the compare-and-
6384 swap-loop expansion later. */
6385 if (!can_compare_and_swap_p (mode
, false))
6387 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
6388 if (libfunc
!= NULL
)
6392 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6393 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6394 mode
, addr
, ptr_mode
,
6399 /* If the test_and_set can't be emitted, eliminate any barrier that might
6400 have been emitted. */
6401 delete_insns_since (last_insn
);
6405 /* This function tries to implement an atomic exchange operation using a
6406 compare_and_swap loop. VAL is written to *MEM. The previous contents of
6407 *MEM are returned, using TARGET if possible. No memory model is required
6408 since a compare_and_swap loop is seq-cst. */
6411 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
6413 machine_mode mode
= GET_MODE (mem
);
6415 if (can_compare_and_swap_p (mode
, true))
6417 if (!target
|| !register_operand (target
, mode
))
6418 target
= gen_reg_rtx (mode
);
6419 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
6426 /* This function tries to implement an atomic test-and-set operation
6427 using the atomic_test_and_set instruction pattern. A boolean value
6428 is returned from the operation, using TARGET if possible. */
6431 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6433 machine_mode pat_bool_mode
;
6434 class expand_operand ops
[3];
6436 if (!targetm
.have_atomic_test_and_set ())
6439 /* While we always get QImode from __atomic_test_and_set, we get
6440 other memory modes from __sync_lock_test_and_set. Note that we
6441 use no endian adjustment here. This matches the 4.6 behavior
6442 in the Sparc backend. */
6443 enum insn_code icode
= targetm
.code_for_atomic_test_and_set
;
6444 gcc_checking_assert (insn_data
[icode
].operand
[1].mode
== QImode
);
6445 if (GET_MODE (mem
) != QImode
)
6446 mem
= adjust_address_nv (mem
, QImode
, 0);
6448 pat_bool_mode
= insn_data
[icode
].operand
[0].mode
;
6449 create_output_operand (&ops
[0], target
, pat_bool_mode
);
6450 create_fixed_operand (&ops
[1], mem
);
6451 create_integer_operand (&ops
[2], model
);
6453 if (maybe_expand_insn (icode
, 3, ops
))
6454 return ops
[0].value
;
6458 /* This function expands the legacy _sync_lock test_and_set operation which is
6459 generally an atomic exchange. Some limited targets only allow the
6460 constant 1 to be stored. This is an ACQUIRE operation.
6462 TARGET is an optional place to stick the return value.
6463 MEM is where VAL is stored. */
6466 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
6470 /* Try an atomic_exchange first. */
6471 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
6475 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
6476 MEMMODEL_SYNC_ACQUIRE
);
6480 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6484 /* If there are no other options, try atomic_test_and_set if the value
6485 being stored is 1. */
6486 if (val
== const1_rtx
)
6487 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
6492 /* This function expands the atomic test_and_set operation:
6493 atomically store a boolean TRUE into MEM and return the previous value.
6495 MEMMODEL is the memory model variant to use.
6496 TARGET is an optional place to stick the return value. */
6499 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6501 machine_mode mode
= GET_MODE (mem
);
6502 rtx ret
, trueval
, subtarget
;
6504 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
6508 /* Be binary compatible with non-default settings of trueval, and different
6509 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6510 another only has atomic-exchange. */
6511 if (targetm
.atomic_test_and_set_trueval
== 1)
6513 trueval
= const1_rtx
;
6514 subtarget
= target
? target
: gen_reg_rtx (mode
);
6518 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
6519 subtarget
= gen_reg_rtx (mode
);
6522 /* Try the atomic-exchange optab... */
6523 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
6525 /* ... then an atomic-compare-and-swap loop ... */
6527 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
6529 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6531 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
6533 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6534 things with the value 1. Thus we try again without trueval. */
6535 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
6536 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
6538 /* Failing all else, assume a single threaded environment and simply
6539 perform the operation. */
6542 /* If the result is ignored skip the move to target. */
6543 if (subtarget
!= const0_rtx
)
6544 emit_move_insn (subtarget
, mem
);
6546 emit_move_insn (mem
, trueval
);
6550 /* Recall that have to return a boolean value; rectify if trueval
6551 is not exactly one. */
6552 if (targetm
.atomic_test_and_set_trueval
!= 1)
6553 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
6558 /* This function expands the atomic exchange operation:
6559 atomically store VAL in MEM and return the previous value in MEM.
6561 MEMMODEL is the memory model variant to use.
6562 TARGET is an optional place to stick the return value. */
6565 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6567 machine_mode mode
= GET_MODE (mem
);
6570 /* If loads are not atomic for the required size and we are not called to
6571 provide a __sync builtin, do not do anything so that we stay consistent
6572 with atomic loads of the same size. */
6573 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6576 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6578 /* Next try a compare-and-swap loop for the exchange. */
6580 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6585 /* This function expands the atomic compare exchange operation:
6587 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6588 *PTARGET_OVAL is an optional place to store the old value from memory.
6589 Both target parameters may be NULL or const0_rtx to indicate that we do
6590 not care about that return value. Both target parameters are updated on
6591 success to the actual location of the corresponding result.
6593 MEMMODEL is the memory model variant to use.
6595 The return value of the function is true for success. */
6598 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
6599 rtx mem
, rtx expected
, rtx desired
,
6600 bool is_weak
, enum memmodel succ_model
,
6601 enum memmodel fail_model
)
6603 machine_mode mode
= GET_MODE (mem
);
6604 class expand_operand ops
[8];
6605 enum insn_code icode
;
6606 rtx target_oval
, target_bool
= NULL_RTX
;
6609 /* If loads are not atomic for the required size and we are not called to
6610 provide a __sync builtin, do not do anything so that we stay consistent
6611 with atomic loads of the same size. */
6612 if (!can_atomic_load_p (mode
) && !is_mm_sync (succ_model
))
6615 /* Load expected into a register for the compare and swap. */
6616 if (MEM_P (expected
))
6617 expected
= copy_to_reg (expected
);
6619 /* Make sure we always have some place to put the return oldval.
6620 Further, make sure that place is distinct from the input expected,
6621 just in case we need that path down below. */
6622 if (ptarget_oval
&& *ptarget_oval
== const0_rtx
)
6623 ptarget_oval
= NULL
;
6625 if (ptarget_oval
== NULL
6626 || (target_oval
= *ptarget_oval
) == NULL
6627 || reg_overlap_mentioned_p (expected
, target_oval
))
6628 target_oval
= gen_reg_rtx (mode
);
6630 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
6631 if (icode
!= CODE_FOR_nothing
)
6633 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
6635 if (ptarget_bool
&& *ptarget_bool
== const0_rtx
)
6636 ptarget_bool
= NULL
;
6638 /* Make sure we always have a place for the bool operand. */
6639 if (ptarget_bool
== NULL
6640 || (target_bool
= *ptarget_bool
) == NULL
6641 || GET_MODE (target_bool
) != bool_mode
)
6642 target_bool
= gen_reg_rtx (bool_mode
);
6644 /* Emit the compare_and_swap. */
6645 create_output_operand (&ops
[0], target_bool
, bool_mode
);
6646 create_output_operand (&ops
[1], target_oval
, mode
);
6647 create_fixed_operand (&ops
[2], mem
);
6648 create_input_operand (&ops
[3], expected
, mode
);
6649 create_input_operand (&ops
[4], desired
, mode
);
6650 create_integer_operand (&ops
[5], is_weak
);
6651 create_integer_operand (&ops
[6], succ_model
);
6652 create_integer_operand (&ops
[7], fail_model
);
6653 if (maybe_expand_insn (icode
, 8, ops
))
6655 /* Return success/failure. */
6656 target_bool
= ops
[0].value
;
6657 target_oval
= ops
[1].value
;
6662 /* Otherwise fall back to the original __sync_val_compare_and_swap
6663 which is always seq-cst. */
6664 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
6665 if (icode
!= CODE_FOR_nothing
)
6669 create_output_operand (&ops
[0], target_oval
, mode
);
6670 create_fixed_operand (&ops
[1], mem
);
6671 create_input_operand (&ops
[2], expected
, mode
);
6672 create_input_operand (&ops
[3], desired
, mode
);
6673 if (!maybe_expand_insn (icode
, 4, ops
))
6676 target_oval
= ops
[0].value
;
6678 /* If the caller isn't interested in the boolean return value,
6679 skip the computation of it. */
6680 if (ptarget_bool
== NULL
)
6683 /* Otherwise, work out if the compare-and-swap succeeded. */
6685 if (have_insn_for (COMPARE
, CCmode
))
6686 note_stores (get_last_insn (), find_cc_set
, &cc_reg
);
6689 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
6690 const0_rtx
, VOIDmode
, 0, 1);
6693 goto success_bool_from_val
;
6696 /* Also check for library support for __sync_val_compare_and_swap. */
6697 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
6698 if (libfunc
!= NULL
)
6700 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6701 rtx target
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6702 mode
, addr
, ptr_mode
,
6703 expected
, mode
, desired
, mode
);
6704 emit_move_insn (target_oval
, target
);
6706 /* Compute the boolean return value only if requested. */
6708 goto success_bool_from_val
;
6716 success_bool_from_val
:
6717 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
6718 expected
, VOIDmode
, 1, 1);
6720 /* Make sure that the oval output winds up where the caller asked. */
6722 *ptarget_oval
= target_oval
;
6724 *ptarget_bool
= target_bool
;
6728 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
6731 expand_asm_memory_blockage (void)
6735 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
6736 rtvec_alloc (0), rtvec_alloc (0),
6737 rtvec_alloc (0), UNKNOWN_LOCATION
);
6738 MEM_VOLATILE_P (asm_op
) = 1;
6740 clob
= gen_rtx_SCRATCH (VOIDmode
);
6741 clob
= gen_rtx_MEM (BLKmode
, clob
);
6742 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
6744 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
6747 /* Do not propagate memory accesses across this point. */
6750 expand_memory_blockage (void)
6752 if (targetm
.have_memory_blockage ())
6753 emit_insn (targetm
.gen_memory_blockage ());
6755 expand_asm_memory_blockage ();
6758 /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
6759 same time clobbering the register set specified by REGS. */
6762 expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs
)
6764 rtx asm_op
, clob_mem
;
6766 unsigned int num_of_regs
= 0;
6767 for (unsigned int i
= 0; i
< FIRST_PSEUDO_REGISTER
; i
++)
6768 if (TEST_HARD_REG_BIT (regs
, i
))
6771 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
6772 rtvec_alloc (0), rtvec_alloc (0),
6773 rtvec_alloc (0), UNKNOWN_LOCATION
);
6774 MEM_VOLATILE_P (asm_op
) = 1;
6776 rtvec v
= rtvec_alloc (num_of_regs
+ 2);
6778 clob_mem
= gen_rtx_SCRATCH (VOIDmode
);
6779 clob_mem
= gen_rtx_MEM (BLKmode
, clob_mem
);
6780 clob_mem
= gen_rtx_CLOBBER (VOIDmode
, clob_mem
);
6782 RTVEC_ELT (v
, 0) = asm_op
;
6783 RTVEC_ELT (v
, 1) = clob_mem
;
6785 if (num_of_regs
> 0)
6788 for (unsigned int i
= 0; i
< FIRST_PSEUDO_REGISTER
; i
++)
6789 if (TEST_HARD_REG_BIT (regs
, i
))
6791 RTVEC_ELT (v
, j
) = gen_rtx_CLOBBER (VOIDmode
, regno_reg_rtx
[i
]);
6794 gcc_assert (j
== (num_of_regs
+ 2));
6797 emit_insn (gen_rtx_PARALLEL (VOIDmode
, v
));
6800 /* This routine will either emit the mem_thread_fence pattern or issue a
6801 sync_synchronize to generate a fence for memory model MEMMODEL. */
6804 expand_mem_thread_fence (enum memmodel model
)
6806 if (is_mm_relaxed (model
))
6808 if (targetm
.have_mem_thread_fence ())
6810 emit_insn (targetm
.gen_mem_thread_fence (GEN_INT (model
)));
6811 expand_memory_blockage ();
6813 else if (targetm
.have_memory_barrier ())
6814 emit_insn (targetm
.gen_memory_barrier ());
6815 else if (synchronize_libfunc
!= NULL_RTX
)
6816 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
);
6818 expand_memory_blockage ();
6821 /* Emit a signal fence with given memory model. */
6824 expand_mem_signal_fence (enum memmodel model
)
6826 /* No machine barrier is required to implement a signal fence, but
6827 a compiler memory barrier must be issued, except for relaxed MM. */
6828 if (!is_mm_relaxed (model
))
6829 expand_memory_blockage ();
6832 /* This function expands the atomic load operation:
6833 return the atomically loaded value in MEM.
6835 MEMMODEL is the memory model variant to use.
6836 TARGET is an option place to stick the return value. */
6839 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
6841 machine_mode mode
= GET_MODE (mem
);
6842 enum insn_code icode
;
6844 /* If the target supports the load directly, great. */
6845 icode
= direct_optab_handler (atomic_load_optab
, mode
);
6846 if (icode
!= CODE_FOR_nothing
)
6848 class expand_operand ops
[3];
6849 rtx_insn
*last
= get_last_insn ();
6850 if (is_mm_seq_cst (model
))
6851 expand_memory_blockage ();
6853 create_output_operand (&ops
[0], target
, mode
);
6854 create_fixed_operand (&ops
[1], mem
);
6855 create_integer_operand (&ops
[2], model
);
6856 if (maybe_expand_insn (icode
, 3, ops
))
6858 if (!is_mm_relaxed (model
))
6859 expand_memory_blockage ();
6860 return ops
[0].value
;
6862 delete_insns_since (last
);
6865 /* If the size of the object is greater than word size on this target,
6866 then we assume that a load will not be atomic. We could try to
6867 emulate a load with a compare-and-swap operation, but the store that
6868 doing this could result in would be incorrect if this is a volatile
6869 atomic load or targetting read-only-mapped memory. */
6870 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
6871 /* If there is no atomic load, leave the library call. */
6874 /* Otherwise assume loads are atomic, and emit the proper barriers. */
6875 if (!target
|| target
== const0_rtx
)
6876 target
= gen_reg_rtx (mode
);
6878 /* For SEQ_CST, emit a barrier before the load. */
6879 if (is_mm_seq_cst (model
))
6880 expand_mem_thread_fence (model
);
6882 emit_move_insn (target
, mem
);
6884 /* Emit the appropriate barrier after the load. */
6885 expand_mem_thread_fence (model
);
6890 /* This function expands the atomic store operation:
6891 Atomically store VAL in MEM.
6892 MEMMODEL is the memory model variant to use.
6893 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6894 function returns const0_rtx if a pattern was emitted. */
6897 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
6899 machine_mode mode
= GET_MODE (mem
);
6900 enum insn_code icode
;
6901 class expand_operand ops
[3];
6903 /* If the target supports the store directly, great. */
6904 icode
= direct_optab_handler (atomic_store_optab
, mode
);
6905 if (icode
!= CODE_FOR_nothing
)
6907 rtx_insn
*last
= get_last_insn ();
6908 if (!is_mm_relaxed (model
))
6909 expand_memory_blockage ();
6910 create_fixed_operand (&ops
[0], mem
);
6911 create_input_operand (&ops
[1], val
, mode
);
6912 create_integer_operand (&ops
[2], model
);
6913 if (maybe_expand_insn (icode
, 3, ops
))
6915 if (is_mm_seq_cst (model
))
6916 expand_memory_blockage ();
6919 delete_insns_since (last
);
6922 /* If using __sync_lock_release is a viable alternative, try it.
6923 Note that this will not be set to true if we are expanding a generic
6924 __atomic_store_n. */
6927 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
6928 if (icode
!= CODE_FOR_nothing
)
6930 create_fixed_operand (&ops
[0], mem
);
6931 create_input_operand (&ops
[1], const0_rtx
, mode
);
6932 if (maybe_expand_insn (icode
, 2, ops
))
6934 /* lock_release is only a release barrier. */
6935 if (is_mm_seq_cst (model
))
6936 expand_mem_thread_fence (model
);
6942 /* If the size of the object is greater than word size on this target,
6943 a default store will not be atomic. */
6944 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
6946 /* If loads are atomic or we are called to provide a __sync builtin,
6947 we can try a atomic_exchange and throw away the result. Otherwise,
6948 don't do anything so that we do not create an inconsistency between
6949 loads and stores. */
6950 if (can_atomic_load_p (mode
) || is_mm_sync (model
))
6952 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
6954 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
,
6962 /* Otherwise assume stores are atomic, and emit the proper barriers. */
6963 expand_mem_thread_fence (model
);
6965 emit_move_insn (mem
, val
);
6967 /* For SEQ_CST, also emit a barrier after the store. */
6968 if (is_mm_seq_cst (model
))
6969 expand_mem_thread_fence (model
);
6975 /* Structure containing the pointers and values required to process the
6976 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
6978 struct atomic_op_functions
6980 direct_optab mem_fetch_before
;
6981 direct_optab mem_fetch_after
;
6982 direct_optab mem_no_result
;
6985 direct_optab no_result
;
6986 enum rtx_code reverse_code
;
6990 /* Fill in structure pointed to by OP with the various optab entries for an
6991 operation of type CODE. */
6994 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
6996 gcc_assert (op
!= NULL
);
6998 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6999 in the source code during compilation, and the optab entries are not
7000 computable until runtime. Fill in the values at runtime. */
7004 op
->mem_fetch_before
= atomic_fetch_add_optab
;
7005 op
->mem_fetch_after
= atomic_add_fetch_optab
;
7006 op
->mem_no_result
= atomic_add_optab
;
7007 op
->fetch_before
= sync_old_add_optab
;
7008 op
->fetch_after
= sync_new_add_optab
;
7009 op
->no_result
= sync_add_optab
;
7010 op
->reverse_code
= MINUS
;
7013 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
7014 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
7015 op
->mem_no_result
= atomic_sub_optab
;
7016 op
->fetch_before
= sync_old_sub_optab
;
7017 op
->fetch_after
= sync_new_sub_optab
;
7018 op
->no_result
= sync_sub_optab
;
7019 op
->reverse_code
= PLUS
;
7022 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
7023 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
7024 op
->mem_no_result
= atomic_xor_optab
;
7025 op
->fetch_before
= sync_old_xor_optab
;
7026 op
->fetch_after
= sync_new_xor_optab
;
7027 op
->no_result
= sync_xor_optab
;
7028 op
->reverse_code
= XOR
;
7031 op
->mem_fetch_before
= atomic_fetch_and_optab
;
7032 op
->mem_fetch_after
= atomic_and_fetch_optab
;
7033 op
->mem_no_result
= atomic_and_optab
;
7034 op
->fetch_before
= sync_old_and_optab
;
7035 op
->fetch_after
= sync_new_and_optab
;
7036 op
->no_result
= sync_and_optab
;
7037 op
->reverse_code
= UNKNOWN
;
7040 op
->mem_fetch_before
= atomic_fetch_or_optab
;
7041 op
->mem_fetch_after
= atomic_or_fetch_optab
;
7042 op
->mem_no_result
= atomic_or_optab
;
7043 op
->fetch_before
= sync_old_ior_optab
;
7044 op
->fetch_after
= sync_new_ior_optab
;
7045 op
->no_result
= sync_ior_optab
;
7046 op
->reverse_code
= UNKNOWN
;
7049 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
7050 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
7051 op
->mem_no_result
= atomic_nand_optab
;
7052 op
->fetch_before
= sync_old_nand_optab
;
7053 op
->fetch_after
= sync_new_nand_optab
;
7054 op
->no_result
= sync_nand_optab
;
7055 op
->reverse_code
= UNKNOWN
;
7062 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7063 using memory order MODEL. If AFTER is true the operation needs to return
7064 the value of *MEM after the operation, otherwise the previous value.
7065 TARGET is an optional place to place the result. The result is unused if
7067 Return the result if there is a better sequence, otherwise NULL_RTX. */
7070 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
7071 enum memmodel model
, bool after
)
7073 /* If the value is prefetched, or not used, it may be possible to replace
7074 the sequence with a native exchange operation. */
7075 if (!after
|| target
== const0_rtx
)
7077 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7078 if (code
== AND
&& val
== const0_rtx
)
7080 if (target
== const0_rtx
)
7081 target
= gen_reg_rtx (GET_MODE (mem
));
7082 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7085 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7086 if (code
== IOR
&& val
== constm1_rtx
)
7088 if (target
== const0_rtx
)
7089 target
= gen_reg_rtx (GET_MODE (mem
));
7090 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7097 /* Try to emit an instruction for a specific operation varaition.
7098 OPTAB contains the OP functions.
7099 TARGET is an optional place to return the result. const0_rtx means unused.
7100 MEM is the memory location to operate on.
7101 VAL is the value to use in the operation.
7102 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7103 MODEL is the memory model, if used.
7104 AFTER is true if the returned result is the value after the operation. */
7107 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
7108 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
7110 machine_mode mode
= GET_MODE (mem
);
7111 class expand_operand ops
[4];
7112 enum insn_code icode
;
7116 /* Check to see if there is a result returned. */
7117 if (target
== const0_rtx
)
7121 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
7122 create_integer_operand (&ops
[2], model
);
7127 icode
= direct_optab_handler (optab
->no_result
, mode
);
7131 /* Otherwise, we need to generate a result. */
7136 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
7137 : optab
->mem_fetch_before
, mode
);
7138 create_integer_operand (&ops
[3], model
);
7143 icode
= optab_handler (after
? optab
->fetch_after
7144 : optab
->fetch_before
, mode
);
7147 create_output_operand (&ops
[op_counter
++], target
, mode
);
7149 if (icode
== CODE_FOR_nothing
)
7152 create_fixed_operand (&ops
[op_counter
++], mem
);
7153 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7154 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
7156 if (maybe_expand_insn (icode
, num_ops
, ops
))
7157 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
7163 /* This function expands an atomic fetch_OP or OP_fetch operation:
7164 TARGET is an option place to stick the return value. const0_rtx indicates
7165 the result is unused.
7166 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7167 CODE is the operation being performed (OP)
7168 MEMMODEL is the memory model variant to use.
7169 AFTER is true to return the result of the operation (OP_fetch).
7170 AFTER is false to return the value before the operation (fetch_OP).
7172 This function will *only* generate instructions if there is a direct
7173 optab. No compare and swap loops or libcalls will be generated. */
7176 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
7177 enum rtx_code code
, enum memmodel model
,
7180 machine_mode mode
= GET_MODE (mem
);
7181 struct atomic_op_functions optab
;
7183 bool unused_result
= (target
== const0_rtx
);
7185 get_atomic_op_for_code (&optab
, code
);
7187 /* Check to see if there are any better instructions. */
7188 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
7192 /* Check for the case where the result isn't used and try those patterns. */
7195 /* Try the memory model variant first. */
7196 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
7200 /* Next try the old style withuot a memory model. */
7201 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
7205 /* There is no no-result pattern, so try patterns with a result. */
7209 /* Try the __atomic version. */
7210 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
7214 /* Try the older __sync version. */
7215 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
7219 /* If the fetch value can be calculated from the other variation of fetch,
7220 try that operation. */
7221 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
7223 /* Try the __atomic version, then the older __sync version. */
7224 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
7226 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
7230 /* If the result isn't used, no need to do compensation code. */
7234 /* Issue compensation code. Fetch_after == fetch_before OP val.
7235 Fetch_before == after REVERSE_OP val. */
7237 code
= optab
.reverse_code
;
7240 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
7241 true, OPTAB_LIB_WIDEN
);
7242 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
7245 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
7246 true, OPTAB_LIB_WIDEN
);
7251 /* No direct opcode can be generated. */
7257 /* This function expands an atomic fetch_OP or OP_fetch operation:
7258 TARGET is an option place to stick the return value. const0_rtx indicates
7259 the result is unused.
7260 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7261 CODE is the operation being performed (OP)
7262 MEMMODEL is the memory model variant to use.
7263 AFTER is true to return the result of the operation (OP_fetch).
7264 AFTER is false to return the value before the operation (fetch_OP). */
7266 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
7267 enum memmodel model
, bool after
)
7269 machine_mode mode
= GET_MODE (mem
);
7271 bool unused_result
= (target
== const0_rtx
);
7273 /* If loads are not atomic for the required size and we are not called to
7274 provide a __sync builtin, do not do anything so that we stay consistent
7275 with atomic loads of the same size. */
7276 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
7279 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
7285 /* Add/sub can be implemented by doing the reverse operation with -(val). */
7286 if (code
== PLUS
|| code
== MINUS
)
7289 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
7292 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
7293 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
7297 /* PLUS worked so emit the insns and return. */
7304 /* PLUS did not work, so throw away the negation code and continue. */
7308 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7309 if (!can_compare_and_swap_p (mode
, false))
7313 enum rtx_code orig_code
= code
;
7314 struct atomic_op_functions optab
;
7316 get_atomic_op_for_code (&optab
, code
);
7317 libfunc
= optab_libfunc (after
? optab
.fetch_after
7318 : optab
.fetch_before
, mode
);
7320 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
7324 code
= optab
.reverse_code
;
7325 libfunc
= optab_libfunc (after
? optab
.fetch_before
7326 : optab
.fetch_after
, mode
);
7328 if (libfunc
!= NULL
)
7330 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
7331 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
7332 addr
, ptr_mode
, val
, mode
);
7334 if (!unused_result
&& fixup
)
7335 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
7336 true, OPTAB_LIB_WIDEN
);
7340 /* We need the original code for any further attempts. */
7344 /* If nothing else has succeeded, default to a compare and swap loop. */
7345 if (can_compare_and_swap_p (mode
, true))
7348 rtx t0
= gen_reg_rtx (mode
), t1
;
7352 /* If the result is used, get a register for it. */
7355 if (!target
|| !register_operand (target
, mode
))
7356 target
= gen_reg_rtx (mode
);
7357 /* If fetch_before, copy the value now. */
7359 emit_move_insn (target
, t0
);
7362 target
= const0_rtx
;
7367 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
7368 true, OPTAB_LIB_WIDEN
);
7369 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
7372 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
7375 /* For after, copy the value now. */
7376 if (!unused_result
&& after
)
7377 emit_move_insn (target
, t1
);
7378 insn
= get_insns ();
7381 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
7388 /* Return true if OPERAND is suitable for operand number OPNO of
7389 instruction ICODE. */
7392 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
7394 return (!insn_data
[(int) icode
].operand
[opno
].predicate
7395 || (insn_data
[(int) icode
].operand
[opno
].predicate
7396 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
7399 /* TARGET is a target of a multiword operation that we are going to
7400 implement as a series of word-mode operations. Return true if
7401 TARGET is suitable for this purpose. */
7404 valid_multiword_target_p (rtx target
)
7409 mode
= GET_MODE (target
);
7410 if (!GET_MODE_SIZE (mode
).is_constant (&size
))
7412 for (i
= 0; i
< size
; i
+= UNITS_PER_WORD
)
7413 if (!validate_subreg (word_mode
, mode
, target
, i
))
7418 /* Make OP describe an input operand that has value INTVAL and that has
7419 no inherent mode. This function should only be used for operands that
7420 are always expand-time constants. The backend may request that INTVAL
7421 be copied into a different kind of rtx, but it must specify the mode
7422 of that rtx if so. */
7425 create_integer_operand (class expand_operand
*op
, poly_int64 intval
)
7427 create_expand_operand (op
, EXPAND_INTEGER
,
7428 gen_int_mode (intval
, MAX_MODE_INT
),
7429 VOIDmode
, false, intval
);
7432 /* Like maybe_legitimize_operand, but do not change the code of the
7433 current rtx value. */
7436 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
7437 class expand_operand
*op
)
7439 /* See if the operand matches in its current form. */
7440 if (insn_operand_matches (icode
, opno
, op
->value
))
7443 /* If the operand is a memory whose address has no side effects,
7444 try forcing the address into a non-virtual pseudo register.
7445 The check for side effects is important because copy_to_mode_reg
7446 cannot handle things like auto-modified addresses. */
7447 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
7452 addr
= XEXP (mem
, 0);
7453 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
7454 && !side_effects_p (addr
))
7459 last
= get_last_insn ();
7460 mode
= get_address_mode (mem
);
7461 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
7462 if (insn_operand_matches (icode
, opno
, mem
))
7467 delete_insns_since (last
);
7474 /* Try to make OP match operand OPNO of instruction ICODE. Return true
7475 on success, storing the new operand value back in OP. */
7478 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
7479 class expand_operand
*op
)
7481 machine_mode mode
, imode
, tmode
;
7488 temporary_volatile_ok
v (true);
7489 return maybe_legitimize_operand_same_code (icode
, opno
, op
);
7493 gcc_assert (mode
!= VOIDmode
);
7495 && op
->value
!= const0_rtx
7496 && GET_MODE (op
->value
) == mode
7497 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
7500 op
->value
= gen_reg_rtx (mode
);
7506 gcc_assert (mode
!= VOIDmode
);
7507 gcc_assert (GET_MODE (op
->value
) == VOIDmode
7508 || GET_MODE (op
->value
) == mode
);
7509 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
7512 op
->value
= copy_to_mode_reg (mode
, op
->value
);
7515 case EXPAND_CONVERT_TO
:
7516 gcc_assert (mode
!= VOIDmode
);
7517 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
7520 case EXPAND_CONVERT_FROM
:
7521 if (GET_MODE (op
->value
) != VOIDmode
)
7522 mode
= GET_MODE (op
->value
);
7524 /* The caller must tell us what mode this value has. */
7525 gcc_assert (mode
!= VOIDmode
);
7527 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7528 tmode
= (VECTOR_MODE_P (imode
) && !VECTOR_MODE_P (mode
)
7529 ? GET_MODE_INNER (imode
) : imode
);
7530 if (tmode
!= VOIDmode
&& tmode
!= mode
)
7532 op
->value
= convert_modes (tmode
, mode
, op
->value
, op
->unsigned_p
);
7535 if (imode
!= VOIDmode
&& imode
!= mode
)
7537 gcc_assert (VECTOR_MODE_P (imode
) && !VECTOR_MODE_P (mode
));
7538 op
->value
= expand_vector_broadcast (imode
, op
->value
);
7543 case EXPAND_ADDRESS
:
7544 op
->value
= convert_memory_address (as_a
<scalar_int_mode
> (mode
),
7548 case EXPAND_INTEGER
:
7549 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7550 if (mode
!= VOIDmode
7551 && known_eq (trunc_int_for_mode (op
->int_value
, mode
),
7554 op
->value
= gen_int_mode (op
->int_value
, mode
);
7559 return insn_operand_matches (icode
, opno
, op
->value
);
7562 /* Make OP describe an input operand that should have the same value
7563 as VALUE, after any mode conversion that the target might request.
7564 TYPE is the type of VALUE. */
7567 create_convert_operand_from_type (class expand_operand
*op
,
7568 rtx value
, tree type
)
7570 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
7571 TYPE_UNSIGNED (type
));
7574 /* Return true if the requirements on operands OP1 and OP2 of instruction
7575 ICODE are similar enough for the result of legitimizing OP1 to be
7576 reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
7577 with OP1 and OP2 respectively. */
7580 can_reuse_operands_p (enum insn_code icode
,
7581 unsigned int opno1
, unsigned int opno2
,
7582 const class expand_operand
*op1
,
7583 const class expand_operand
*op2
)
7585 /* Check requirements that are common to all types. */
7586 if (op1
->type
!= op2
->type
7587 || op1
->mode
!= op2
->mode
7588 || (insn_data
[(int) icode
].operand
[opno1
].mode
7589 != insn_data
[(int) icode
].operand
[opno2
].mode
))
7592 /* Check the requirements for specific types. */
7596 /* Outputs must remain distinct. */
7601 case EXPAND_ADDRESS
:
7602 case EXPAND_INTEGER
:
7605 case EXPAND_CONVERT_TO
:
7606 case EXPAND_CONVERT_FROM
:
7607 return op1
->unsigned_p
== op2
->unsigned_p
;
7612 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7613 of instruction ICODE. Return true on success, leaving the new operand
7614 values in the OPS themselves. Emit no code on failure. */
7617 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
7618 unsigned int nops
, class expand_operand
*ops
)
7620 rtx_insn
*last
= get_last_insn ();
7621 rtx
*orig_values
= XALLOCAVEC (rtx
, nops
);
7622 for (unsigned int i
= 0; i
< nops
; i
++)
7624 orig_values
[i
] = ops
[i
].value
;
7626 /* First try reusing the result of an earlier legitimization.
7627 This avoids duplicate rtl and ensures that tied operands
7630 This search is linear, but NOPS is bounded at compile time
7631 to a small number (current a single digit). */
7634 if (can_reuse_operands_p (icode
, opno
+ j
, opno
+ i
, &ops
[j
], &ops
[i
])
7635 && rtx_equal_p (orig_values
[j
], orig_values
[i
])
7637 && insn_operand_matches (icode
, opno
+ i
, ops
[j
].value
))
7639 ops
[i
].value
= copy_rtx (ops
[j
].value
);
7643 /* Otherwise try legitimizing the operand on its own. */
7644 if (j
== i
&& !maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
7646 delete_insns_since (last
);
7653 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7654 as its operands. Return the instruction pattern on success,
7655 and emit any necessary set-up code. Return null and emit no
7659 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
7660 class expand_operand
*ops
)
7662 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
7663 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
7669 return GEN_FCN (icode
) (ops
[0].value
);
7671 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
7673 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
7675 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7678 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7679 ops
[3].value
, ops
[4].value
);
7681 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7682 ops
[3].value
, ops
[4].value
, ops
[5].value
);
7684 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7685 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7688 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7689 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7690 ops
[6].value
, ops
[7].value
);
7692 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7693 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7694 ops
[6].value
, ops
[7].value
, ops
[8].value
);
7699 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7700 as its operands. Return true on success and emit no code on failure. */
7703 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
7704 class expand_operand
*ops
)
7706 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7715 /* Like maybe_expand_insn, but for jumps. */
7718 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7719 class expand_operand
*ops
)
7721 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7724 emit_jump_insn (pat
);
7730 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7734 expand_insn (enum insn_code icode
, unsigned int nops
,
7735 class expand_operand
*ops
)
7737 if (!maybe_expand_insn (icode
, nops
, ops
))
7741 /* Like expand_insn, but for jumps. */
7744 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7745 class expand_operand
*ops
)
7747 if (!maybe_expand_jump_insn (icode
, nops
, ops
))